From 8aaafec663034fa245efef0039a2efdeaca52e6c Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 3 Apr 2026 17:41:56 +0800 Subject: [PATCH 01/35] UCT/ZE/ZE_IPC: enable level zero ipc support for Intel GPUs --- src/uct/ze/Makefile.am | 12 +- src/uct/ze/base/ze_base.c | 42 ++ src/uct/ze/base/ze_base.h | 12 + src/uct/ze/copy/ze_copy_ep.c | 10 +- src/uct/ze/copy/ze_copy_iface.c | 6 + src/uct/ze/ze_ipc/ze_ipc_cache.c | 448 ++++++++++++++++++ src/uct/ze/ze_ipc/ze_ipc_cache.h | 98 ++++ src/uct/ze/ze_ipc/ze_ipc_ep.c | 355 +++++++++++++++ src/uct/ze/ze_ipc/ze_ipc_ep.h | 43 ++ src/uct/ze/ze_ipc/ze_ipc_iface.c | 759 +++++++++++++++++++++++++++++++ src/uct/ze/ze_ipc/ze_ipc_iface.h | 98 ++++ src/uct/ze/ze_ipc/ze_ipc_md.c | 328 +++++++++++++ src/uct/ze/ze_ipc/ze_ipc_md.h | 47 ++ 13 files changed, 2254 insertions(+), 4 deletions(-) create mode 100644 src/uct/ze/ze_ipc/ze_ipc_cache.c create mode 100644 src/uct/ze/ze_ipc/ze_ipc_cache.h create mode 100644 src/uct/ze/ze_ipc/ze_ipc_ep.c create mode 100644 src/uct/ze/ze_ipc/ze_ipc_ep.h create mode 100644 src/uct/ze/ze_ipc/ze_ipc_iface.c create mode 100644 src/uct/ze/ze_ipc/ze_ipc_iface.h create mode 100644 src/uct/ze/ze_ipc/ze_ipc_md.c create mode 100644 src/uct/ze/ze_ipc/ze_ipc_md.h diff --git a/src/uct/ze/Makefile.am b/src/uct/ze/Makefile.am index 6b7ea473bab..02224208788 100644 --- a/src/uct/ze/Makefile.am +++ b/src/uct/ze/Makefile.am @@ -27,12 +27,20 @@ libuct_ze_la_SOURCES = \ noinst_HEADERS += \ copy/ze_copy_md.h \ copy/ze_copy_iface.h \ - copy/ze_copy_ep.h + copy/ze_copy_ep.h \ + ze_ipc/ze_ipc_md.h \ + ze_ipc/ze_ipc_iface.h \ + ze_ipc/ze_ipc_ep.h \ + ze_ipc/ze_ipc_cache.h libuct_ze_la_SOURCES += \ copy/ze_copy_md.c \ copy/ze_copy_iface.c \ - copy/ze_copy_ep.c + copy/ze_copy_ep.c \ + ze_ipc/ze_ipc_md.c \ + ze_ipc/ze_ipc_iface.c \ + ze_ipc/ze_ipc_ep.c \ + ze_ipc/ze_ipc_cache.c PKG_CONFIG_NAME=ze diff --git a/src/uct/ze/base/ze_base.c b/src/uct/ze/base/ze_base.c index ab09ae89dd7..d105887a2a8 100644 --- a/src/uct/ze/base/ze_base.c +++ b/src/uct/ze/base/ze_base.c @@ -374,6 +374,33 @@ const uct_ze_subdevice_t *uct_ze_base_get_subdevice_by_global_id(int global_id) return &ze_subdevices[global_id]; } +int uct_ze_base_get_device_ordinal(ze_device_handle_t device) +{ + int i; + + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + return -1; + } + + for (i = 0; i < uct_ze_base.num_devices; i++) { + if (uct_ze_base.devices[i].root_device == device) { + return i; + } + } + + return -1; /* not found */ +} + + +int uct_ze_base_get_num_devices(void) +{ + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + return 0; + } + + return uct_ze_base.num_devices; +} + ze_device_handle_t uct_ze_base_get_device_handle_from_subdevice(const uct_ze_subdevice_t *subdevice) { @@ -512,6 +539,21 @@ static void UCS_F_DTOR uct_ze_base_cleanup(void) /* ZE driver cleanup happens automatically on process exit */ } + +ucs_status_t +uct_ze_base_query_devices_common(uct_md_h md, uct_device_type_t dev_type, + uct_tl_device_resource_t **tl_devices_p, + unsigned *num_tl_devices_p) +{ + ucs_info("ze_base: query_devices_common for md component=%s type=%d", + md->component->name, dev_type); + + return uct_single_device_resource(md, md->component->name, dev_type, + UCS_SYS_DEVICE_ID_UNKNOWN, tl_devices_p, + num_tl_devices_p); +} + + UCS_MODULE_INIT() { return UCS_OK; diff --git a/src/uct/ze/base/ze_base.h b/src/uct/ze/base/ze_base.h index 40ca69f3346..f2f4118189e 100644 --- a/src/uct/ze/base/ze_base.h +++ b/src/uct/ze/base/ze_base.h @@ -65,6 +65,13 @@ ze_device_handle_t uct_ze_base_get_device_handle_from_subdevice( ucs_sys_device_t uct_ze_base_get_sys_dev_from_handle(ze_device_handle_t device); +/* Get device ordinal from device handle, returns -1 if not found */ +int uct_ze_base_get_device_ordinal(ze_device_handle_t device); + + +int uct_ze_base_get_num_devices(void); + + ucs_status_t uct_ze_base_query_md_resources(uct_component_h component, uct_md_resource_desc_t **resources_p, @@ -74,4 +81,9 @@ ucs_status_t uct_ze_base_query_devices(uct_md_h md, uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p); +ucs_status_t +uct_ze_base_query_devices_common(uct_md_h md, uct_device_type_t dev_type, + uct_tl_device_resource_t **tl_devices_p, + unsigned *num_tl_devices_p); + #endif diff --git a/src/uct/ze/copy/ze_copy_ep.c b/src/uct/ze/copy/ze_copy_ep.c index 8cc89371c51..a46611b9aeb 100644 --- a/src/uct/ze/copy/ze_copy_ep.c +++ b/src/uct/ze/copy/ze_copy_ep.c @@ -50,8 +50,8 @@ ucs_status_t uct_ze_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr, ze_result_t ret; void *src, *dst; - ucs_trace("remote addr %p rkey %p size %zu", (void*)remote_addr, - (void*)rkey, size); + ucs_info("ze_copy_ep: zcopy %s remote_addr=%p local_buf=%p size=%zu", + is_put ? "PUT" : "GET", (void*)remote_addr, iov->buffer, size); if (is_put) { src = iov->buffer; @@ -147,6 +147,9 @@ ucs_status_t uct_ze_copy_ep_put_short(uct_ep_h tl_ep, const void *buffer, }; ucs_status_t status; + ucs_info("ze_copy_ep: PUT_SHORT buffer=%p length=%u remote_addr=%p", + buffer, length, (void*)remote_addr); + status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 1); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), PUT, SHORT, length); @@ -166,6 +169,9 @@ ucs_status_t uct_ze_copy_ep_get_short(uct_ep_h tl_ep, void *buffer, }; ucs_status_t status; + ucs_info("ze_copy_ep: GET_SHORT buffer=%p length=%u remote_addr=%p", + buffer, length, (void*)remote_addr); + status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 0); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), GET, SHORT, length); diff --git a/src/uct/ze/copy/ze_copy_iface.c b/src/uct/ze/copy/ze_copy_iface.c index fbc6ed4c8a0..8b7e65e1e06 100644 --- a/src/uct/ze/copy/ze_copy_iface.c +++ b/src/uct/ze/copy/ze_copy_iface.c @@ -259,16 +259,19 @@ static UCS_CLASS_INIT_FUNC(uct_ze_copy_iface_t, uct_md_h md, /* Use the device configured in the MD */ device = ze_md->ze_device; if (device == NULL) { + ucs_error("ze_copy_iface: uct_ze_base_get_device(0) returned NULL"); return UCS_ERR_NO_DEVICE; } ret = zeCommandQueueCreate(ze_md->ze_context, device, &cq_desc, &cmdq); if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_copy_iface: zeCommandQueueCreate failed with 0x%x", ret); return UCS_ERR_NO_DEVICE; } ret = zeCommandListCreate(ze_md->ze_context, device, &cl_desc, &cmdl); if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_copy_iface: zeCommandListCreate failed with 0x%x", ret); zeCommandQueueDestroy(cmdq); return UCS_ERR_NO_DEVICE; } @@ -277,6 +280,9 @@ static UCS_CLASS_INIT_FUNC(uct_ze_copy_iface_t, uct_md_h md, self->ze_cmdl = cmdl; self->id = ucs_generate_uuid((uintptr_t)self); + ucs_info("ze_copy_iface: initialized iface device=%p cmdq=%p cmdl=%p", + device, cmdq, cmdl); + return UCS_OK; } diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c new file mode 100644 index 00000000000..8bdd77a683d --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -0,0 +1,448 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "ze_ipc_cache.h" +#include "ze_ipc_iface.h" +#include "ze_ipc_ep.h" +#include +#include +#include +#include +#include +#include +#include +#include + + +typedef struct uct_ze_ipc_cache_hash_key { + pid_t pid; + uint32_t _pad; /* Ensure 8-byte alignment for ze_context */ + ze_context_handle_t ze_context; +} UCS_S_PACKED uct_ze_ipc_cache_hash_key_t; + + +static UCS_F_ALWAYS_INLINE int +uct_ze_ipc_cache_hash_equal(uct_ze_ipc_cache_hash_key_t key1, + uct_ze_ipc_cache_hash_key_t key2) +{ + return (key1.pid == key2.pid) && (key1.ze_context == key2.ze_context); +} + + +static UCS_F_ALWAYS_INLINE khint32_t +uct_ze_ipc_cache_hash_func(uct_ze_ipc_cache_hash_key_t key) +{ + return kh_int_hash_func((uintptr_t)key.pid ^ (uintptr_t)key.ze_context); +} + + +KHASH_INIT(ze_ipc_rem_cache, uct_ze_ipc_cache_hash_key_t, + uct_ze_ipc_cache_t*, 1, uct_ze_ipc_cache_hash_func, + uct_ze_ipc_cache_hash_equal); + + +typedef struct uct_ze_ipc_remote_cache { + khash_t(ze_ipc_rem_cache) hash; + ucs_recursive_spinlock_t lock; +} uct_ze_ipc_remote_cache_t; + + +static uct_ze_ipc_remote_cache_t uct_ze_ipc_remote_cache; + + +static ucs_pgt_dir_t *uct_ze_ipc_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) +{ + void *ptr; + int ret; + + ret = ucs_posix_memalign(&ptr, + ucs_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), + sizeof(ucs_pgt_dir_t), "ze_ipc_cache_pgdir"); + return (ret == 0) ? ptr : NULL; +} + + +static void uct_ze_ipc_cache_pgt_dir_release(const ucs_pgtable_t *pgtable, + ucs_pgt_dir_t *dir) +{ + ucs_free(dir); +} + + +static void +uct_ze_ipc_cache_region_collect_callback(const ucs_pgtable_t *pgtable, + ucs_pgt_region_t *pgt_region, + void *arg) +{ + ucs_list_link_t *list = arg; + uct_ze_ipc_cache_region_t *region; + + region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + ucs_list_add_tail(list, ®ion->list); +} + + +static ucs_status_t +uct_ze_ipc_close_memhandle(uct_ze_ipc_cache_region_t *region) +{ + ze_result_t ret; + + ret = zeMemCloseIpcHandle(region->ze_context, region->mapped_addr); + if (ret != ZE_RESULT_SUCCESS) { + ucs_warn("zeMemCloseIpcHandle failed with error 0x%x", ret); + return UCS_ERR_IO_ERROR; + } + + if (region->dup_fd >= 0) { + close(region->dup_fd); + } + + return UCS_OK; +} + + +static void uct_ze_ipc_cache_purge(uct_ze_ipc_cache_t *cache) +{ + uct_ze_ipc_cache_region_t *region, *tmp; + ucs_list_link_t region_list; + + ucs_list_head_init(®ion_list); + ucs_pgtable_purge(&cache->pgtable, uct_ze_ipc_cache_region_collect_callback, + ®ion_list); + ucs_list_for_each_safe(region, tmp, ®ion_list, list) { + uct_ze_ipc_close_memhandle(region); + ucs_free(region); + } + ucs_trace("%s: ze ipc cache purged", cache->name); +} + + +static ucs_status_t +uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + void **mapped_addr, int *dup_fd) +{ + ze_ipc_mem_handle_t local_handle; + int remote_fd; + ze_result_t ret; + + /* Extract fd from IPC handle */ + memcpy(&local_handle, &key->ipc_handle, sizeof(local_handle)); + remote_fd = *(int*)local_handle.data; + + /* Duplicate the file descriptor from remote process */ + if (key->pid != getpid() && remote_fd > 0 && remote_fd < 65536) { + *dup_fd = uct_ze_ipc_dup_fd_from_pid(key->pid, remote_fd); + if (*dup_fd < 0) { + ucs_error("failed to duplicate fd %d from pid %d", remote_fd, key->pid); + return UCS_ERR_IO_ERROR; + } + /* Update handle with local fd */ + *(int*)local_handle.data = *dup_fd; + } else { + *dup_fd = -1; + } + + ret = zeMemOpenIpcHandle(ze_context, ze_device, local_handle, + 0, mapped_addr); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeMemOpenIpcHandle failed with error 0x%x", ret); + if (*dup_fd >= 0) { + close(*dup_fd); + *dup_fd = -1; + } + return UCS_ERR_IO_ERROR; + } + + return UCS_OK; +} + + +static void uct_ze_ipc_cache_invalidate_regions(uct_ze_ipc_cache_t *cache, + void *from, void *to) +{ + ucs_list_link_t region_list; + ucs_status_t status; + uct_ze_ipc_cache_region_t *region, *tmp; + + ucs_list_head_init(®ion_list); + ucs_pgtable_search_range(&cache->pgtable, (ucs_pgt_addr_t)from, + (ucs_pgt_addr_t)to - 1, + uct_ze_ipc_cache_region_collect_callback, + ®ion_list); + ucs_list_for_each_safe(region, tmp, ®ion_list, list) { + status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_error("failed to remove address:%p from cache (%s)", + (void *)region->key.address, ucs_status_string(status)); + } + + status = uct_ze_ipc_close_memhandle(region); + if (status != UCS_OK) { + ucs_error("failed to close memhandle for base addr:%p (%s)", + (void *)region->key.address, ucs_status_string(status)); + } + + ucs_free(region); + } + ucs_trace("%s: closed memhandles in the range [%p..%p]", + cache->name, from, to); +} + + +static ucs_status_t +uct_ze_ipc_get_remote_cache(pid_t pid, ze_context_handle_t ze_context, + uct_ze_ipc_cache_t **cache) +{ + ucs_status_t status = UCS_OK; + char target_name[64]; + uct_ze_ipc_cache_hash_key_t key; + khiter_t khiter; + int khret; + + ucs_recursive_spin_lock(&uct_ze_ipc_remote_cache.lock); + + key.ze_context = ze_context; + key.pid = pid; + key._pad = 0; + + khiter = kh_put(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, key, + &khret); + if ((khret == UCS_KH_PUT_BUCKET_EMPTY) || + (khret == UCS_KH_PUT_BUCKET_CLEAR)) { + ucs_snprintf_safe(target_name, sizeof(target_name), "dest:%d:%p", + key.pid, key.ze_context); + status = uct_ze_ipc_create_cache(cache, target_name); + if (status != UCS_OK) { + kh_del(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, khiter); + ucs_error("could not create ze ipc cache: %s", + ucs_status_string(status)); + goto err_unlock; + } + + kh_val(&uct_ze_ipc_remote_cache.hash, khiter) = *cache; + } else if (khret == UCS_KH_PUT_KEY_PRESENT) { + *cache = kh_val(&uct_ze_ipc_remote_cache.hash, khiter); + } else { + ucs_error("unable to use ze_ipc remote_cache hash"); + status = UCS_ERR_NO_RESOURCE; + } +err_unlock: + ucs_recursive_spin_unlock(&uct_ze_ipc_remote_cache.lock); + return status; +} + + +ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, + void *mapped_addr, + ze_context_handle_t ze_context, + int dup_fd, int cache_enabled) +{ + ucs_status_t status = UCS_OK; + uct_ze_ipc_cache_t *cache; + ucs_pgt_region_t *pgt_region; + uct_ze_ipc_cache_region_t *region; + + status = uct_ze_ipc_get_remote_cache(pid, ze_context, &cache); + if (status != UCS_OK) { + return status; + } + + /* use write lock because cache maybe modified */ + pthread_rwlock_wrlock(&cache->lock); + pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, &cache->pgtable, address); + ucs_assert(pgt_region != NULL); + region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + + ucs_assert(region->refcount >= 1); + region->refcount--; + + /* + * check refcount to see if an in-flight transfer is using the same mapping + */ + if (!region->refcount && !cache_enabled) { + status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_error("failed to remove address:%p from cache (%s)", + (void *)region->key.address, ucs_status_string(status)); + } + ucs_assert(region->mapped_addr == mapped_addr); + status = uct_ze_ipc_close_memhandle(region); + ucs_free(region); + } + + pthread_rwlock_unlock(&cache->lock); + return status; +} + + +UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, + (key, ze_context, ze_device, mapped_addr, dup_fd), + uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + void **mapped_addr, int *dup_fd) +{ + uct_ze_ipc_cache_t *cache; + ucs_status_t status; + ucs_pgt_region_t *pgt_region; + uct_ze_ipc_cache_region_t *region; + int ret; + + status = uct_ze_ipc_get_remote_cache(key->pid, ze_context, &cache); + if (status != UCS_OK) { + return status; + } + + pthread_rwlock_wrlock(&cache->lock); + pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, + &cache->pgtable, key->address); + if (ucs_likely(pgt_region != NULL)) { + region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + + /* cache hit */ + ucs_trace("%s: ze_ipc cache hit addr:%p size:%lu region:" + UCS_PGT_REGION_FMT, cache->name, (void *)key->address, + key->length, UCS_PGT_REGION_ARG(®ion->super)); + + *mapped_addr = region->mapped_addr; + *dup_fd = region->dup_fd; + ucs_assert(region->refcount < UINT64_MAX); + region->refcount++; + pthread_rwlock_unlock(&cache->lock); + return UCS_OK; + } + + status = uct_ze_ipc_open_memhandle(key, ze_context, ze_device, mapped_addr, dup_fd); + if (ucs_unlikely(status != UCS_OK)) { + goto err; + } + + /* create new cache entry */ + ret = ucs_posix_memalign((void **)®ion, + ucs_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), + sizeof(uct_ze_ipc_cache_region_t), + "uct_ze_ipc_cache_region"); + if (ret != 0) { + ucs_warn("failed to allocate uct_ze_ipc_cache region"); + status = UCS_ERR_NO_MEMORY; + goto err; + } + + region->super.start = ucs_align_down_pow2((uintptr_t)key->address, + UCS_PGT_ADDR_ALIGN); + region->super.end = ucs_align_up_pow2((uintptr_t)key->address + key->length, + UCS_PGT_ADDR_ALIGN); + region->key = *key; + region->mapped_addr = *mapped_addr; + region->refcount = 1; + region->ze_context = ze_context; + region->dup_fd = *dup_fd; + + status = UCS_PROFILE_CALL(ucs_pgtable_insert, + &cache->pgtable, ®ion->super); + if (status == UCS_ERR_ALREADY_EXISTS) { + /* overlapped region means memory freed at source. remove and try insert */ + uct_ze_ipc_cache_invalidate_regions(cache, + (void *)region->super.start, + (void *)region->super.end); + status = UCS_PROFILE_CALL(ucs_pgtable_insert, + &cache->pgtable, ®ion->super); + } + if (status != UCS_OK) { + ucs_error("%s: failed to insert region:"UCS_PGT_REGION_FMT" size:%lu :%s", + cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length, + ucs_status_string(status)); + ucs_free(region); + goto err; + } + + ucs_trace("%s: ze_ipc cache new region:"UCS_PGT_REGION_FMT" size:%lu", + cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length); + + status = UCS_OK; + +err: + pthread_rwlock_unlock(&cache->lock); + return status; +} + + +ucs_status_t uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, + const char *name) +{ + ucs_status_t status; + uct_ze_ipc_cache_t *cache_desc; + int ret; + + cache_desc = ucs_malloc(sizeof(uct_ze_ipc_cache_t), "uct_ze_ipc_cache_t"); + if (cache_desc == NULL) { + ucs_error("failed to allocate memory for ze_ipc cache"); + return UCS_ERR_NO_MEMORY; + } + + ret = pthread_rwlock_init(&cache_desc->lock, NULL); + if (ret) { + ucs_error("pthread_rwlock_init() failed: %m"); + status = UCS_ERR_INVALID_PARAM; + goto err; + } + + status = ucs_pgtable_init(&cache_desc->pgtable, + uct_ze_ipc_cache_pgt_dir_alloc, + uct_ze_ipc_cache_pgt_dir_release); + if (status != UCS_OK) { + goto err_destroy_rwlock; + } + + cache_desc->name = strdup(name); + if (cache_desc->name == NULL) { + status = UCS_ERR_NO_MEMORY; + goto err_destroy_rwlock; + } + + *cache = cache_desc; + return UCS_OK; + +err_destroy_rwlock: + pthread_rwlock_destroy(&cache_desc->lock); +err: + free(cache_desc); + return status; +} + + +void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache) +{ + uct_ze_ipc_cache_purge(cache); + ucs_pgtable_cleanup(&cache->pgtable); + pthread_rwlock_destroy(&cache->lock); + free(cache->name); + ucs_free(cache); +} + + +UCS_STATIC_INIT { + ucs_recursive_spinlock_init(&uct_ze_ipc_remote_cache.lock, 0); + kh_init_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); +} + + +UCS_STATIC_CLEANUP { + uct_ze_ipc_cache_t *rem_cache; + + kh_foreach_value(&uct_ze_ipc_remote_cache.hash, rem_cache, { + uct_ze_ipc_destroy_cache(rem_cache); + }) + kh_destroy_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); + ucs_recursive_spinlock_destroy(&uct_ze_ipc_remote_cache.lock); +} + + diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.h b/src/uct/ze/ze_ipc/ze_ipc_cache.h new file mode 100644 index 00000000000..7899da770e5 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.h @@ -0,0 +1,98 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_CACHE_H_ +#define UCT_ZE_IPC_CACHE_H_ + +#include +#include +#include +#include +#include "ze_ipc_md.h" + + +typedef struct uct_ze_ipc_cache uct_ze_ipc_cache_t; +typedef struct uct_ze_ipc_cache_region uct_ze_ipc_cache_region_t; +typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; + + +/** + * Cache region structure for storing mapped IPC handles + */ +struct uct_ze_ipc_cache_region { + ucs_pgt_region_t super; /**< Base class - page table region */ + ucs_list_link_t list; /**< List element */ + uct_ze_ipc_key_t key; /**< Remote memory key */ + void *mapped_addr; /**< Local mapped address */ + uint64_t refcount; /**< Track in-flight ops before unmapping*/ + ze_context_handle_t ze_context; /**< Level Zero context */ + int dup_fd; /**< Duplicated file descriptor */ +}; + + +/** + * Cache structure for managing IPC handle mappings + */ +struct uct_ze_ipc_cache { + pthread_rwlock_t lock; /**< Protects the page table */ + ucs_pgtable_t pgtable; /**< Page table to hold the regions */ + char *name; /**< Name for debugging */ +}; + + +/** + * Create a new IPC cache + * + * @param cache Pointer to store the created cache + * @param name Name for the cache (for debugging) + * @return UCS_OK on success, error code otherwise + */ +ucs_status_t uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, + const char *name); + + +/** + * Destroy an IPC cache + * + * @param cache Cache to destroy + */ +void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache); + + +/** + * Map an IPC memory handle to a local address (with caching) + * + * @param key Remote memory key containing IPC handle + * @param ze_context Level Zero context + * @param ze_device Level Zero device + * @param mapped_addr Pointer to store the mapped address + * @param dup_fd Pointer to store the duplicated file descriptor + * @return UCS_OK on success, error code otherwise + */ +ucs_status_t uct_ze_ipc_map_memhandle(uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + void **mapped_addr, + int *dup_fd); + + +/** + * Unmap an IPC memory handle (with caching) + * + * @param pid Process ID of the remote process + * @param address Base address of the remote memory + * @param mapped_addr Mapped local address + * @param ze_context Level Zero context + * @param dup_fd Duplicated file descriptor + * @param cache_enabled Whether caching is enabled + * @return UCS_OK on success, error code otherwise + */ +ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, + void *mapped_addr, + ze_context_handle_t ze_context, + int dup_fd, int cache_enabled); + +#endif /* UCT_ZE_IPC_CACHE_H_ */ + diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c new file mode 100644 index 00000000000..24528869b63 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -0,0 +1,355 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "ze_ipc_ep.h" +#include "ze_ipc_iface.h" +#include "ze_ipc_md.h" +#include "ze_ipc_cache.h" +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#define UCT_ZE_IPC_PUT 0 +#define UCT_ZE_IPC_GET 1 + +/* + * + * @param remote_pid PID of the process that owns the fd + * @param remote_fd File descriptor in the remote process + * @return Local fd on success, -1 on error + */ +int uct_ze_ipc_dup_fd_from_pid(pid_t remote_pid, int remote_fd) +{ +#if defined(SYS_pidfd_open) && defined(SYS_pidfd_getfd) + int pidfd, local_fd; + + /* Skip if same process */ + if (remote_pid == getpid()) { + return remote_fd; + } + + /* Open pidfd for the remote process */ + pidfd = syscall(SYS_pidfd_open, remote_pid, 0); + if (pidfd < 0) { + ucs_error("ze_ipc: pidfd_open(%d) failed: %m", remote_pid); + return -1; + } + + /* Duplicate the fd from remote process */ + local_fd = syscall(SYS_pidfd_getfd, pidfd, remote_fd, 0); + if (local_fd < 0) { + ucs_error("ze_ipc: pidfd_getfd(pidfd=%d, remote_fd=%d) failed: %m", + pidfd, remote_fd); + close(pidfd); + return -1; + } + + close(pidfd); + ucs_debug("ze_ipc: duplicated fd %d from pid %d -> local fd %d (uncached)", + remote_fd, remote_pid, local_fd); + return local_fd; +#else + ucs_error("ze_ipc: pidfd_getfd not supported on this system"); + return -1; +#endif +} + + +static UCS_CLASS_INIT_FUNC(uct_ze_ipc_ep_t, const uct_ep_params_t *params) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(params->iface, + uct_ze_ipc_iface_t); + + UCT_EP_PARAMS_CHECK_DEV_IFACE_ADDRS(params); + UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super); + + self->remote_pid = *(const pid_t*)params->iface_addr; + + ucs_debug("ze_ipc_ep: created endpoint to remote pid %d (local pid %d)", + self->remote_pid, getpid()); + return UCS_OK; +} + + +static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_ep_t) +{ +} + + +UCS_CLASS_DEFINE(uct_ze_ipc_ep_t, uct_base_ep_t) +UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t *); +UCS_CLASS_DEFINE_DELETE_FUNC(uct_ze_ipc_ep_t, uct_ep_t); + + +#define uct_ze_ipc_trace_data(_addr, _rkey, _fmt, ...) \ + ucs_trace_data(_fmt " to %"PRIx64"(%+ld)", ## __VA_ARGS__, (_addr), (_rkey)) + + +/* Timeline profiling for performance analysis */ +typedef struct uct_ze_ipc_timeline { + struct timespec start; + struct timespec get_ipc_handle; + struct timespec event_alloc; + struct timespec async_copy; + struct timespec end; +} uct_ze_ipc_timeline_t; + +static UCS_F_ALWAYS_INLINE void +uct_ze_ipc_timeline_record(struct timespec *ts) +{ + clock_gettime(CLOCK_MONOTONIC, ts); +} + +static UCS_F_ALWAYS_INLINE void +uct_ze_ipc_timeline_report(const uct_ze_ipc_timeline_t *timeline, + size_t length, unsigned cmd_list_idx) +{ + double ipc_time, event_time, copy_time, other_time, total_time; + + ipc_time = (timeline->get_ipc_handle.tv_sec - timeline->start.tv_sec) * 1000.0 + + (timeline->get_ipc_handle.tv_nsec - timeline->start.tv_nsec) / 1000000.0; + event_time = (timeline->event_alloc.tv_sec - timeline->get_ipc_handle.tv_sec) * 1000.0 + + (timeline->event_alloc.tv_nsec - timeline->get_ipc_handle.tv_nsec) / 1000000.0; + copy_time = (timeline->async_copy.tv_sec - timeline->event_alloc.tv_sec) * 1000.0 + + (timeline->async_copy.tv_nsec - timeline->event_alloc.tv_nsec) / 1000000.0; + other_time = (timeline->end.tv_sec - timeline->async_copy.tv_sec) * 1000.0 + + (timeline->end.tv_nsec - timeline->async_copy.tv_nsec) / 1000000.0; + total_time = (timeline->end.tv_sec - timeline->start.tv_sec) * 1000.0 + + (timeline->end.tv_nsec - timeline->start.tv_nsec) / 1000000.0; + + ucs_trace("ze_ipc timeline: total=%.3fms (ipc=%.3fms event=%.3fms copy=%.3fms other=%.3fms) " + "size=%zu cmd_list=%u", + total_time, ipc_time, event_time, copy_time, other_time, length, cmd_list_idx); +} + +#define UCT_ZE_IPC_TIMELINE_INIT(_tl) \ + uct_ze_ipc_timeline_t _tl = {{0}} + +#define UCT_ZE_IPC_TIMELINE_RECORD(_tl, _field) \ + if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE)) { \ + uct_ze_ipc_timeline_record(&(_tl)._field); \ + } + +#define UCT_ZE_IPC_TIMELINE_REPORT(_tl, _len, _idx) \ + if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE)) { \ + uct_ze_ipc_timeline_report(&(_tl), (_len), (_idx)); \ + } + + +int uct_ze_ipc_ep_is_connected(const uct_ep_h tl_ep, + const uct_ep_is_connected_params_t *params) +{ + const uct_ze_ipc_ep_t *ep = ucs_derived_of(tl_ep, uct_ze_ipc_ep_t); + + if (!uct_base_ep_is_connected(tl_ep, params)) { + return 0; + } + + return ep->remote_pid == *(pid_t*)params->iface_addr; +} + + +static UCS_F_ALWAYS_INLINE ucs_status_t +uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, + const uct_iov_t *iov, uct_rkey_t rkey, + uct_completion_t *comp, int direction) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ze_ipc_iface_t); + uct_ze_ipc_ep_t *ep = ucs_derived_of(tl_ep, uct_ze_ipc_ep_t); + uct_ze_ipc_key_t *key = (uct_ze_ipc_key_t *)rkey; + uct_ze_ipc_event_desc_t *event_desc; + uct_ze_ipc_queue_desc_t *q_desc; + ze_event_handle_t event = NULL; + void *mapped_addr = NULL; + void *mapped_rem_addr; + void *dst, *src; + size_t offset; + ze_result_t ret; + ucs_status_t status; + int local_fd; + int event_index = -1; + unsigned cmd_list_idx; + UCT_ZE_IPC_TIMELINE_INIT(timeline); + + UCT_ZE_IPC_TIMELINE_RECORD(timeline, start); + + if (ucs_unlikely(iov[0].length == 0)) { + ucs_trace_data("Zero length request: skip it"); + return UCS_OK; + } + + /* Use cache to map IPC handle */ + status = uct_ze_ipc_map_memhandle(key, iface->ze_context, iface->ze_device, + &mapped_addr, &local_fd); + if (status != UCS_OK) { + ucs_error("ze_ipc_ep: uct_ze_ipc_map_memhandle failed"); + return status; + } + UCT_ZE_IPC_TIMELINE_RECORD(timeline, get_ipc_handle); + + ucs_debug("ze_ipc_ep: IPC handle mapped (cached), mapped_addr=%p", mapped_addr); + + /* Calculate offset within the allocation */ + offset = remote_addr - key->address; + mapped_rem_addr = (void *)((uintptr_t)mapped_addr + offset); + + /* Allocate event from shared pool for performance */ + event_index = uct_ze_ipc_alloc_event(iface, &event); + if (event_index < 0) { + ucs_error("failed to allocate event from shared pool"); + uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, + iface->ze_context, local_fd, + iface->config.enable_cache); + return UCS_ERR_NO_RESOURCE; + } + + /* Allocate event descriptor */ + event_desc = ucs_malloc(sizeof(*event_desc), "uct_ze_ipc_event_desc_t"); + if (event_desc == NULL) { + ucs_error("failed to allocate event descriptor"); + uct_ze_ipc_free_event(iface, event, event_index); + uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, + iface->ze_context, local_fd, + iface->config.enable_cache); + return UCS_ERR_NO_MEMORY; + } + + /* Store information for cache-based cleanup and event tracking */ + event_desc->event = event; + event_desc->event_pool = NULL; /* Using shared pool, not private */ + event_desc->event_index = event_index; + event_desc->dup_fd = local_fd; + event_desc->pid = ep->remote_pid; + event_desc->address = key->address; + + /* Set up source and destination based on direction */ + if (direction == UCT_ZE_IPC_PUT) { + dst = mapped_rem_addr; + src = iov[0].buffer; + } else { + dst = iov[0].buffer; + src = mapped_rem_addr; + } + + UCT_ZE_IPC_TIMELINE_RECORD(timeline, event_alloc); + + /* + * Select command list using round-robin scheduling + * Similar to CUDA IPC's stream selection: key->stream_id % iface->config.max_streams + * This distributes copy operations across multiple command lists for parallel execution + */ + cmd_list_idx = iface->next_cmd_list % iface->num_cmd_lists; + iface->next_cmd_list++; + + q_desc = &iface->queue_desc[cmd_list_idx]; + + /* + * Append memory copy to selected immediate command list with event signaling. + * Immediate command lists execute asynchronously without needing + * explicit close/execute/reset calls. + * + * Using event for completion tracking improves performance by: + * - Allowing async progress checking + * - Enabling proper completion callbacks + * - Supporting multiple outstanding operations + */ + ret = zeCommandListAppendMemoryCopy(q_desc->cmd_list, dst, src, + iov[0].length, event, 0, NULL); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeCommandListAppendMemoryCopy failed with error 0x%x", ret); + goto err_cleanup; + } + + UCT_ZE_IPC_TIMELINE_RECORD(timeline, async_copy); + + /* Store event info for progress tracking */ + event_desc->mapped_addr = mapped_addr; + event_desc->comp = comp; + + /* Add to active queue if this is the first event for this command list */ + if (ucs_queue_is_empty(&q_desc->event_queue)) { + ucs_queue_push(&iface->active_queue, &q_desc->queue); + } + + /* Push event to this command list's event queue */ + ucs_queue_push(&q_desc->event_queue, &event_desc->queue); + + UCT_ZE_IPC_TIMELINE_RECORD(timeline, end); + + ucs_trace("zeCommandListAppendMemoryCopy issued (async): cmd_list[%u/%u]=%p dst=%p src=%p len=%zu", + cmd_list_idx, iface->num_cmd_lists, q_desc->cmd_list, dst, src, iov[0].length); + UCT_ZE_IPC_TIMELINE_REPORT(timeline, iov[0].length, cmd_list_idx); + + return UCS_INPROGRESS; + +err_cleanup: + uct_ze_ipc_free_event(iface, event, event_index); + ucs_free(event_desc); + uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, + iface->ze_context, local_fd, + iface->config.enable_cache); + return UCS_ERR_IO_ERROR; +} + +UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_get_zcopy, + (tl_ep, iov, iovcnt, remote_addr, rkey, comp), + uct_ep_h tl_ep, const uct_iov_t *iov, size_t iovcnt, + uint64_t remote_addr, uct_rkey_t rkey, + uct_completion_t *comp) +{ + ucs_status_t status; + + status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, + UCT_ZE_IPC_GET); + if (UCS_STATUS_IS_ERR(status)) { + ucs_error("ze_ipc_ep: GET_ZCOPY failed with status %s", ucs_status_string(status)); + return status; + } + + UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), GET, ZCOPY, + uct_iov_total_length(iov, iovcnt)); + uct_ze_ipc_trace_data(remote_addr, rkey, "GET_ZCOPY [length %zu]", + uct_iov_total_length(iov, iovcnt)); + return status; +} + + +UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_put_zcopy, + (tl_ep, iov, iovcnt, remote_addr, rkey, comp), + uct_ep_h tl_ep, const uct_iov_t *iov, size_t iovcnt, + uint64_t remote_addr, uct_rkey_t rkey, + uct_completion_t *comp) +{ + ucs_status_t status; + + status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, + UCT_ZE_IPC_PUT); + if (UCS_STATUS_IS_ERR(status)) { + ucs_error("ze_ipc_ep: PUT_ZCOPY failed with status %s", ucs_status_string(status)); + return status; + } + + UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), PUT, ZCOPY, + uct_iov_total_length(iov, iovcnt)); + uct_ze_ipc_trace_data(remote_addr, rkey, "PUT_ZCOPY [length %zu]", + uct_iov_total_length(iov, iovcnt)); + return status; +} diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.h b/src/uct/ze/ze_ipc/ze_ipc_ep.h new file mode 100644 index 00000000000..f65ab68d603 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_EP_H +#define UCT_ZE_IPC_EP_H + +#include +#include +#include + +#include + +/* Forward declaration */ +typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; + + +typedef struct uct_ze_ipc_ep { + uct_base_ep_t super; + pid_t remote_pid; +} uct_ze_ipc_ep_t; + + +UCS_CLASS_DECLARE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t *); +UCS_CLASS_DECLARE_DELETE_FUNC(uct_ze_ipc_ep_t, uct_ep_t); + +ucs_status_t uct_ze_ipc_ep_get_zcopy(uct_ep_h tl_ep, + const uct_iov_t *iov, size_t iovcnt, + uint64_t remote_addr, uct_rkey_t rkey, + uct_completion_t *comp); + +ucs_status_t uct_ze_ipc_ep_put_zcopy(uct_ep_h tl_ep, + const uct_iov_t *iov, size_t iovcnt, + uint64_t remote_addr, uct_rkey_t rkey, + uct_completion_t *comp); + +int uct_ze_ipc_ep_is_connected(const uct_ep_h tl_ep, + const uct_ep_is_connected_params_t *params); + +int uct_ze_ipc_dup_fd_from_pid(pid_t remote_pid, int remote_fd); + +#endif diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c new file mode 100644 index 00000000000..d91f5d826ba --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -0,0 +1,759 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "ze_ipc_iface.h" +#include "ze_ipc_ep.h" +#include "ze_ipc_cache.h" + +#include +#include +#include +#include +#include + +#include +#include + + +static ucs_config_field_t uct_ze_ipc_iface_config_table[] = { + {"", "", NULL, + ucs_offsetof(uct_ze_ipc_iface_config_t, super), + UCS_CONFIG_TYPE_TABLE(uct_iface_config_table)}, + + {"MAX_POLL", "16", + "Max number of event completions to pick during ze events polling", + ucs_offsetof(uct_ze_ipc_iface_config_t, max_poll), + UCS_CONFIG_TYPE_UINT}, + + {"MAX_CMD_LISTS", UCS_PP_MAKE_STRING(UCT_ZE_IPC_MAX_PEERS), + "Max number of command lists (upper limit, actual count matches copy engine queues)", + ucs_offsetof(uct_ze_ipc_iface_config_t, max_cmd_lists), + UCS_CONFIG_TYPE_UINT}, + + {"ENABLE_CACHE", "yes", + "Enable IPC handle caching to improve performance", + ucs_offsetof(uct_ze_ipc_iface_config_t, enable_cache), + UCS_CONFIG_TYPE_BOOL}, + + {"BW", "50000MBs", + "Effective p2p memory bandwidth", + ucs_offsetof(uct_ze_ipc_iface_config_t, bandwidth), + UCS_CONFIG_TYPE_BW}, + + {"LAT", "1.8us", + "Estimated latency", + ucs_offsetof(uct_ze_ipc_iface_config_t, latency), + UCS_CONFIG_TYPE_TIME}, + + {"OVERHEAD", "4.0us", + "Estimated CPU overhead for transferring GPU memory", + ucs_offsetof(uct_ze_ipc_iface_config_t, overhead), + UCS_CONFIG_TYPE_TIME}, + + {NULL} +}; + + +/* Forward declaration for the delete function */ +static void UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t)(uct_iface_t*); + + +static ucs_status_t +uct_ze_ipc_iface_event_fd_get(uct_iface_h tl_iface, int *fd_p) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + ucs_status_t status; + + if (iface->eventfd == UCS_ASYNC_EVENTFD_INVALID_FD) { + status = ucs_async_eventfd_create(&iface->eventfd); + if (status != UCS_OK) { + return status; + } + } + + *fd_p = iface->eventfd; + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_iface_event_arm(uct_iface_h tl_iface, unsigned events) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + ucs_status_t status; + + /* Poll the eventfd to clear any pending signals */ + if (iface->eventfd != UCS_ASYNC_EVENTFD_INVALID_FD) { + status = ucs_async_eventfd_poll(iface->eventfd); + if (status == UCS_OK) { + /* There was a pending event, return BUSY */ + return UCS_ERR_BUSY; + } else if (status == UCS_ERR_IO_ERROR) { + return status; + } + } + + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_iface_get_device_address(uct_iface_t *tl_iface, + uct_device_addr_t *addr) +{ + *(uint64_t*)addr = ucs_get_system_id(); + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_iface_get_address(uct_iface_h tl_iface, uct_iface_addr_t *iface_addr) +{ + *(pid_t*)iface_addr = getpid(); + return UCS_OK; +} + + +static int +uct_ze_ipc_iface_is_reachable_v2(const uct_iface_h tl_iface, + const uct_iface_is_reachable_params_t *params) +{ + uint64_t *dev_addr; + pid_t remote_pid; + + if (!uct_iface_is_reachable_params_addrs_valid(params)) { + return 0; + } + + dev_addr = (uint64_t *)params->device_addr; + remote_pid = *(pid_t*)params->iface_addr; + + /* Reject same process */ + if (remote_pid == getpid()) { + uct_iface_fill_info_str_buf(params, "same process"); + return 0; + } + + /* Check same system */ + if (ucs_get_system_id() != *dev_addr) { + uct_iface_fill_info_str_buf(params, "different system"); + return 0; + } + + return uct_iface_scope_is_reachable(tl_iface, params); +} + +static ucs_status_t +uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + + uct_base_iface_query(&iface->super, iface_attr); + + iface_attr->iface_addr_len = sizeof(pid_t); + iface_attr->device_addr_len = sizeof(uint64_t); + iface_attr->ep_addr_len = 0; + iface_attr->max_conn_priv = 0; + iface_attr->cap.flags = UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE | + UCT_IFACE_FLAG_CONNECT_TO_IFACE | + UCT_IFACE_FLAG_PENDING | + UCT_IFACE_FLAG_GET_ZCOPY | + UCT_IFACE_FLAG_PUT_ZCOPY; + + iface_attr->cap.event_flags = UCT_IFACE_FLAG_EVENT_SEND_COMP | + UCT_IFACE_FLAG_EVENT_RECV | + UCT_IFACE_FLAG_EVENT_FD; + + iface_attr->cap.put.max_short = 0; + iface_attr->cap.put.max_bcopy = 0; + iface_attr->cap.put.min_zcopy = 0; + iface_attr->cap.put.max_zcopy = ULONG_MAX; + iface_attr->cap.put.opt_zcopy_align = 1; + iface_attr->cap.put.align_mtu = iface_attr->cap.put.opt_zcopy_align; + iface_attr->cap.put.max_iov = 1; + + iface_attr->cap.get.max_bcopy = 0; + iface_attr->cap.get.min_zcopy = 0; + iface_attr->cap.get.max_zcopy = ULONG_MAX; + iface_attr->cap.get.opt_zcopy_align = 1; + iface_attr->cap.get.align_mtu = iface_attr->cap.get.opt_zcopy_align; + iface_attr->cap.get.max_iov = 1; + + iface_attr->latency = ucs_linear_func_make(1e-6, 0); + iface_attr->bandwidth.dedicated = 0; + iface_attr->bandwidth.shared = iface->config.bandwidth; + iface_attr->overhead = 7.0e-6; + iface_attr->priority = 0; + + return UCS_OK; +} + + +/** + * Allocate an event from the shared event pool + * Returns event index on success, -1 on failure + */ +int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t *event_p) +{ + unsigned i, word_idx, bit_idx; + uint64_t mask; + ze_event_desc_t event_desc; + ze_result_t ret; + + ucs_spin_lock(&iface->event_lock); + + /* Find first free event in bitmap */ + for (i = 0; i < iface->event_pool_size; i++) { + word_idx = i / 64; + bit_idx = i % 64; + mask = 1ULL << bit_idx; + + if (!(iface->event_bitmap[word_idx] & mask)) { + /* Found free event, mark as used */ + iface->event_bitmap[word_idx] |= mask; + ucs_spin_unlock(&iface->event_lock); + + /* Create event from shared pool */ + event_desc.stype = ZE_STRUCTURE_TYPE_EVENT_DESC; + event_desc.pNext = NULL; + event_desc.index = i; + event_desc.signal = ZE_EVENT_SCOPE_FLAG_HOST; + event_desc.wait = ZE_EVENT_SCOPE_FLAG_HOST; + + ret = zeEventCreate(iface->ze_event_pool, &event_desc, event_p); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeEventCreate failed with error 0x%x", ret); + /* Mark as free again */ + ucs_spin_lock(&iface->event_lock); + iface->event_bitmap[word_idx] &= ~mask; + ucs_spin_unlock(&iface->event_lock); + return -1; + } + + return i; + } + } + + ucs_spin_unlock(&iface->event_lock); + ucs_warn("ze_ipc: event pool exhausted (size=%u)", iface->event_pool_size); + return -1; +} + +/** + * Free an event back to the shared event pool + */ +void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, + ze_event_handle_t event, + unsigned event_index) +{ + unsigned word_idx, bit_idx; + uint64_t mask; + + if (event != NULL) { + zeEventDestroy(event); + } + + if (event_index != (unsigned)-1) { + word_idx = event_index / 64; + bit_idx = event_index % 64; + mask = 1ULL << bit_idx; + + ucs_spin_lock(&iface->event_lock); + iface->event_bitmap[word_idx] &= ~mask; + ucs_spin_unlock(&iface->event_lock); + } +} + +static unsigned +uct_ze_ipc_iface_progress(uct_iface_h tl_iface) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + uct_ze_ipc_event_desc_t *event_desc; + uct_ze_ipc_queue_desc_t *q_desc; + ucs_queue_iter_t iter; + unsigned count = 0; + unsigned max_poll = iface->config.max_poll; + ze_result_t ret; + + /* Early exit if no active queues */ + if (ucs_queue_is_empty(&iface->active_queue)) { + return 0; + } + + /* + * Progress all active command list queues + * Similar to CUDA IPC's uct_cuda_base_progress_event_queue + */ + ucs_queue_for_each_extract(q_desc, &iface->active_queue, queue, 1) { + ucs_queue_for_each_safe(event_desc, iter, &q_desc->event_queue, queue) { + /* Check if we've reached max_poll limit */ + if (count >= max_poll) { + /* Put queue back and exit */ + ucs_queue_push(&iface->active_queue, &q_desc->queue); + return count; + } + + ret = zeEventQueryStatus(event_desc->event); + if (ret == ZE_RESULT_NOT_READY) { + continue; + } + + ucs_queue_del_iter(&q_desc->event_queue, iter); + + /* Unmap IPC handle using cache */ + if (event_desc->mapped_addr != NULL) { + ucs_status_t status; + status = uct_ze_ipc_unmap_memhandle(event_desc->pid, + event_desc->address, + event_desc->mapped_addr, + iface->ze_context, + event_desc->dup_fd, + iface->config.enable_cache); + if (status != UCS_OK) { + ucs_warn("failed to unmap IPC handle addr:%p", + event_desc->mapped_addr); + } + } + + /* Invoke completion callback */ + if (event_desc->comp != NULL) { + uct_invoke_completion(event_desc->comp, UCS_OK); + } + + /* Free event back to shared pool or destroy private pool */ + if (event_desc->event_index != (unsigned)-1) { + /* Using shared event pool */ + uct_ze_ipc_free_event(iface, event_desc->event, event_desc->event_index); + } else { + /* Using private event pool (backward compatibility) */ + if (event_desc->event != NULL) { + zeEventDestroy(event_desc->event); + } + if (event_desc->event_pool != NULL) { + zeEventPoolDestroy(event_desc->event_pool); + } + } + ucs_free(event_desc); + + count++; + } + + /* If queue still has events, put it back to active queue */ + if (!ucs_queue_is_empty(&q_desc->event_queue)) { + ucs_queue_push(&iface->active_queue, &q_desc->queue); + } + } + + return count; +} + + +static ucs_status_t +uct_ze_ipc_iface_flush(uct_iface_h tl_iface, unsigned flags, + uct_completion_t *comp) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + unsigned i; + + /* Check if all command list queues are empty */ + for (i = 0; i < iface->num_cmd_lists; i++) { + if (!ucs_queue_is_empty(&iface->queue_desc[i].event_queue)) { + UCT_TL_IFACE_STAT_FLUSH_WAIT(ucs_derived_of(tl_iface, uct_base_iface_t)); + return UCS_INPROGRESS; + } + } + + UCT_TL_IFACE_STAT_FLUSH(ucs_derived_of(tl_iface, uct_base_iface_t)); + return UCS_OK; +} + + +static uct_iface_ops_t uct_ze_ipc_iface_ops = { + .ep_get_zcopy = uct_ze_ipc_ep_get_zcopy, + .ep_put_zcopy = uct_ze_ipc_ep_put_zcopy, + .ep_pending_add = (uct_ep_pending_add_func_t)ucs_empty_function_return_busy, + .ep_pending_purge = (uct_ep_pending_purge_func_t)ucs_empty_function, + .ep_flush = uct_base_ep_flush, + .ep_fence = uct_base_ep_fence, + .ep_check = (uct_ep_check_func_t)ucs_empty_function_return_unsupported, + .ep_create = UCS_CLASS_NEW_FUNC_NAME(uct_ze_ipc_ep_t), + .ep_destroy = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_ep_t), + .iface_flush = uct_ze_ipc_iface_flush, + .iface_fence = uct_base_iface_fence, + .iface_progress_enable = uct_base_iface_progress_enable, + .iface_progress_disable = uct_base_iface_progress_disable, + .iface_progress = uct_ze_ipc_iface_progress, + .iface_event_fd_get = uct_ze_ipc_iface_event_fd_get, + .iface_event_arm = uct_ze_ipc_iface_event_arm, + .iface_close = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t), + .iface_query = uct_ze_ipc_iface_query, + .iface_get_device_address = uct_ze_ipc_iface_get_device_address, + .iface_get_address = uct_ze_ipc_iface_get_address, + .iface_is_reachable = uct_base_iface_is_reachable, +}; + + +static ucs_status_t +uct_ze_ipc_estimate_perf(uct_iface_h tl_iface, uct_perf_attr_t *perf_attr) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_BANDWIDTH) { + perf_attr->bandwidth.dedicated = 0; + perf_attr->bandwidth.shared = iface->config.bandwidth; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_PATH_BANDWIDTH) { + perf_attr->path_bandwidth.dedicated = 0; + perf_attr->path_bandwidth.shared = iface->config.bandwidth; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_SEND_PRE_OVERHEAD) { + perf_attr->send_pre_overhead = iface->config.overhead; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_SEND_POST_OVERHEAD) { + perf_attr->send_post_overhead = 0; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_RECV_OVERHEAD) { + perf_attr->recv_overhead = 0; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_LATENCY) { + perf_attr->latency = ucs_linear_func_make(iface->config.latency, 0.0); + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_MAX_INFLIGHT_EPS) { + perf_attr->max_inflight_eps = SIZE_MAX; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_FLAGS) { + perf_attr->flags = 0; + } + + return UCS_OK; +} + +static uct_iface_internal_ops_t uct_ze_ipc_iface_internal_ops = { + .iface_query_v2 = uct_iface_base_query_v2, + .iface_estimate_perf = uct_ze_ipc_estimate_perf, + .iface_vfs_refresh = (uct_iface_vfs_refresh_func_t)ucs_empty_function, + .iface_mem_element_pack = (uct_iface_mem_element_pack_func_t)ucs_empty_function_return_unsupported, + .ep_query = (uct_ep_query_func_t)ucs_empty_function_return_unsupported, + .ep_invalidate = (uct_ep_invalidate_func_t)ucs_empty_function_return_unsupported, + .ep_connect_to_ep_v2 = (uct_ep_connect_to_ep_v2_func_t)ucs_empty_function_return_unsupported, + .iface_is_reachable_v2 = uct_ze_ipc_iface_is_reachable_v2, + .ep_is_connected = uct_ze_ipc_ep_is_connected +}; + + +/* Find the copy engine queue group ordinal for a device */ +static ucs_status_t +uct_ze_ipc_find_copy_ordinal(ze_device_handle_t device, uint32_t *ordinal_p, + uint32_t *num_queues_p) +{ + ze_command_queue_group_properties_t queue_props[16]; + uint32_t num_queue_groups = 16; + uint32_t i; + ze_result_t ret; + + /* Initialize structure types */ + for (i = 0; i < 16; i++) { + queue_props[i].stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES; + queue_props[i].pNext = NULL; + } + + /* Get queue group properties */ + ret = zeDeviceGetCommandQueueGroupProperties(device, &num_queue_groups, + queue_props); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_ipc_iface: zeDeviceGetCommandQueueGroupProperties failed: 0x%x", + ret); + return UCS_ERR_IO_ERROR; + } + + /* First pass: find dedicated copy engine (COPY flag but NOT COMPUTE) */ + for (i = 0; i < num_queue_groups; i++) { + if ((queue_props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) && + !(queue_props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE)) { + *ordinal_p = i; + *num_queues_p = queue_props[i].numQueues; + ucs_debug("ze_ipc_iface: using dedicated copy engine queue group %u (numQueues=%u)", + i, queue_props[i].numQueues); + return UCS_OK; + } + } + + /* Second pass: find any queue group that supports COPY */ + for (i = 0; i < num_queue_groups; i++) { + if (queue_props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) { + *ordinal_p = i; + *num_queues_p = queue_props[i].numQueues; + ucs_debug("ze_ipc_iface: using copy-capable queue group %u (numQueues=%u)", + i, queue_props[i].numQueues); + return UCS_OK; + } + } + + ucs_error("ze_ipc_iface: no copy-capable queue group found"); + return UCS_ERR_NO_DEVICE; +} + +static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, + const uct_iface_params_t *params, + const uct_iface_config_t *tl_config) +{ + uct_ze_ipc_iface_config_t *config; + uct_ze_ipc_md_t *ze_md; + ze_command_queue_desc_t queue_desc = {}; + ze_event_pool_desc_t event_pool_desc; + uint32_t copy_ordinal = 0; + uint32_t num_queues = 0; + ucs_status_t status; + ze_result_t ret; + unsigned i; + size_t bitmap_size; + + config = ucs_derived_of(tl_config, uct_ze_ipc_iface_config_t); + ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + + UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_ze_ipc_iface_ops, + &uct_ze_ipc_iface_internal_ops, md, worker, + params, tl_config + UCS_STATS_ARG(params->stats_root) + UCS_STATS_ARG(UCT_ZE_IPC_TL_NAME)); + + self->ze_context = ze_md->ze_context; + self->ze_device = ze_md->ze_device; + self->config = *config; + self->eventfd = UCS_ASYNC_EVENTFD_INVALID_FD; + self->next_cmd_list = 0; + + /* Find copy engine queue group ordinal and available queue count */ + status = uct_ze_ipc_find_copy_ordinal(self->ze_device, ©_ordinal, &num_queues); + if (status != UCS_OK) { + return status; + } + + /* + * Determine number of command lists based on hardware Copy Engine availability. + * + * Strategy: + * 1. If dedicated Copy Engines exist: create one command list per Copy Engine queue + * 2. If no Copy Engines: create 1 command list (fallback to compute engine) + * 3. Respect user configuration limit (MAX_CMD_LISTS) as upper bound + * + * Rationale: + * - Each Copy Engine queue can execute independently in parallel + * - Creating more command lists than hardware queues provides no benefit + * - One command list per Copy Engine queue maximizes hardware utilization + */ + if (num_queues == 0) { + /* No copy engines available, use single command list on compute engine */ + self->num_cmd_lists = 1; + ucs_info("ze_ipc_iface: no dedicated copy engines found, using 1 command list " + "on queue group %u (compute engine fallback)", copy_ordinal); + } else { + /* Match command list count to available Copy Engine queues */ + self->num_cmd_lists = ucs_min(num_queues, config->max_cmd_lists); + + if (self->num_cmd_lists < num_queues) { + ucs_info("ze_ipc_iface: limiting command lists to %u (hardware has %u copy queues, " + "config limit is %u)", self->num_cmd_lists, num_queues, + config->max_cmd_lists); + } else { + ucs_info("ze_ipc_iface: creating %u command list(s) to match %u copy engine queue(s) " + "on queue group %u", self->num_cmd_lists, num_queues, copy_ordinal); + } + } + + /* Validate final count */ + if (self->num_cmd_lists > UCT_ZE_IPC_MAX_PEERS) { + ucs_error("ze_ipc_iface: computed num_cmd_lists (%u) exceeds maximum (%u)", + self->num_cmd_lists, UCT_ZE_IPC_MAX_PEERS); + return UCS_ERR_INVALID_PARAM; + } + + /* Initialize active queue for command lists with pending operations */ + ucs_queue_head_init(&self->active_queue); + + /* + * Create multiple immediate command lists for parallel progress. + * + * According to Level Zero spec, for immediate command lists: + * - queue_desc.index MUST be 0 (not 1, 2, 3...) + * - Multiple immediate command lists with index=0 are allowed + * - Driver manages parallel execution automatically + * + * Benefits of multiple immediate command lists: + * - Round-robin distribution of IPC copy operations + * - Reduced contention on single command list + * - Better hardware utilization for concurrent operations + * - Independent event queues for each command list + */ + queue_desc.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC; + queue_desc.ordinal = copy_ordinal; + queue_desc.mode = ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS; + queue_desc.index = 0; /* MUST be 0 for immediate command lists */ + queue_desc.flags = 0; + queue_desc.priority = ZE_COMMAND_QUEUE_PRIORITY_NORMAL; + + for (i = 0; i < self->num_cmd_lists; i++) { + ret = zeCommandListCreateImmediate(self->ze_context, self->ze_device, + &queue_desc, &self->queue_desc[i].cmd_list); + if (ret != ZE_RESULT_SUCCESS) { + unsigned j; + ucs_error("ze_ipc_iface: zeCommandListCreateImmediate[%u] failed with error 0x%x", + i, ret); + /* Clean up previously created command lists */ + for (j = 0; j < i; j++) { + zeCommandListDestroy(self->queue_desc[j].cmd_list); + } + return UCS_ERR_IO_ERROR; + } + + /* Initialize event queue for this command list */ + ucs_queue_head_init(&self->queue_desc[i].event_queue); + + ucs_debug("ze_ipc_iface: created immediate command list %u/%u: %p", + i + 1, self->num_cmd_lists, self->queue_desc[i].cmd_list); + } + + /* + * Create a shared event pool for all copy operations to avoid + * per-operation event pool creation/destruction overhead. + * + * Performance optimization: + * - Single event pool shared across all operations + * - Pre-allocated events reduce allocation overhead + * - Event reuse via bitmap tracking + * - Typical pool size: 1024 events (configurable) + */ + self->event_pool_size = 1024; /* TODO: make this configurable */ + + event_pool_desc.stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC; + event_pool_desc.pNext = NULL; + event_pool_desc.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE; + event_pool_desc.count = self->event_pool_size; + + ret = zeEventPoolCreate(self->ze_context, &event_pool_desc, 1, + &self->ze_device, &self->ze_event_pool); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_ipc_iface: zeEventPoolCreate failed with error 0x%x", ret); + /* Clean up command lists */ + for (i = 0; i < self->num_cmd_lists; i++) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + return UCS_ERR_IO_ERROR; + } + + /* Initialize event bitmap for tracking free events */ + bitmap_size = (self->event_pool_size + 63) / 64; /* Round up to 64-bit words */ + self->event_bitmap = ucs_calloc(bitmap_size, sizeof(uint64_t), "ze_ipc_event_bitmap"); + if (self->event_bitmap == NULL) { + ucs_error("ze_ipc_iface: failed to allocate event bitmap"); + zeEventPoolDestroy(self->ze_event_pool); + for (i = 0; i < self->num_cmd_lists; i++) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + return UCS_ERR_NO_MEMORY; + } + + /* Initialize spinlock for event allocation */ + status = ucs_spinlock_init(&self->event_lock, 0); + if (status != UCS_OK) { + ucs_error("ze_ipc_iface: failed to initialize event_lock spinlock"); + ucs_free(self->event_bitmap); + zeEventPoolDestroy(self->ze_event_pool); + for (i = 0; i < self->num_cmd_lists; i++) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + return status; + } + + ucs_info("ze_ipc_iface: initialized iface for device %p context %p " + "(pid=%d, copy_ordinal=%u, num_cmd_lists=%u, event_pool_size=%u)", + self->ze_device, self->ze_context, getpid(), copy_ordinal, + self->num_cmd_lists, self->event_pool_size); + + return UCS_OK; +} + + +static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) +{ + uct_ze_ipc_event_desc_t *event_desc; + unsigned i; + + /* Clean up outstanding events from all command list queues */ + for (i = 0; i < self->num_cmd_lists; i++) { + while (!ucs_queue_is_empty(&self->queue_desc[i].event_queue)) { + event_desc = ucs_queue_pull_elem_non_empty(&self->queue_desc[i].event_queue, + uct_ze_ipc_event_desc_t, + queue); + if (event_desc->mapped_addr != NULL) { + zeMemCloseIpcHandle(self->ze_context, event_desc->mapped_addr); + } + if (event_desc->dup_fd >= 0) { + close(event_desc->dup_fd); + } + if (event_desc->event != NULL) { + zeEventDestroy(event_desc->event); + } + if (event_desc->event_pool != NULL) { + zeEventPoolDestroy(event_desc->event_pool); + } + ucs_free(event_desc); + } + } + + /* Destroy all command lists */ + for (i = 0; i < self->num_cmd_lists; i++) { + if (self->queue_desc[i].cmd_list != NULL) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + } + /* cmd_queue is not used with immediate command lists */ + + /* Destroy shared event pool */ + if (self->ze_event_pool != NULL) { + zeEventPoolDestroy(self->ze_event_pool); + } + + /* Free event bitmap */ + ucs_free(self->event_bitmap); + + /* Destroy spinlock */ + ucs_spinlock_destroy(&self->event_lock); + + /* Close eventfd if created */ + if (self->eventfd != UCS_ASYNC_EVENTFD_INVALID_FD) { + close(self->eventfd); + } +} + + +static ucs_status_t +uct_ze_ipc_query_devices(uct_md_h uct_md, uct_tl_device_resource_t **tl_devices_p, + unsigned *num_tl_devices_p) +{ + return uct_ze_base_query_devices_common(uct_md, UCT_DEVICE_TYPE_ACC, + tl_devices_p, num_tl_devices_p); +} + + +UCS_CLASS_DEFINE(uct_ze_ipc_iface_t, uct_base_iface_t); +UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_iface_t, uct_iface_t, uct_md_h, uct_worker_h, + const uct_iface_params_t*, const uct_iface_config_t*); +static UCS_CLASS_DEFINE_DELETE_FUNC(uct_ze_ipc_iface_t, uct_iface_t); + +UCT_TL_DEFINE(&uct_ze_ipc_component, ze_ipc, + uct_ze_ipc_query_devices, uct_ze_ipc_iface_t, "ZE_IPC_", + uct_ze_ipc_iface_config_table, uct_ze_ipc_iface_config_t); diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.h b/src/uct/ze/ze_ipc/ze_ipc_iface.h new file mode 100644 index 00000000000..06cf31319de --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.h @@ -0,0 +1,98 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_IFACE_H +#define UCT_ZE_IPC_IFACE_H + +#include +#include +#include +#include +#include + +#include "ze_ipc_md.h" + + +#define UCT_ZE_IPC_TL_NAME "ze_ipc" +#define UCT_ZE_IPC_MAX_PEERS 16 + + +typedef struct uct_ze_ipc_iface_config { + uct_iface_config_t super; + unsigned max_poll; /* query attempts w.o success */ + unsigned max_cmd_lists; /* max number of command lists for parallel progress */ + int enable_cache; /* enable/disable ipc handle cache */ + double bandwidth; /* estimated bandwidth */ + double latency; /* estimated latency */ + double overhead; /* estimated CPU overhead */ +} uct_ze_ipc_iface_config_t; + + +/** + * Queue descriptor for each command list + * Similar to CUDA IPC's uct_cuda_queue_desc_t design + */ +typedef struct uct_ze_ipc_queue_desc { + ucs_queue_elem_t queue; /* element in active queue list */ + ze_command_list_handle_t cmd_list; /* immediate command list */ + ucs_queue_head_t event_queue; /* queue of outstanding events */ +} uct_ze_ipc_queue_desc_t; + + +typedef struct uct_ze_ipc_iface { + uct_base_iface_t super; + ze_context_handle_t ze_context; + ze_device_handle_t ze_device; + uct_ze_ipc_iface_config_t config; + int eventfd; /* event fd for async notifications */ + + /* Multi-command-list support for parallel progress */ + uct_ze_ipc_queue_desc_t queue_desc[UCT_ZE_IPC_MAX_PEERS]; /* array of queue descriptors */ + ucs_queue_head_t active_queue; /* queue of active queue descriptors */ + unsigned num_cmd_lists; /* actual number of command lists */ + unsigned next_cmd_list; /* round-robin index for load balancing */ + + /* Pre-allocated event pool for performance optimization */ + ze_event_pool_handle_t ze_event_pool; /* shared event pool for all operations */ + unsigned event_pool_size; /* number of events in the pool */ + ucs_spinlock_t event_lock; /* lock for event allocation */ + uint64_t *event_bitmap; /* bitmap to track free events */ +} uct_ze_ipc_iface_t; + + +typedef struct uct_ze_ipc_event_desc { + ze_event_handle_t event; + ze_event_pool_handle_t event_pool; /* NULL when using shared pool */ + void *mapped_addr; + uct_completion_t *comp; + ucs_queue_elem_t queue; + int dup_fd; /* duplicated fd to close, or -1 if none */ + pid_t pid; /* remote process id for cache lookup */ + uintptr_t address; /* remote base address for cache lookup */ + unsigned event_index; /* index in shared event pool, or -1 if using private pool */ +} uct_ze_ipc_event_desc_t; + + +/** + * Allocate an event from the shared event pool + * + * @param iface Pointer to ze_ipc interface + * @param event_p Pointer to store the allocated event handle + * @return Event index on success, -1 on failure + */ +int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t *event_p); + +/** + * Free an event back to the shared event pool + * + * @param iface Pointer to ze_ipc interface + * @param event Event handle to destroy + * @param event_index Index of the event in the pool + */ +void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t event, + unsigned event_index); + + +#endif diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c new file mode 100644 index 00000000000..27311aa0657 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -0,0 +1,328 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "ze_ipc_md.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +/* Helper function to compute a simple checksum of IPC handle for verification */ +static uint32_t uct_ze_ipc_handle_checksum(const ze_ipc_mem_handle_t *handle) +{ + const unsigned char *bytes = (const unsigned char *)handle; + uint32_t sum = 0; + for (size_t i = 0; i < sizeof(ze_ipc_mem_handle_t); i++) { + sum += bytes[i]; + sum = (sum << 1) | (sum >> 31); /* rotate left */ + } + return sum; +} + + +/* Helper function to print IPC handle bytes for debugging */ +static void uct_ze_ipc_print_handle(const char *prefix, + const ze_ipc_mem_handle_t *handle, + uintptr_t addr, size_t len, int dev_num) +{ + const unsigned char *bytes = (const unsigned char *)handle; + uint32_t checksum = uct_ze_ipc_handle_checksum(handle); + + /* Print first 16 bytes of handle + checksum for quick comparison */ + ucs_info("%s: addr=0x%lx len=%zu dev=%d checksum=0x%08x " + "handle[0-15]=%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x%02x%02x%02x", + prefix, (unsigned long)addr, len, dev_num, checksum, + bytes[0], bytes[1], bytes[2], bytes[3], + bytes[4], bytes[5], bytes[6], bytes[7], + bytes[8], bytes[9], bytes[10], bytes[11], + bytes[12], bytes[13], bytes[14], bytes[15]); +} + + +static ucs_config_field_t uct_ze_ipc_md_config_table[] = { + {"", "", NULL, + ucs_offsetof(uct_ze_ipc_md_config_t, super), + UCS_CONFIG_TYPE_TABLE(uct_md_config_table)}, + + {"DEVICE_ORDINAL", "0", + "Ordinal of the GPU device to use for IPC.", + ucs_offsetof(uct_ze_ipc_md_config_t, device_ordinal), + UCS_CONFIG_TYPE_INT}, + + {NULL} +}; + +static ucs_status_t +uct_ze_ipc_md_query(uct_md_h md, uct_md_attr_v2_t *md_attr) +{ + uct_md_base_md_query(md_attr); + md_attr->rkey_packed_size = sizeof(uct_ze_ipc_key_t); + md_attr->flags = UCT_MD_FLAG_REG | UCT_MD_FLAG_NEED_RKEY; + md_attr->reg_mem_types = UCS_BIT(UCS_MEMORY_TYPE_ZE_DEVICE); + md_attr->cache_mem_types = UCS_BIT(UCS_MEMORY_TYPE_ZE_DEVICE); + md_attr->access_mem_types = UCS_BIT(UCS_MEMORY_TYPE_ZE_DEVICE); + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_mkey_pack(uct_md_h uct_md, uct_mem_h memh, void *address, + size_t length, const uct_md_mkey_pack_params_t *params, + void *mkey_buffer) +{ + uct_ze_ipc_key_t *packed = mkey_buffer; + uct_ze_ipc_key_t *key = memh; + + *packed = *key; + + /* Print handle info for verification - compare with rkey_unpack output */ + uct_ze_ipc_print_handle("PACK(sender)", &packed->ipc_handle, + packed->address, packed->length, packed->dev_num); + + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, size_t length, + uct_ze_ipc_key_t *key) +{ + ze_memory_allocation_properties_t props = { + .stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES + }; + ze_device_handle_t alloc_device = NULL; + void *base_address; + size_t alloc_size; + ze_result_t ret; + ucs_status_t status; + int dev_ordinal; + + /* Get memory allocation properties to verify this is ZE device memory + * Also get the device where memory was allocated */ + ret = zeMemGetAllocProperties(md->ze_context, address, &props, &alloc_device); + if ((ret != ZE_RESULT_SUCCESS) || (props.type == ZE_MEMORY_TYPE_UNKNOWN)) { + ucs_error("failed to get allocation properties for %p", address); + return UCS_ERR_INVALID_ADDR; + } + + /* Get device ordinal from the device handle */ + dev_ordinal = uct_ze_base_get_device_ordinal(alloc_device); + if (dev_ordinal < 0) { + ucs_error("failed to get device ordinal for device %p (ptr=%p)", + (void*)alloc_device, address); + return UCS_ERR_INVALID_ADDR; + } + + /* Get the base address and allocation size */ + ret = zeMemGetAddressRange(md->ze_context, address, &base_address, + &alloc_size); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("failed to get address range for %p", address); + return UCS_ERR_INVALID_ADDR; + } + + /* Get IPC handle for the memory */ + status = UCT_ZE_FUNC_LOG_ERR( + zeMemGetIpcHandle(md->ze_context, base_address, &key->ipc_handle)); + if (status != UCS_OK) { + ucs_error("failed to get IPC handle for %p", address); + return status; + } + + key->pid = getpid(); + key->address = (uintptr_t)base_address; + key->length = alloc_size; + key->dev_num = dev_ordinal; + + ucs_trace("packed IPC handle for %p base=%p len=%zu dev=%d pid=%d", address, + base_address, alloc_size, dev_ordinal, key->pid); + + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, + const uct_md_mem_reg_params_t *params, uct_mem_h *memh_p) +{ + uct_ze_ipc_md_t *md = ucs_derived_of(uct_md, uct_ze_ipc_md_t); + uct_ze_ipc_key_t *key; + ucs_status_t status; + + ucs_info("ze_ipc_md: mem_reg called address=%p length=%zu", address, length); + + key = ucs_malloc(sizeof(*key), "uct_ze_ipc_key_t"); + if (key == NULL) { + ucs_error("ze_ipc_md: failed to allocate memory for uct_ze_ipc_key_t"); + return UCS_ERR_NO_MEMORY; + } + + status = uct_ze_ipc_pack_key(md, address, length, key); + if (status != UCS_OK) { + ucs_error("ze_ipc_md: pack_key failed for address=%p status=%s", + address, ucs_status_string(status)); + ucs_free(key); + return status; + } + + ucs_info("ze_ipc_md: mem_reg succeeded address=%p key->address=0x%lx key->length=%zu", + address, (unsigned long)key->address, key->length); + + *memh_p = key; + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_mem_dereg(uct_md_h md, const uct_md_mem_dereg_params_t *params) +{ + uct_ze_ipc_key_t *key; + + UCT_MD_MEM_DEREG_CHECK_PARAMS(params, 0); + + key = params->memh; + ucs_free(key); + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, + const uct_rkey_unpack_params_t *params, + uct_rkey_t *rkey_p, void **handle_p) +{ + uct_ze_ipc_key_t *packed = (uct_ze_ipc_key_t *)rkey_buffer; + uct_ze_ipc_key_t *key; + + /* Print handle info for verification - compare with mkey_pack output */ + uct_ze_ipc_print_handle("UNPACK(receiver)", &packed->ipc_handle, + packed->address, packed->length, packed->dev_num); + + key = ucs_malloc(sizeof(uct_ze_ipc_key_t), "uct_ze_ipc_key_t"); + if (key == NULL) { + ucs_error("ze_ipc_md: failed to allocate memory for uct_ze_ipc_key_t"); + return UCS_ERR_NO_MEMORY; + } + + *key = *packed; + *handle_p = NULL; + *rkey_p = (uintptr_t)key; + + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_rkey_release(uct_component_t *component, uct_rkey_t rkey, + void *handle) +{ + ucs_assert(handle == NULL); + ucs_free((void *)rkey); + return UCS_OK; +} + + +static void uct_ze_ipc_md_close(uct_md_h uct_md) +{ + uct_ze_ipc_md_t *md = ucs_derived_of(uct_md, uct_ze_ipc_md_t); + + if (md->ze_context != NULL) { + zeContextDestroy(md->ze_context); + } + ucs_free(md); +} + + +static ucs_status_t +uct_ze_ipc_md_open(uct_component_h component, const char *md_name, + const uct_md_config_t *uct_md_config, uct_md_h *md_p) +{ + static uct_md_ops_t md_ops = { + .close = uct_ze_ipc_md_close, + .query = uct_ze_ipc_md_query, + .mem_alloc = (uct_md_mem_alloc_func_t)ucs_empty_function_return_unsupported, + .mem_free = (uct_md_mem_free_func_t)ucs_empty_function_return_unsupported, + .mem_advise = (uct_md_mem_advise_func_t)ucs_empty_function_return_unsupported, + .mem_reg = uct_ze_ipc_mem_reg, + .mem_dereg = uct_ze_ipc_mem_dereg, + .mem_query = (uct_md_mem_query_func_t)ucs_empty_function_return_unsupported, + .mkey_pack = uct_ze_ipc_mkey_pack, + .mem_attach = (uct_md_mem_attach_func_t)ucs_empty_function_return_unsupported, + .detect_memory_type = (uct_md_detect_memory_type_func_t)ucs_empty_function_return_unsupported, + }; + uct_ze_ipc_md_config_t *config = ucs_derived_of(uct_md_config, + uct_ze_ipc_md_config_t); + uct_ze_ipc_md_t *md; + ze_driver_handle_t ze_driver; + ze_context_desc_t context_desc = {}; + ze_result_t ret; + + ze_driver = uct_ze_base_get_driver(); + if (ze_driver == NULL) { + return UCS_ERR_NO_DEVICE; + } + + md = ucs_malloc(sizeof(uct_ze_ipc_md_t), "uct_ze_ipc_md_t"); + if (md == NULL) { + ucs_error("failed to allocate memory for uct_ze_ipc_md_t"); + return UCS_ERR_NO_MEMORY; + } + + md->ze_device = uct_ze_base_get_device(config->device_ordinal); + if (md->ze_device == NULL) { + ucs_error("failed to get device at ordinal %d", config->device_ordinal); + ucs_free(md); + return UCS_ERR_NO_DEVICE; + } + + ret = zeContextCreate(ze_driver, &context_desc, &md->ze_context); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeContextCreate failed with error 0x%x", ret); + ucs_free(md); + return UCS_ERR_NO_DEVICE; + } + + md->super.ops = &md_ops; + md->super.component = &uct_ze_ipc_component; + + *md_p = (uct_md_h)md; + return UCS_OK; +} + + +uct_component_t uct_ze_ipc_component = { + .query_md_resources = uct_ze_base_query_md_resources, + .md_open = uct_ze_ipc_md_open, + .cm_open = (uct_component_cm_open_func_t)ucs_empty_function_return_unsupported, + .rkey_unpack = uct_ze_ipc_rkey_unpack, + .rkey_ptr = (uct_component_rkey_ptr_func_t)ucs_empty_function_return_unsupported, + .rkey_release = uct_ze_ipc_rkey_release, + .rkey_compare = uct_base_rkey_compare, + .name = "ze_ipc", + .md_config = { + .name = "ZE-IPC memory domain", + .prefix = "ZE_IPC_", + .table = uct_ze_ipc_md_config_table, + .size = sizeof(uct_ze_ipc_md_config_t), + }, + .cm_config = UCS_CONFIG_EMPTY_GLOBAL_LIST_ENTRY, + .tl_list = UCT_COMPONENT_TL_LIST_INITIALIZER(&uct_ze_ipc_component), + .flags = 0, + .md_vfs_init = (uct_component_md_vfs_init_func_t)ucs_empty_function +}; +UCT_COMPONENT_REGISTER(&uct_ze_ipc_component); diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.h b/src/uct/ze/ze_ipc/ze_ipc_md.h new file mode 100644 index 00000000000..f816749fc0f --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_md.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_MD_H +#define UCT_ZE_IPC_MD_H + +#include +#include + + +extern uct_component_t uct_ze_ipc_component; + + +/** + * @brief ze ipc MD descriptor + */ +typedef struct uct_ze_ipc_md { + uct_md_t super; /**< Domain info */ + ze_context_handle_t ze_context; /**< Level Zero context */ + ze_device_handle_t ze_device; /**< Level Zero device */ +} uct_ze_ipc_md_t; + + +/** + * @brief ze ipc domain configuration. + */ +typedef struct uct_ze_ipc_md_config { + uct_md_config_t super; + int device_ordinal; +} uct_ze_ipc_md_config_t; + + +/** + * @brief ze ipc remote key for put/get + */ +typedef struct uct_ze_ipc_key { + ze_ipc_mem_handle_t ipc_handle; /**< IPC memory handle */ + pid_t pid; /**< Remote process ID (for cache) */ + uintptr_t address; /**< Base address of the allocation */ + size_t length; /**< Size of the allocation */ + int dev_num; /**< GPU device number */ +} uct_ze_ipc_key_t; + + +#endif From 930f1de6c12fa4b031dd8f8b5cbb46dbd28ac194 Mon Sep 17 00:00:00 2001 From: yuanwu Date: Thu, 7 May 2026 08:28:26 +0000 Subject: [PATCH 02/35] UCT/ZE/IPC: adapt to upstream API changes after merge - Remove iface_mem_element_pack from uct_iface_internal_ops (field removed in upstream) - Add uct_ze_base_get_device(int ordinal) helper that returns the root device handle by ordinal, replacing the legacy direct lookup via uct_ze_base_info.devices[i] which was renamed to uct_ze_base.devices[i].root_device Signed-off-by: yuanwu --- src/uct/ze/base/ze_base.c | 13 +++++++++++++ src/uct/ze/base/ze_base.h | 3 +++ src/uct/ze/ze_ipc/ze_ipc_iface.c | 1 - 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/uct/ze/base/ze_base.c b/src/uct/ze/base/ze_base.c index d105887a2a8..a7a21453980 100644 --- a/src/uct/ze/base/ze_base.c +++ b/src/uct/ze/base/ze_base.c @@ -391,6 +391,19 @@ int uct_ze_base_get_device_ordinal(ze_device_handle_t device) return -1; /* not found */ } +ze_device_handle_t uct_ze_base_get_device(int ordinal) +{ + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + return NULL; + } + + if ((ordinal < 0) || (ordinal >= uct_ze_base.num_devices)) { + return NULL; + } + + return uct_ze_base.devices[ordinal].root_device; +} + int uct_ze_base_get_num_devices(void) { diff --git a/src/uct/ze/base/ze_base.h b/src/uct/ze/base/ze_base.h index f2f4118189e..26166f02c05 100644 --- a/src/uct/ze/base/ze_base.h +++ b/src/uct/ze/base/ze_base.h @@ -68,6 +68,9 @@ ucs_sys_device_t uct_ze_base_get_sys_dev_from_handle(ze_device_handle_t device); /* Get device ordinal from device handle, returns -1 if not found */ int uct_ze_base_get_device_ordinal(ze_device_handle_t device); +/* Get root device handle by ordinal, returns NULL if out of range */ +ze_device_handle_t uct_ze_base_get_device(int ordinal); + int uct_ze_base_get_num_devices(void); diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index d91f5d826ba..7464b6dade1 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -445,7 +445,6 @@ static uct_iface_internal_ops_t uct_ze_ipc_iface_internal_ops = { .iface_query_v2 = uct_iface_base_query_v2, .iface_estimate_perf = uct_ze_ipc_estimate_perf, .iface_vfs_refresh = (uct_iface_vfs_refresh_func_t)ucs_empty_function, - .iface_mem_element_pack = (uct_iface_mem_element_pack_func_t)ucs_empty_function_return_unsupported, .ep_query = (uct_ep_query_func_t)ucs_empty_function_return_unsupported, .ep_invalidate = (uct_ep_invalidate_func_t)ucs_empty_function_return_unsupported, .ep_connect_to_ep_v2 = (uct_ep_connect_to_ep_v2_func_t)ucs_empty_function_return_unsupported, From 080ebe153ec7aee0232831f629f3ca5cb86315af Mon Sep 17 00:00:00 2001 From: yuanwu Date: Thu, 7 May 2026 08:55:38 +0000 Subject: [PATCH 03/35] TEST/UCT/ZE: add ze_ipc gtest unit tests - test_ze_base: validates uct_ze_base_get_device / get_num_devices / get_device_ordinal helpers added during the upstream-master merge. - test_ze_ipc_md: smoke-tests component registration, MD resource query, MD open/close (verifies ze_device + ze_context populated per sub-device after the merge), and md_attr.reg_mem_types. - Wire HAVE_ZE block in test/gtest/Makefile.am: appends sources, ZE_CPPFLAGS, ZE_LDFLAGS, ZE_LIBS and libuct_ze.la. Signed-off-by: yuanwu --- test/gtest/Makefile.am | 12 +++ test/gtest/uct/ze/test_ze_base.cc | 84 +++++++++++++++++ test/gtest/uct/ze/test_ze_ipc_md.cc | 140 ++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+) create mode 100644 test/gtest/uct/ze/test_ze_base.cc create mode 100644 test/gtest/uct/ze/test_ze_ipc_md.cc diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index d6f676ebd4d..30fa8063049 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -350,6 +350,18 @@ gtest_LDADD += \ $(ZE_LIBS) endif +if HAVE_ZE +gtest_SOURCES += \ + uct/ze/test_ze_base.cc \ + uct/ze/test_ze_ipc_md.cc +gtest_CPPFLAGS += \ + $(ZE_CPPFLAGS) +gtest_LDADD += \ + $(ZE_LDFLAGS) \ + $(ZE_LIBS) \ + $(top_builddir)/src/uct/ze/libuct_ze.la +endif + noinst_HEADERS = \ common/mem_buffer.h \ common/test.h \ diff --git a/test/gtest/uct/ze/test_ze_base.cc b/test/gtest/uct/ze/test_ze_base.cc new file mode 100644 index 00000000000..6c05acb6146 --- /dev/null +++ b/test/gtest/uct/ze/test_ze_base.cc @@ -0,0 +1,84 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +} + + +/* + * Unit tests for the helpers added to make ZE_IPC compatible with the + * post-merge upstream uct_ze_base data layout (root device per index). + */ +class test_ze_base : public ucs::test { +}; + + +UCS_TEST_F(test_ze_base, init_succeeds_or_skip) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } +} + + +UCS_TEST_F(test_ze_base, get_num_devices_consistent) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + + int n = uct_ze_base_get_num_devices(); + EXPECT_GE(n, 0); + + /* second call must be stable */ + EXPECT_EQ(n, uct_ze_base_get_num_devices()); +} + + +UCS_TEST_F(test_ze_base, get_device_by_ordinal_round_trip) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + + int n = uct_ze_base_get_num_devices(); + if (n == 0) { + UCS_TEST_SKIP_R("No Level Zero devices available"); + } + + for (int i = 0; i < n; ++i) { + ze_device_handle_t dev = uct_ze_base_get_device(i); + ASSERT_TRUE(dev != NULL) << "ordinal " << i; + + /* round-trip: handle -> ordinal must give same index back */ + EXPECT_EQ(i, uct_ze_base_get_device_ordinal(dev)); + } +} + + +UCS_TEST_F(test_ze_base, get_device_out_of_range) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + + int n = uct_ze_base_get_num_devices(); + + EXPECT_TRUE(uct_ze_base_get_device(-1) == NULL); + EXPECT_TRUE(uct_ze_base_get_device(n) == NULL); + EXPECT_TRUE(uct_ze_base_get_device(n + 100) == NULL); +} + + +UCS_TEST_F(test_ze_base, get_device_ordinal_unknown_handle) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + + /* Bogus pointer that cannot be a registered device handle. */ + ze_device_handle_t bogus = + reinterpret_cast(uintptr_t(0xdead)); + EXPECT_EQ(-1, uct_ze_base_get_device_ordinal(bogus)); + EXPECT_EQ(-1, uct_ze_base_get_device_ordinal(NULL)); +} diff --git a/test/gtest/uct/ze/test_ze_ipc_md.cc b/test/gtest/uct/ze/test_ze_ipc_md.cc new file mode 100644 index 00000000000..a60561bea69 --- /dev/null +++ b/test/gtest/uct/ze/test_ze_ipc_md.cc @@ -0,0 +1,140 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +#include +#include +} + + +/* + * Lightweight smoke tests for the ZE_IPC component. They do not require a + * peer process; they validate that the component is registered, that it + * advertises at least one MD resource, and that the MD opens/closes cleanly + * with the post-merge per-sub-device layout. + */ +class test_ze_ipc_md : public ucs::test { +protected: + void SetUp() override { + ucs::test::SetUp(); + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + if (uct_ze_base_get_num_devices() == 0) { + UCS_TEST_SKIP_R("No Level Zero devices available"); + } + } + + /* Query MD resources via the component API used by the rest of UCT. */ + static ucs_status_t query_md_resources(uct_md_resource_desc_t **resources_p, + unsigned *num_p) + { + uct_component_attr_t attr; + ucs_status_t status; + + attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT; + status = uct_component_query(&uct_ze_ipc_component, &attr); + if (status != UCS_OK) { + return status; + } + + unsigned count = attr.md_resource_count; + uct_md_resource_desc_t *res = (uct_md_resource_desc_t *) + ucs_calloc(count, sizeof(uct_md_resource_desc_t), + "ze_ipc_md_resources"); + if ((res == NULL) && (count > 0)) { + return UCS_ERR_NO_MEMORY; + } + + attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES; + attr.md_resources = res; + status = uct_component_query(&uct_ze_ipc_component, &attr); + if (status != UCS_OK) { + ucs_free(res); + return status; + } + + *resources_p = res; + *num_p = count; + return UCS_OK; + } + + static uct_md_h open_first_md() { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + + if (query_md_resources(&resources, &num) != UCS_OK) { + return NULL; + } + if (num == 0) { + ucs_free(resources); + return NULL; + } + + uct_md_config_t *md_config = NULL; + ucs_status_t status = uct_md_config_read(&uct_ze_ipc_component, NULL, + NULL, &md_config); + if (status != UCS_OK) { + ucs_free(resources); + return NULL; + } + + uct_md_h md = NULL; + status = uct_md_open(&uct_ze_ipc_component, resources[0].md_name, + md_config, &md); + uct_config_release(md_config); + ucs_free(resources); + + return (status == UCS_OK) ? md : NULL; + } +}; + + +UCS_TEST_F(test_ze_ipc_md, component_is_registered) { + EXPECT_STREQ("ze_ipc", uct_ze_ipc_component.name); +} + + +UCS_TEST_F(test_ze_ipc_md, query_md_resources_returns_at_least_one) { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + + EXPECT_UCS_OK(query_md_resources(&resources, &num)); + EXPECT_GE(num, 1u); + ucs_free(resources); +} + + +UCS_TEST_F(test_ze_ipc_md, open_close_md) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + + /* The post-merge ze_ipc_md must populate device + context. */ + auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + EXPECT_TRUE(ze_md->ze_device != NULL); + EXPECT_TRUE(ze_md->ze_context != NULL); + + uct_md_close(md); +} + + +UCS_TEST_F(test_ze_ipc_md, md_attr_advertises_registerable_mem) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + + uct_md_attr_v2_t attr; + attr.field_mask = UINT64_MAX; + EXPECT_UCS_OK(uct_md_query_v2(md, &attr)); + EXPECT_NE(0u, attr.reg_mem_types) << "no registerable mem types"; + + uct_md_close(md); +} From f84c422312f9e23d948b377dd97a0384b6bd7535 Mon Sep 17 00:00:00 2001 From: yuanwu Date: Fri, 8 May 2026 01:23:47 +0000 Subject: [PATCH 04/35] TEST/UCT/ZE: extend ze_ipc/ze_copy MD coverage Add end-to-end mem_reg/mkey_pack/mem_dereg test for ze_ipc_md (NIXL KV path), ze_ipc_cache lifecycle tests, and full ze_copy_md coverage (open/close, alloc/free, detect_memory_type, mem_query). 21 tests total under HAVE_ZE; all pass on Intel PVC. Signed-off-by: yuanwu --- test/gtest/Makefile.am | 4 +- test/gtest/uct/ze/test_ze_copy_md.cc | 200 +++++++++++++++++++++++++ test/gtest/uct/ze/test_ze_ipc_cache.cc | 62 ++++++++ test/gtest/uct/ze/test_ze_ipc_md.cc | 69 +++++++++ 4 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 test/gtest/uct/ze/test_ze_copy_md.cc create mode 100644 test/gtest/uct/ze/test_ze_ipc_cache.cc diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index 30fa8063049..4c4a287793f 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -353,7 +353,9 @@ endif if HAVE_ZE gtest_SOURCES += \ uct/ze/test_ze_base.cc \ - uct/ze/test_ze_ipc_md.cc + uct/ze/test_ze_ipc_md.cc \ + uct/ze/test_ze_ipc_cache.cc \ + uct/ze/test_ze_copy_md.cc gtest_CPPFLAGS += \ $(ZE_CPPFLAGS) gtest_LDADD += \ diff --git a/test/gtest/uct/ze/test_ze_copy_md.cc b/test/gtest/uct/ze/test_ze_copy_md.cc new file mode 100644 index 00000000000..7dc73f14b5d --- /dev/null +++ b/test/gtest/uct/ze/test_ze_copy_md.cc @@ -0,0 +1,200 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +#include +#include +} + + +/* + * Smoke tests for the ze_copy MD: lifecycle, capability query, allocation + * and memory-type detection. The ze_copy MD is the path used by UCP for + * VRAM<->host staging on Intel XPU. + */ +class test_ze_copy_md : public ucs::test { +protected: + void SetUp() override { + ucs::test::SetUp(); + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + if (uct_ze_base_get_num_devices() == 0) { + UCS_TEST_SKIP_R("No Level Zero devices available"); + } + } + + static uct_md_h open_first_md() { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + ucs_status_t status; + + status = uct_ze_copy_component.query_md_resources(&uct_ze_copy_component, + &resources, &num); + EXPECT_UCS_OK(status); + if (num == 0) { + ucs_free(resources); + return NULL; + } + + uct_md_config_t *md_config = NULL; + status = uct_md_config_read(&uct_ze_copy_component, NULL, NULL, + &md_config); + EXPECT_UCS_OK(status); + + uct_md_h md = NULL; + status = uct_md_open(&uct_ze_copy_component, resources[0].md_name, + md_config, &md); + uct_config_release(md_config); + ucs_free(resources); + + return (status == UCS_OK) ? md : NULL; + } +}; + + +UCS_TEST_F(test_ze_copy_md, component_is_registered) { + EXPECT_STREQ("ze_cpy", uct_ze_copy_component.name); +} + + +UCS_TEST_F(test_ze_copy_md, query_md_resources_returns_at_least_one) { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + + EXPECT_UCS_OK(uct_ze_copy_component.query_md_resources( + &uct_ze_copy_component, &resources, &num)); + EXPECT_GE(num, 1u); + ucs_free(resources); +} + + +UCS_TEST_F(test_ze_copy_md, open_close_md) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + auto *ze_md = ucs_derived_of(md, uct_ze_copy_md_t); + EXPECT_TRUE(ze_md->ze_device != NULL); + EXPECT_TRUE(ze_md->ze_context != NULL); + + uct_md_close(md); +} + + +UCS_TEST_F(test_ze_copy_md, md_attr_advertises_alloc_caps) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + uct_md_attr_v2_t attr; + attr.field_mask = UINT64_MAX; + EXPECT_UCS_OK(uct_md_query_v2(md, &attr)); + + /* ze_copy MD must be able to allocate ZE memory. */ + EXPECT_NE(0u, attr.alloc_mem_types); + EXPECT_GT(attr.max_alloc, 0u); + + uct_md_close(md); +} + + +/* + * Allocate ZE_DEVICE memory through the MD, then free it. + */ +UCS_TEST_F(test_ze_copy_md, mem_alloc_free_device) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + size_t length = 4096; + void *addr = NULL; + uct_mem_h memh = NULL; + + ucs_status_t status = + md->ops->mem_alloc(md, &length, &addr, UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); + if (status == UCS_ERR_UNSUPPORTED) { + UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); + } + ASSERT_UCS_OK(status); + ASSERT_TRUE(addr != NULL); + ASSERT_GE(length, 4096u); + + EXPECT_UCS_OK(md->ops->mem_free(md, memh)); + uct_md_close(md); +} + + +/* + * detect_memory_type must classify ZE device pointers as ZE_DEVICE. + */ +UCS_TEST_F(test_ze_copy_md, detect_memory_type_device) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + size_t length = 4096; + void *addr = NULL; + uct_mem_h memh = NULL; + ucs_status_t status = + md->ops->mem_alloc(md, &length, &addr, UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); + if (status == UCS_ERR_UNSUPPORTED) { + UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); + } + ASSERT_UCS_OK(status); + + ucs_memory_type_t mem_type = UCS_MEMORY_TYPE_HOST; + EXPECT_UCS_OK(md->ops->detect_memory_type(md, addr, length, &mem_type)); + EXPECT_EQ(UCS_MEMORY_TYPE_ZE_DEVICE, mem_type); + + EXPECT_UCS_OK(md->ops->mem_free(md, memh)); + uct_md_close(md); +} + + +/* + * mem_query reports base address and allocation length back. + */ +UCS_TEST_F(test_ze_copy_md, mem_query_base_and_length) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + size_t length = 8192; + void *addr = NULL; + uct_mem_h memh = NULL; + ucs_status_t status = + md->ops->mem_alloc(md, &length, &addr, UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); + if (status == UCS_ERR_UNSUPPORTED) { + UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); + } + ASSERT_UCS_OK(status); + + uct_md_mem_attr_t mem_attr = {}; + mem_attr.field_mask = UCT_MD_MEM_ATTR_FIELD_MEM_TYPE | + UCT_MD_MEM_ATTR_FIELD_BASE_ADDRESS | + UCT_MD_MEM_ATTR_FIELD_ALLOC_LENGTH; + EXPECT_UCS_OK(md->ops->mem_query(md, addr, length, &mem_attr)); + EXPECT_EQ(UCS_MEMORY_TYPE_ZE_DEVICE, mem_attr.mem_type); + EXPECT_TRUE(mem_attr.base_address != NULL); + EXPECT_GE(mem_attr.alloc_length, length); + + EXPECT_UCS_OK(md->ops->mem_free(md, memh)); + uct_md_close(md); +} diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc new file mode 100644 index 00000000000..a774069203a --- /dev/null +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -0,0 +1,62 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +#include +} + + +/* + * Lifecycle tests for the ze_ipc cache. The cache is a simple pgtable- + * based handle map with no LRU; we only validate that it can be created + * and destroyed cleanly, including back-to-back and multiple instances. + */ +class test_ze_ipc_cache : public ucs::test { +protected: + void SetUp() override { + ucs::test::SetUp(); + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + } +}; + + +UCS_TEST_F(test_ze_ipc_cache, create_destroy) { + uct_ze_ipc_cache_t *cache = NULL; + ASSERT_UCS_OK(uct_ze_ipc_create_cache(&cache, "test")); + ASSERT_TRUE(cache != NULL); + uct_ze_ipc_destroy_cache(cache); +} + + +UCS_TEST_F(test_ze_ipc_cache, create_destroy_repeated) { + for (int i = 0; i < 16; ++i) { + uct_ze_ipc_cache_t *cache = NULL; + ASSERT_UCS_OK(uct_ze_ipc_create_cache(&cache, "test_repeat")); + ASSERT_TRUE(cache != NULL); + uct_ze_ipc_destroy_cache(cache); + } +} + + +UCS_TEST_F(test_ze_ipc_cache, multiple_instances) { + const int N = 8; + uct_ze_ipc_cache_t *caches[N] = {}; + + for (int i = 0; i < N; ++i) { + ASSERT_UCS_OK(uct_ze_ipc_create_cache(&caches[i], "test_multi")); + ASSERT_TRUE(caches[i] != NULL); + } + for (int i = 0; i < N; ++i) { + uct_ze_ipc_destroy_cache(caches[i]); + } +} + + + diff --git a/test/gtest/uct/ze/test_ze_ipc_md.cc b/test/gtest/uct/ze/test_ze_ipc_md.cc index a60561bea69..2d7708462e0 100644 --- a/test/gtest/uct/ze/test_ze_ipc_md.cc +++ b/test/gtest/uct/ze/test_ze_ipc_md.cc @@ -135,6 +135,75 @@ UCS_TEST_F(test_ze_ipc_md, md_attr_advertises_registerable_mem) { attr.field_mask = UINT64_MAX; EXPECT_UCS_OK(uct_md_query_v2(md, &attr)); EXPECT_NE(0u, attr.reg_mem_types) << "no registerable mem types"; + /* IPC MD must advertise the rkey size and require an rkey. */ + EXPECT_GT(attr.rkey_packed_size, 0u); + EXPECT_TRUE(attr.flags & UCT_MD_FLAG_NEED_RKEY); + EXPECT_TRUE(attr.flags & UCT_MD_FLAG_REG); + + uct_md_close(md); +} + + +/* + * End-to-end: allocate ZE device memory, register with the IPC MD, pack + * the IPC key, and deregister. This is the path NIXL relies on for KV + * cache exchange (mem_reg -> mkey_pack -> mem_dereg). + */ +UCS_TEST_F(test_ze_ipc_md, mem_reg_pack_dereg_device) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + + const size_t size = 4096; + void *dev_ptr = NULL; + ze_device_mem_alloc_desc_t dev_desc = {}; + dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, + ze_md->ze_device, &dev_ptr)); + ASSERT_TRUE(dev_ptr != NULL); + + uct_mem_h memh = NULL; + EXPECT_UCS_OK(md->ops->mem_reg(md, dev_ptr, size, NULL, &memh)); + ASSERT_TRUE(memh != NULL); + + uct_ze_ipc_key_t packed = {}; + EXPECT_UCS_OK(md->ops->mkey_pack(md, memh, dev_ptr, size, NULL, &packed)); + EXPECT_EQ((pid_t)getpid(), packed.pid); + EXPECT_GE(packed.length, size); + EXPECT_NE(0u, packed.address); + + uct_md_mem_dereg_params_t dereg_params = {}; + dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; + dereg_params.memh = memh; + EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); + + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, dev_ptr)); + uct_md_close(md); +} + + +/* + * mem_reg must reject non-ZE pointers (the IPC MD only supports VRAM). + * The MD logs an error in this path, so we suppress error log capture. + */ +UCS_TEST_F(test_ze_ipc_md, mem_reg_rejects_host_pointer) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + + char buf[64]; + uct_mem_h memh = NULL; + ucs_status_t status; + { + scoped_log_handler slh(hide_errors_logger); + status = md->ops->mem_reg(md, buf, sizeof(buf), NULL, &memh); + } + EXPECT_EQ(UCS_ERR_INVALID_ADDR, status); uct_md_close(md); } From 5939faf963149426483870b51e86c709871e756d Mon Sep 17 00:00:00 2001 From: yuanwu Date: Mon, 11 May 2026 05:20:11 +0000 Subject: [PATCH 05/35] AUTHORS: add yuanwu Signed-off-by: yuanwu --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 1f7b2218fb9..6ea6c87fefa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -145,6 +145,7 @@ Zihao Zhao lzhang2 michaelzli ybenabou +yuanwu In addition we would like to acknowledge the following members of UCX community for their participation in annual face-to-face meeting, design discussions, and From 4c050011a8bec224602f873eff93f7b8753a2726 Mon Sep 17 00:00:00 2001 From: yuanwu Date: Sun, 31 May 2026 20:01:50 -0700 Subject: [PATCH 06/35] TEST/GTEST/ZE: apply clang-format fixes CI codestyle check (git-clang-format) flagged two alignment issues in test_ze_ipc_md.cc: - align uct_md_mem_dereg_params_t initializer columns with the surrounding declaration block - drop the over-aligned spaces in the host-pointer rejection test so only ucs_status_t (the longest type on that block) drives alignment Signed-off-by: yuanwu --- test/gtest/uct/ze/test_ze_ipc_md.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/gtest/uct/ze/test_ze_ipc_md.cc b/test/gtest/uct/ze/test_ze_ipc_md.cc index 2d7708462e0..18c39bf4dc9 100644 --- a/test/gtest/uct/ze/test_ze_ipc_md.cc +++ b/test/gtest/uct/ze/test_ze_ipc_md.cc @@ -177,8 +177,8 @@ UCS_TEST_F(test_ze_ipc_md, mem_reg_pack_dereg_device) { EXPECT_NE(0u, packed.address); uct_md_mem_dereg_params_t dereg_params = {}; - dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; - dereg_params.memh = memh; + dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; + dereg_params.memh = memh; EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, dev_ptr)); @@ -196,8 +196,8 @@ UCS_TEST_F(test_ze_ipc_md, mem_reg_rejects_host_pointer) { UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); } - char buf[64]; - uct_mem_h memh = NULL; + char buf[64]; + uct_mem_h memh = NULL; ucs_status_t status; { scoped_log_handler slh(hide_errors_logger); From 7360c5acdf081af75d42d14ffd0e0b6e235a44e3 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 21 May 2026 10:02:51 +0800 Subject: [PATCH 07/35] UCT/ZE/ZE_IPC: Update src/uct/ze/copy/ze_copy_ep.c Co-authored-by: Yaser Afshar --- src/uct/ze/copy/ze_copy_ep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uct/ze/copy/ze_copy_ep.c b/src/uct/ze/copy/ze_copy_ep.c index a46611b9aeb..3da5da08314 100644 --- a/src/uct/ze/copy/ze_copy_ep.c +++ b/src/uct/ze/copy/ze_copy_ep.c @@ -169,7 +169,7 @@ ucs_status_t uct_ze_copy_ep_get_short(uct_ep_h tl_ep, void *buffer, }; ucs_status_t status; - ucs_info("ze_copy_ep: GET_SHORT buffer=%p length=%u remote_addr=%p", + ucs_trace("ze_copy_ep: GET_SHORT buffer=%p length=%u remote_addr=%p", buffer, length, (void*)remote_addr); status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 0); From 4bc827a3fb36ac28ddbf213a4890ed3091aaefc6 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 21 May 2026 10:03:08 +0800 Subject: [PATCH 08/35] UCT/ZE/ZE_IPC: Update src/uct/ze/copy/ze_copy_ep.c Co-authored-by: Yaser Afshar --- src/uct/ze/copy/ze_copy_ep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uct/ze/copy/ze_copy_ep.c b/src/uct/ze/copy/ze_copy_ep.c index 3da5da08314..e203bbcb652 100644 --- a/src/uct/ze/copy/ze_copy_ep.c +++ b/src/uct/ze/copy/ze_copy_ep.c @@ -50,7 +50,7 @@ ucs_status_t uct_ze_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr, ze_result_t ret; void *src, *dst; - ucs_info("ze_copy_ep: zcopy %s remote_addr=%p local_buf=%p size=%zu", + ucs_trace("ze_copy_ep: zcopy %s remote_addr=%p local_buf=%p size=%zu", is_put ? "PUT" : "GET", (void*)remote_addr, iov->buffer, size); if (is_put) { From 094c2b97aceda4e586944a1eab539612b1e34b81 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 21 May 2026 10:03:21 +0800 Subject: [PATCH 09/35] UCT/ZE/ZE_IPC: Update src/uct/ze/copy/ze_copy_ep.c Co-authored-by: Yaser Afshar --- src/uct/ze/copy/ze_copy_ep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uct/ze/copy/ze_copy_ep.c b/src/uct/ze/copy/ze_copy_ep.c index e203bbcb652..c3e3e462235 100644 --- a/src/uct/ze/copy/ze_copy_ep.c +++ b/src/uct/ze/copy/ze_copy_ep.c @@ -147,7 +147,7 @@ ucs_status_t uct_ze_copy_ep_put_short(uct_ep_h tl_ep, const void *buffer, }; ucs_status_t status; - ucs_info("ze_copy_ep: PUT_SHORT buffer=%p length=%u remote_addr=%p", + ucs_trace("ze_copy_ep: PUT_SHORT buffer=%p length=%u remote_addr=%p", buffer, length, (void*)remote_addr); status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 1); From 19a62554bc40c80ea4d32dfdf1bf6bdbbf84f0af Mon Sep 17 00:00:00 2001 From: yuanwu Date: Mon, 1 Jun 2026 23:37:10 -0700 Subject: [PATCH 10/35] AUTHORS: add missing contributors from ZE IPC PR Signed-off-by: yuanwu --- AUTHORS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AUTHORS b/AUTHORS index 6ea6c87fefa..ecd8912c8df 100644 --- a/AUTHORS +++ b/AUTHORS @@ -63,6 +63,7 @@ Jeff Daily Jeffrey Mahou Jianxin Xiong John Snyder +Jon Stevens <159235294+jon-hotaisle@users.noreply.github.com> Jonas Zhou Joseph Schuchart Keisuke Fukuda @@ -83,6 +84,7 @@ Michael Braverman Michal Shalev Mike Dubman Mikhail Brinskii +Mikhail Brinskiy Min Fang Nathan Bellalou Nathan Hjelm @@ -144,6 +146,8 @@ Zhu Yanjun Zihao Zhao lzhang2 michaelzli +shasson5 <103439971+shasson5@users.noreply.github.com> +wehy <18174858434@163.com> ybenabou yuanwu From ac3cd635d21413b97998aea770d0f42faacf7b3d Mon Sep 17 00:00:00 2001 From: yuanwu Date: Mon, 29 Jun 2026 01:40:21 -0700 Subject: [PATCH 11/35] TEST/ZE: Adapt test_ze_copy_md to uct_md_mem_attr_v2_t The mem_query MD op signature was changed on master to take uct_md_mem_attr_v2_t* instead of uct_md_mem_attr_t*. Update the ZE test to use the v2 struct and the corresponding V2 field-mask enums, matching the new API and fixing the build error. Signed-off-by: yuanwu --- test/gtest/uct/ze/test_ze_copy_md.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/gtest/uct/ze/test_ze_copy_md.cc b/test/gtest/uct/ze/test_ze_copy_md.cc index 7dc73f14b5d..cad68b03610 100644 --- a/test/gtest/uct/ze/test_ze_copy_md.cc +++ b/test/gtest/uct/ze/test_ze_copy_md.cc @@ -186,10 +186,10 @@ UCS_TEST_F(test_ze_copy_md, mem_query_base_and_length) { } ASSERT_UCS_OK(status); - uct_md_mem_attr_t mem_attr = {}; - mem_attr.field_mask = UCT_MD_MEM_ATTR_FIELD_MEM_TYPE | - UCT_MD_MEM_ATTR_FIELD_BASE_ADDRESS | - UCT_MD_MEM_ATTR_FIELD_ALLOC_LENGTH; + uct_md_mem_attr_v2_t mem_attr = {}; + mem_attr.field_mask = UCT_MD_MEM_ATTR_V2_FIELD_MEM_TYPE | + UCT_MD_MEM_ATTR_V2_FIELD_BASE_ADDRESS | + UCT_MD_MEM_ATTR_V2_FIELD_ALLOC_LENGTH; EXPECT_UCS_OK(md->ops->mem_query(md, addr, length, &mem_attr)); EXPECT_EQ(UCS_MEMORY_TYPE_ZE_DEVICE, mem_attr.mem_type); EXPECT_TRUE(mem_attr.base_address != NULL); From 060e4a52b982565dcf417ead473557101af19557 Mon Sep 17 00:00:00 2001 From: yuanwu Date: Fri, 17 Jul 2026 02:58:03 -0700 Subject: [PATCH 12/35] AUTHORS: Revert adding contributors already covered by name/email aliases This reverts commit 59c0d6a394c250a834318b42d4700fd30dd7bb0a. Reviewer @brminich noted that his newly added entry duplicates an existing one (`Mikhail Brinskii `). The extra lines were originally appended by contrib/authors_update.sh on an earlier merge range; those authors are no longer present in the current PR range (upstream/master..HEAD contains only yuanwu and Cherry Zhang commits), so the AUTHORS-file CI check will still pass without them. Signed-off-by: yuanwu --- AUTHORS | 4 ---- 1 file changed, 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index ecd8912c8df..6ea6c87fefa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -63,7 +63,6 @@ Jeff Daily Jeffrey Mahou Jianxin Xiong John Snyder -Jon Stevens <159235294+jon-hotaisle@users.noreply.github.com> Jonas Zhou Joseph Schuchart Keisuke Fukuda @@ -84,7 +83,6 @@ Michael Braverman Michal Shalev Mike Dubman Mikhail Brinskii -Mikhail Brinskiy Min Fang Nathan Bellalou Nathan Hjelm @@ -146,8 +144,6 @@ Zhu Yanjun Zihao Zhao lzhang2 michaelzli -shasson5 <103439971+shasson5@users.noreply.github.com> -wehy <18174858434@163.com> ybenabou yuanwu From ffa74e8e5375a01c274b02bec705b12119faca1f Mon Sep 17 00:00:00 2001 From: yuanwu Date: Fri, 17 Jul 2026 03:23:22 -0700 Subject: [PATCH 13/35] AUTHORS: Add Jon Stevens Restore the Jon Stevens entry that the previous revert dropped. The Azure AUTHORS-file check runs contrib/authors_update.sh over the PR merge range and flagged this contributor as missing (++ in the CI log). The other three names removed by the revert (Mikhail Brinskiy, shasson5, wehy) are still covered by existing AUTHORS lines and are not reported as missing by the check. Signed-off-by: yuanwu --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 6ea6c87fefa..ccc562e89b9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -63,6 +63,7 @@ Jeff Daily Jeffrey Mahou Jianxin Xiong John Snyder +Jon Stevens <159235294+jon-hotaisle@users.noreply.github.com> Jonas Zhou Joseph Schuchart Keisuke Fukuda From edcd8799faf8702c81acc0c641c522d9b9b0ed4c Mon Sep 17 00:00:00 2001 From: yuanwu Date: Fri, 17 Jul 2026 03:27:40 -0700 Subject: [PATCH 14/35] AUTHORS: Add shasson5 and wehy CI AUTHORS-file check still flags these two as missing (++). The authors_update.sh dedup is name-substring based and does not recognize shasson5 / wehy as aliases of any existing entry, so add them explicitly to satisfy git diff --exit-code. Signed-off-by: yuanwu --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index ccc562e89b9..149e0d3056b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -145,6 +145,8 @@ Zhu Yanjun Zihao Zhao lzhang2 michaelzli +shasson5 <103439971+shasson5@users.noreply.github.com> +wehy <18174858434@163.com> ybenabou yuanwu From 09c47113d2ceef7d3f5d72e30126b89213e82f9b Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 08:41:22 -0700 Subject: [PATCH 15/35] UCT/ZE_IPC: release IPC handle on mem_dereg mem_reg called zeMemGetIpcHandle() but mem_dereg never released it with zeMemPutIpcHandle(), leaking a driver resource on every dereg. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_md.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index 27311aa0657..768691f8869 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -189,13 +189,21 @@ uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, static ucs_status_t -uct_ze_ipc_mem_dereg(uct_md_h md, const uct_md_mem_dereg_params_t *params) +uct_ze_ipc_mem_dereg(uct_md_h uct_md, const uct_md_mem_dereg_params_t *params) { + uct_ze_ipc_md_t *md = ucs_derived_of(uct_md, uct_ze_ipc_md_t); uct_ze_ipc_key_t *key; + ze_result_t ret; UCT_MD_MEM_DEREG_CHECK_PARAMS(params, 0); key = params->memh; + + ret = zeMemPutIpcHandle(md->ze_context, key->ipc_handle); + if (ret != ZE_RESULT_SUCCESS) { + ucs_warn("zeMemPutIpcHandle failed with error 0x%x", ret); + } + ucs_free(key); return UCS_OK; } From 195e8177ab6fa18ca6c940ab617e1c079105e4f4 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 08:41:41 -0700 Subject: [PATCH 16/35] UCT/ZE_IPC: invalidate stale cache mappings, purge on MD close Cache hits matched on (pid, address) alone, so a freed-and-reused remote VA returned a mapping to the wrong allocation. Compare the IPC handle and length on hit and remap on mismatch. Also purge cached regions tied to a context before it is destroyed in md_close(), so a later close doesn't reference a torn-down context. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_cache.c | 62 +++++++++++++++++++++++++------- src/uct/ze/ze_ipc/ze_ipc_cache.h | 11 +++++- src/uct/ze/ze_ipc/ze_ipc_md.c | 2 ++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index 8bdd77a683d..90062a59edd 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -307,17 +307,33 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); - /* cache hit */ - ucs_trace("%s: ze_ipc cache hit addr:%p size:%lu region:" - UCS_PGT_REGION_FMT, cache->name, (void *)key->address, - key->length, UCS_PGT_REGION_ARG(®ion->super)); - - *mapped_addr = region->mapped_addr; - *dup_fd = region->dup_fd; - ucs_assert(region->refcount < UINT64_MAX); - region->refcount++; - pthread_rwlock_unlock(&cache->lock); - return UCS_OK; + if (ucs_likely((region->key.length == key->length) && + !memcmp(®ion->key.ipc_handle, &key->ipc_handle, + sizeof(key->ipc_handle)))) { + /* cache hit */ + ucs_trace("%s: ze_ipc cache hit addr:%p size:%lu region:" + UCS_PGT_REGION_FMT, cache->name, (void *)key->address, + key->length, UCS_PGT_REGION_ARG(®ion->super)); + + *mapped_addr = region->mapped_addr; + *dup_fd = region->dup_fd; + ucs_assert(region->refcount < UINT64_MAX); + region->refcount++; + pthread_rwlock_unlock(&cache->lock); + return UCS_OK; + } + + /* remote allocation at this address was freed and a different one + * reused the same VA - drop the stale mapping and remap below */ + ucs_trace("%s: ze_ipc cache stale mapping addr:%p, invalidating", + cache->name, (void *)key->address); + status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_error("failed to remove address:%p from cache (%s)", + (void *)region->key.address, ucs_status_string(status)); + } + uct_ze_ipc_close_memhandle(region); + ucs_free(region); } status = uct_ze_ipc_open_memhandle(key, ze_context, ze_device, mapped_addr, dup_fd); @@ -429,6 +445,28 @@ void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache) } +void uct_ze_ipc_purge_cache_by_context(ze_context_handle_t ze_context) +{ + uct_ze_ipc_cache_t *cache; + khint_t khiter; + + ucs_recursive_spin_lock(&uct_ze_ipc_remote_cache.lock); + for (khiter = kh_begin(&uct_ze_ipc_remote_cache.hash); + khiter != kh_end(&uct_ze_ipc_remote_cache.hash); ++khiter) { + if (!kh_exist(&uct_ze_ipc_remote_cache.hash, khiter) || + (kh_key(&uct_ze_ipc_remote_cache.hash, khiter).ze_context != + ze_context)) { + continue; + } + + cache = kh_val(&uct_ze_ipc_remote_cache.hash, khiter); + kh_del(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, khiter); + uct_ze_ipc_destroy_cache(cache); + } + ucs_recursive_spin_unlock(&uct_ze_ipc_remote_cache.lock); +} + + UCS_STATIC_INIT { ucs_recursive_spinlock_init(&uct_ze_ipc_remote_cache.lock, 0); kh_init_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); @@ -444,5 +482,3 @@ UCS_STATIC_CLEANUP { kh_destroy_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); ucs_recursive_spinlock_destroy(&uct_ze_ipc_remote_cache.lock); } - - diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.h b/src/uct/ze/ze_ipc/ze_ipc_cache.h index 7899da770e5..c1e80b8380b 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.h +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.h @@ -94,5 +94,14 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, ze_context_handle_t ze_context, int dup_fd, int cache_enabled); -#endif /* UCT_ZE_IPC_CACHE_H_ */ +/** + * Purge and destroy all remote caches keyed by the given local Level Zero + * context, so that closed contexts do not leave behind cache entries that + * would later be used with a destroyed context. + * + * @param ze_context Level Zero context being closed + */ +void uct_ze_ipc_purge_cache_by_context(ze_context_handle_t ze_context); + +#endif /* UCT_ZE_IPC_CACHE_H_ */ diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index 768691f8869..9327ef04a9b 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -8,6 +8,7 @@ #endif #include "ze_ipc_md.h" +#include "ze_ipc_cache.h" #include #include @@ -250,6 +251,7 @@ static void uct_ze_ipc_md_close(uct_md_h uct_md) uct_ze_ipc_md_t *md = ucs_derived_of(uct_md, uct_ze_ipc_md_t); if (md->ze_context != NULL) { + uct_ze_ipc_purge_cache_by_context(md->ze_context); zeContextDestroy(md->ze_context); } ucs_free(md); From b259d2d5a9ea3062e47596375b21e69dfb970433 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 08:41:49 -0700 Subject: [PATCH 17/35] UCT/ZE_IPC: reject multi-iov and out-of-range rkeys in zcopy Only iov[0] was ever copied, silently dropping data for iovcnt > 1; reject it like other single-iov transports do. Also validate remote_addr/length against the rkey's [address, address+length) range before mapping, closing an OOB/underflow access via a bad rkey. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_ep.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c index 24528869b63..d2d47739045 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_ep.c +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -196,6 +196,15 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, return UCS_OK; } + if (ucs_unlikely((remote_addr < key->address) || + ((offset = remote_addr - key->address) > key->length) || + (iov[0].length > (key->length - offset)))) { + ucs_error("ze_ipc_ep: remote_addr 0x%"PRIx64" length %zu is out of " + "rkey range [0x%lx..0x%lx)", remote_addr, iov[0].length, + key->address, key->address + key->length); + return UCS_ERR_INVALID_PARAM; + } + /* Use cache to map IPC handle */ status = uct_ze_ipc_map_memhandle(key, iface->ze_context, iface->ze_device, &mapped_addr, &local_fd); @@ -207,8 +216,6 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, ucs_debug("ze_ipc_ep: IPC handle mapped (cached), mapped_addr=%p", mapped_addr); - /* Calculate offset within the allocation */ - offset = remote_addr - key->address; mapped_rem_addr = (void *)((uintptr_t)mapped_addr + offset); /* Allocate event from shared pool for performance */ @@ -317,6 +324,8 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_get_zcopy, { ucs_status_t status; + UCT_CHECK_IOV_SIZE(iovcnt, 1ul, "uct_ze_ipc_ep_get_zcopy"); + status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, UCT_ZE_IPC_GET); if (UCS_STATUS_IS_ERR(status)) { @@ -340,6 +349,8 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_put_zcopy, { ucs_status_t status; + UCT_CHECK_IOV_SIZE(iovcnt, 1ul, "uct_ze_ipc_ep_put_zcopy"); + status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, UCT_ZE_IPC_PUT); if (UCS_STATUS_IS_ERR(status)) { From b53766e496ea9fa3c4ae43061af90fe21a266c33 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 08:41:58 -0700 Subject: [PATCH 18/35] UCT/ZE_IPC: drop unsignaled EVENT_FD, fix teardown double-close The eventfd was armed but never signaled on completion, so a worker could block indefinitely; drop the EVENT_FD capability instead of advertising a fd that never fires. Also route iface teardown through uct_ze_ipc_unmap_memhandle() instead of closing the IPC handle/fd directly, which double-closed shared cache entries. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_iface.c | 62 +++++--------------------------- src/uct/ze/ze_ipc/ze_ipc_iface.h | 2 -- 2 files changed, 8 insertions(+), 56 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index 7464b6dade1..d9eda561b59 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -64,45 +63,6 @@ static ucs_config_field_t uct_ze_ipc_iface_config_table[] = { static void UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t)(uct_iface_t*); -static ucs_status_t -uct_ze_ipc_iface_event_fd_get(uct_iface_h tl_iface, int *fd_p) -{ - uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); - ucs_status_t status; - - if (iface->eventfd == UCS_ASYNC_EVENTFD_INVALID_FD) { - status = ucs_async_eventfd_create(&iface->eventfd); - if (status != UCS_OK) { - return status; - } - } - - *fd_p = iface->eventfd; - return UCS_OK; -} - - -static ucs_status_t -uct_ze_ipc_iface_event_arm(uct_iface_h tl_iface, unsigned events) -{ - uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); - ucs_status_t status; - - /* Poll the eventfd to clear any pending signals */ - if (iface->eventfd != UCS_ASYNC_EVENTFD_INVALID_FD) { - status = ucs_async_eventfd_poll(iface->eventfd); - if (status == UCS_OK) { - /* There was a pending event, return BUSY */ - return UCS_ERR_BUSY; - } else if (status == UCS_ERR_IO_ERROR) { - return status; - } - } - - return UCS_OK; -} - - static ucs_status_t uct_ze_ipc_iface_get_device_address(uct_iface_t *tl_iface, uct_device_addr_t *addr) @@ -167,8 +127,7 @@ uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) UCT_IFACE_FLAG_PUT_ZCOPY; iface_attr->cap.event_flags = UCT_IFACE_FLAG_EVENT_SEND_COMP | - UCT_IFACE_FLAG_EVENT_RECV | - UCT_IFACE_FLAG_EVENT_FD; + UCT_IFACE_FLAG_EVENT_RECV; iface_attr->cap.put.max_short = 0; iface_attr->cap.put.max_bcopy = 0; @@ -389,8 +348,8 @@ static uct_iface_ops_t uct_ze_ipc_iface_ops = { .iface_progress_enable = uct_base_iface_progress_enable, .iface_progress_disable = uct_base_iface_progress_disable, .iface_progress = uct_ze_ipc_iface_progress, - .iface_event_fd_get = uct_ze_ipc_iface_event_fd_get, - .iface_event_arm = uct_ze_ipc_iface_event_arm, + .iface_event_fd_get = (uct_iface_event_fd_get_func_t)ucs_empty_function_return_unsupported, + .iface_event_arm = (uct_iface_event_arm_func_t)ucs_empty_function_return_unsupported, .iface_close = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t), .iface_query = uct_ze_ipc_iface_query, .iface_get_device_address = uct_ze_ipc_iface_get_device_address, @@ -532,7 +491,6 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, self->ze_context = ze_md->ze_context; self->ze_device = ze_md->ze_device; self->config = *config; - self->eventfd = UCS_ASYNC_EVENTFD_INVALID_FD; self->next_cmd_list = 0; /* Find copy engine queue group ordinal and available queue count */ @@ -698,10 +656,11 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) uct_ze_ipc_event_desc_t, queue); if (event_desc->mapped_addr != NULL) { - zeMemCloseIpcHandle(self->ze_context, event_desc->mapped_addr); - } - if (event_desc->dup_fd >= 0) { - close(event_desc->dup_fd); + uct_ze_ipc_unmap_memhandle(event_desc->pid, event_desc->address, + event_desc->mapped_addr, + self->ze_context, + event_desc->dup_fd, + self->config.enable_cache); } if (event_desc->event != NULL) { zeEventDestroy(event_desc->event); @@ -731,11 +690,6 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) /* Destroy spinlock */ ucs_spinlock_destroy(&self->event_lock); - - /* Close eventfd if created */ - if (self->eventfd != UCS_ASYNC_EVENTFD_INVALID_FD) { - close(self->eventfd); - } } diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.h b/src/uct/ze/ze_ipc/ze_ipc_iface.h index 06cf31319de..d09d5320e62 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.h +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.h @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -46,7 +45,6 @@ typedef struct uct_ze_ipc_iface { ze_context_handle_t ze_context; ze_device_handle_t ze_device; uct_ze_ipc_iface_config_t config; - int eventfd; /* event fd for async notifications */ /* Multi-command-list support for parallel progress */ uct_ze_ipc_queue_desc_t queue_desc[UCT_ZE_IPC_MAX_PEERS]; /* array of queue descriptors */ From 2606f0065db1a6186df157d5adfd00f5ba362f5e Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 08:43:08 -0700 Subject: [PATCH 19/35] UCT/ZE_IPC: supply ep_outstanding_purge to unblock iface init uct_base_iface_t asserts every transport supplies internal_ops.ep_outstanding_purge; ze_ipc left it NULL, aborting on iface construction. Stub it out like rocm_ipc/ze_copy/cuda_ipc do. This only unblocks startup; it does not fix the IPC correctness/ lifetime issues already addressed in prior commits. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_iface.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index d9eda561b59..3820be1787e 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -408,7 +408,8 @@ static uct_iface_internal_ops_t uct_ze_ipc_iface_internal_ops = { .ep_invalidate = (uct_ep_invalidate_func_t)ucs_empty_function_return_unsupported, .ep_connect_to_ep_v2 = (uct_ep_connect_to_ep_v2_func_t)ucs_empty_function_return_unsupported, .iface_is_reachable_v2 = uct_ze_ipc_iface_is_reachable_v2, - .ep_is_connected = uct_ze_ipc_ep_is_connected + .ep_is_connected = uct_ze_ipc_ep_is_connected, + .ep_outstanding_purge = (uct_ep_outstanding_purge_func_t)ucs_empty_function_return_unsupported }; From ab631f4d57829d3c0a2f12fd1eb0c955ed363d43 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 09:01:41 -0700 Subject: [PATCH 20/35] UCT/ZE_IPC: disable iface progress on teardown Iface cleanup freed the iface without unregistering its progress callback, leaving a dangling entry in the worker's callbackq that would later run against freed memory. Disable progress first, like tcp/mm ifaces do. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_iface.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index 3820be1787e..7608f2c476e 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -650,6 +650,9 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) uct_ze_ipc_event_desc_t *event_desc; unsigned i; + uct_base_iface_progress_disable(&self->super.super, + UCT_PROGRESS_SEND | UCT_PROGRESS_RECV); + /* Clean up outstanding events from all command list queues */ for (i = 0; i < self->num_cmd_lists; i++) { while (!ucs_queue_is_empty(&self->queue_desc[i].event_queue)) { From de5ff1d86015b289ccd780afd13086b41c3e20e2 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 09:02:02 -0700 Subject: [PATCH 21/35] TEST/UCT/ZE_IPC: add iface RMA and cache identity/purge coverage New test_ze_ipc_rma.cc opens a real ze_ipc iface pair and exercises put/get_zcopy plus the multi-iov/bounds rejections; this is what caught the missing ep_outstanding_purge and progress-teardown bugs fixed in prior commits. test_ze_ipc_cache.cc gains coverage for the stale-address remap and purge-by-context paths. Signed-off-by: Tushar Gohad --- test/gtest/Makefile.am | 1 + test/gtest/uct/ze/test_ze_ipc_cache.cc | 165 +++++++++++++++++++++++++ test/gtest/uct/ze/test_ze_ipc_rma.cc | 117 ++++++++++++++++++ 3 files changed, 283 insertions(+) create mode 100644 test/gtest/uct/ze/test_ze_ipc_rma.cc diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index 4c4a287793f..58b151112ce 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -355,6 +355,7 @@ gtest_SOURCES += \ uct/ze/test_ze_base.cc \ uct/ze/test_ze_ipc_md.cc \ uct/ze/test_ze_ipc_cache.cc \ + uct/ze/test_ze_ipc_rma.cc \ uct/ze/test_ze_copy_md.cc gtest_CPPFLAGS += \ $(ZE_CPPFLAGS) diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc index a774069203a..938bad66e51 100644 --- a/test/gtest/uct/ze/test_ze_ipc_cache.cc +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -6,8 +6,10 @@ #include extern "C" { +#include #include #include +#include } @@ -24,6 +26,50 @@ class test_ze_ipc_cache : public ucs::test { UCS_TEST_SKIP_R("Level Zero runtime not available"); } } + + static uct_md_h open_first_md() { + uct_component_attr_t attr; + attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT; + if (uct_component_query(&uct_ze_ipc_component, &attr) != UCS_OK) { + return NULL; + } + + unsigned count = attr.md_resource_count; + if (count == 0) { + return NULL; + } + + uct_md_resource_desc_t *res = (uct_md_resource_desc_t *) + ucs_calloc(count, sizeof(uct_md_resource_desc_t), + "ze_ipc_md_resources"); + if (res == NULL) { + return NULL; + } + + attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES; + attr.md_resources = res; + ucs_status_t status = uct_component_query(&uct_ze_ipc_component, &attr); + if (status != UCS_OK) { + ucs_free(res); + return NULL; + } + + uct_md_config_t *md_config = NULL; + status = uct_md_config_read(&uct_ze_ipc_component, NULL, NULL, + &md_config); + if (status != UCS_OK) { + ucs_free(res); + return NULL; + } + + uct_md_h md = NULL; + status = uct_md_open(&uct_ze_ipc_component, res[0].md_name, md_config, + &md); + uct_config_release(md_config); + ucs_free(res); + + return (status == UCS_OK) ? md : NULL; + } }; @@ -59,4 +105,123 @@ UCS_TEST_F(test_ze_ipc_cache, multiple_instances) { } +/* + * Simulates a peer freeing an allocation and a later, different allocation + * reusing the same VA: two distinct real IPC keys are forced to report the + * same base address. The cache must detect the IPC-handle mismatch on the + * second map and remap, rather than returning the first mapping. + */ +UCS_TEST_F(test_ze_ipc_cache, stale_address_is_remapped) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + + const size_t size = 4096; + void *ptr_a = NULL, *ptr_b = NULL; + ze_device_mem_alloc_desc_t dev_desc = {}; + dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, + ze_md->ze_device, &ptr_a)); + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, + ze_md->ze_device, &ptr_b)); + + uct_mem_h memh_a = NULL, memh_b = NULL; + ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_a, size, NULL, &memh_a)); + ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_b, size, NULL, &memh_b)); + + uct_ze_ipc_key_t key_a = {}, key_b = {}; + ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_a, ptr_a, size, NULL, &key_a)); + ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_b, ptr_b, size, NULL, &key_b)); + + /* simulate allocation B being reported at the VA allocation A used */ + key_b.address = key_a.address; + + void *mapped_a = NULL, *mapped_b = NULL; + int dup_fd_a = -1, dup_fd_b = -1; + + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_a, ze_md->ze_context, + ze_md->ze_device, &mapped_a, + &dup_fd_a)); + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_b, ze_md->ze_context, + ze_md->ze_device, &mapped_b, + &dup_fd_b)); + + EXPECT_NE(mapped_a, mapped_b) + << "cache returned a's stale mapping for a different allocation"; + + /* the second map_memhandle() already invalidated and freed a's region + * (both keys alias the same address), so only b's mapping is still + * live in the cache to unmap */ + uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, + ze_md->ze_context, dup_fd_b, 0); + + uct_md_mem_dereg_params_t dereg_params = {}; + dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; + dereg_params.memh = memh_a; + EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); + dereg_params.memh = memh_b; + EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); + + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_a)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_b)); + uct_md_close(md); +} + + +/* + * uct_ze_ipc_purge_cache_by_context() must drop cached regions for a + * closed context without leaving a dangling pgtable entry: mapping the + * same key again afterwards must go through a fresh open, not reuse a + * freed region. + */ +UCS_TEST_F(test_ze_ipc_cache, purge_by_context_then_remap) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + + const size_t size = 4096; + void *ptr = NULL; + ze_device_mem_alloc_desc_t dev_desc = {}; + dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, + ze_md->ze_device, &ptr)); + + uct_mem_h memh = NULL; + ASSERT_UCS_OK(md->ops->mem_reg(md, ptr, size, NULL, &memh)); + + uct_ze_ipc_key_t key = {}; + ASSERT_UCS_OK(md->ops->mkey_pack(md, memh, ptr, size, NULL, &key)); + void *mapped = NULL; + int dup_fd = -1; + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, + ze_md->ze_device, &mapped, + &dup_fd)); + + uct_ze_ipc_purge_cache_by_context(ze_md->ze_context); + + /* the purged region must not be referenced again; a fresh map/unmap + * must succeed cleanly */ + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, + ze_md->ze_device, &mapped, + &dup_fd)); + uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped, ze_md->ze_context, + dup_fd, 0); + + uct_md_mem_dereg_params_t dereg_params = {}; + dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; + dereg_params.memh = memh; + EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); + + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr)); + uct_md_close(md); +} diff --git a/test/gtest/uct/ze/test_ze_ipc_rma.cc b/test/gtest/uct/ze/test_ze_ipc_rma.cc new file mode 100644 index 00000000000..460313c7662 --- /dev/null +++ b/test/gtest/uct/ze/test_ze_ipc_rma.cc @@ -0,0 +1,117 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +} + + +/* + * Endpoint-level RMA tests for ze_ipc. Both entities live in this process, + * so the transport takes its same-pid loopback path (no pidfd_getfd fd + * duplication), but the iface open/connect/put_zcopy/get_zcopy/flush and + * cache map/unmap paths are exercised the same way they would be across + * processes. + */ +class test_ze_ipc_rma : public uct_test { +protected: + void init() + { + uct_test::init(); + + m_receiver = uct_test::create_entity(0); + m_entities.push_back(m_receiver); + + m_sender = uct_test::create_entity(0); + m_entities.push_back(m_sender); + + m_sender->connect(0, *m_receiver, 0); + } + + entity *m_sender; + entity *m_receiver; + + static const uint64_t SEED1 = 0xABClu; + static const uint64_t SEED2 = 0xDEFlu; +}; + + +UCS_TEST_P(test_ze_ipc_rma, put_zcopy) +{ + size_t length = 4096; + + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_put_zcopy(m_sender->ep(0), sendbuf.iov(), + 1, recvbuf.addr(), + recvbuf.rkey(), NULL)); + m_sender->flush(); + recvbuf.pattern_check(SEED1); +} + +UCS_TEST_P(test_ze_ipc_rma, get_zcopy) +{ + size_t length = 4096; + + mapped_buffer sendbuf(length, SEED1, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_get_zcopy(m_sender->ep(0), recvbuf.iov(), + 1, sendbuf.addr(), + sendbuf.rkey(), NULL)); + m_sender->flush(); + recvbuf.pattern_check(SEED1); +} + +UCS_TEST_P(test_ze_ipc_rma, put_zcopy_rejects_multi_iov) +{ + size_t length = 64; + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + uct_iov_t iov[2]; + + iov[0] = *sendbuf.iov(); + iov[1] = *sendbuf.iov(); + + ucs_status_t status; + { + scoped_log_handler slh(hide_errors_logger); + status = uct_ep_put_zcopy(m_sender->ep(0), iov, 2, recvbuf.addr(), + recvbuf.rkey(), NULL); + } + EXPECT_EQ(UCS_ERR_INVALID_PARAM, status); +} + +UCS_TEST_P(test_ze_ipc_rma, put_zcopy_rejects_out_of_range_rkey) +{ + size_t length = 64; + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ucs_status_t status; + { + /* the rkey covers the whole driver allocation, which is page-rounded + * and therefore larger than 'length' - go far enough past the base + * address to land outside it */ + scoped_log_handler slh(hide_errors_logger); + status = uct_ep_put_zcopy(m_sender->ep(0), sendbuf.iov(), 1, + recvbuf.addr() + UCS_MBYTE, recvbuf.rkey(), + NULL); + } + EXPECT_EQ(UCS_ERR_INVALID_PARAM, status); +} + +_UCT_INSTANTIATE_TEST_CASE(test_ze_ipc_rma, ze_ipc) From d1e2ec8ac17a114ed4b36c173717c70861ce9e32 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 09:04:26 -0700 Subject: [PATCH 22/35] UCT/ZE_IPC: reject MAX_CMD_LISTS=0, propagate completion errors MAX_CMD_LISTS=0 produced num_cmd_lists=0, causing a div-by-zero in the ep's round-robin cmd-list selection; reject it at iface init instead. Also stop treating any non-NOT_READY zeEventQueryStatus() result as success - propagate real ZE errors to the completion callback. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_iface.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index 7608f2c476e..974841d6cb6 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -262,6 +262,9 @@ uct_ze_ipc_iface_progress(uct_iface_h tl_iface) if (ret == ZE_RESULT_NOT_READY) { continue; } + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeEventQueryStatus failed with error 0x%x", ret); + } ucs_queue_del_iter(&q_desc->event_queue, iter); @@ -282,7 +285,9 @@ uct_ze_ipc_iface_progress(uct_iface_h tl_iface) /* Invoke completion callback */ if (event_desc->comp != NULL) { - uct_invoke_completion(event_desc->comp, UCS_OK); + uct_invoke_completion(event_desc->comp, + (ret == ZE_RESULT_SUCCESS) ? + UCS_OK : UCS_ERR_IO_ERROR); } /* Free event back to shared pool or destroy private pool */ @@ -533,6 +538,11 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, } /* Validate final count */ + if (self->num_cmd_lists == 0) { + ucs_error("ze_ipc_iface: MAX_CMD_LISTS=0 is invalid, at least 1 " + "command list is required"); + return UCS_ERR_INVALID_PARAM; + } if (self->num_cmd_lists > UCT_ZE_IPC_MAX_PEERS) { ucs_error("ze_ipc_iface: computed num_cmd_lists (%u) exceeds maximum (%u)", self->num_cmd_lists, UCT_ZE_IPC_MAX_PEERS); From 6c522ab0c981212404d061988a5a377a38c22176 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 09:04:27 -0700 Subject: [PATCH 23/35] UCT/ZE_IPC: resolve remote allocation's own device on map post_copy always mapped remote memory using the local iface's own device, ignoring the peer's packed dev_num. On multi-GPU nodes this could open a GPU-1 allocation against GPU-0. Resolve the device by the rkey's dev_num instead. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_ep.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c index d2d47739045..92f94b1d3bd 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_ep.c +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -178,6 +178,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, uct_ze_ipc_event_desc_t *event_desc; uct_ze_ipc_queue_desc_t *q_desc; ze_event_handle_t event = NULL; + ze_device_handle_t remote_device; void *mapped_addr = NULL; void *mapped_rem_addr; void *dst, *src; @@ -205,8 +206,18 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, return UCS_ERR_INVALID_PARAM; } + /* Resolve the local device matching the remote allocation's device + * ordinal - on multi-GPU nodes this may differ from the iface's own + * device */ + remote_device = uct_ze_base_get_device(key->dev_num); + if (remote_device == NULL) { + ucs_error("ze_ipc_ep: failed to get local device for ordinal %d", + key->dev_num); + return UCS_ERR_NO_DEVICE; + } + /* Use cache to map IPC handle */ - status = uct_ze_ipc_map_memhandle(key, iface->ze_context, iface->ze_device, + status = uct_ze_ipc_map_memhandle(key, iface->ze_context, remote_device, &mapped_addr, &local_fd); if (status != UCS_OK) { ucs_error("ze_ipc_ep: uct_ze_ipc_map_memhandle failed"); From e1a9025038bf13a59d4c464e44c4d8ac3e4d0430 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 09:21:53 -0700 Subject: [PATCH 24/35] UCT/ZE: fix codestyle findings from Azure format-code check Applies the exact reformatting Azure build 131309's "format code" job (git-clang-format) flagged for this PR, resolved by hand against my prior commits: pointer-cast spacing, brace placement, wrapped-line alignment. No functional change. Signed-off-by: Tushar Gohad --- src/uct/ze/base/ze_base.c | 2 +- src/uct/ze/copy/ze_copy_ep.c | 6 +- src/uct/ze/ze_ipc/ze_ipc_cache.c | 119 ++++++------ src/uct/ze/ze_ipc/ze_ipc_cache.h | 32 ++-- src/uct/ze/ze_ipc/ze_ipc_ep.c | 101 +++++----- src/uct/ze/ze_ipc/ze_ipc_ep.h | 20 +- src/uct/ze/ze_ipc/ze_ipc_iface.c | 254 +++++++++++++------------ src/uct/ze/ze_ipc/ze_ipc_iface.h | 65 ++++--- src/uct/ze/ze_ipc/ze_ipc_md.c | 105 +++++----- src/uct/ze/ze_ipc/ze_ipc_md.h | 12 +- test/gtest/uct/ze/test_ze_base.cc | 4 +- test/gtest/uct/ze/test_ze_copy_md.cc | 75 ++++---- test/gtest/uct/ze/test_ze_ipc_cache.cc | 9 +- test/gtest/uct/ze/test_ze_ipc_md.cc | 39 ++-- 14 files changed, 441 insertions(+), 402 deletions(-) diff --git a/src/uct/ze/base/ze_base.c b/src/uct/ze/base/ze_base.c index a7a21453980..c664447bb6d 100644 --- a/src/uct/ze/base/ze_base.c +++ b/src/uct/ze/base/ze_base.c @@ -388,7 +388,7 @@ int uct_ze_base_get_device_ordinal(ze_device_handle_t device) } } - return -1; /* not found */ + return -1; /* not found */ } ze_device_handle_t uct_ze_base_get_device(int ordinal) diff --git a/src/uct/ze/copy/ze_copy_ep.c b/src/uct/ze/copy/ze_copy_ep.c index c3e3e462235..f74cb6edd6e 100644 --- a/src/uct/ze/copy/ze_copy_ep.c +++ b/src/uct/ze/copy/ze_copy_ep.c @@ -51,7 +51,7 @@ ucs_status_t uct_ze_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr, void *src, *dst; ucs_trace("ze_copy_ep: zcopy %s remote_addr=%p local_buf=%p size=%zu", - is_put ? "PUT" : "GET", (void*)remote_addr, iov->buffer, size); + is_put ? "PUT" : "GET", (void*)remote_addr, iov->buffer, size); if (is_put) { src = iov->buffer; @@ -148,7 +148,7 @@ ucs_status_t uct_ze_copy_ep_put_short(uct_ep_h tl_ep, const void *buffer, ucs_status_t status; ucs_trace("ze_copy_ep: PUT_SHORT buffer=%p length=%u remote_addr=%p", - buffer, length, (void*)remote_addr); + buffer, length, (void*)remote_addr); status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 1); @@ -170,7 +170,7 @@ ucs_status_t uct_ze_copy_ep_get_short(uct_ep_h tl_ep, void *buffer, ucs_status_t status; ucs_trace("ze_copy_ep: GET_SHORT buffer=%p length=%u remote_addr=%p", - buffer, length, (void*)remote_addr); + buffer, length, (void*)remote_addr); status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 0); diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index 90062a59edd..72080e6be6e 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -4,7 +4,7 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif #include "ze_ipc_cache.h" @@ -22,7 +22,7 @@ typedef struct uct_ze_ipc_cache_hash_key { pid_t pid; - uint32_t _pad; /* Ensure 8-byte alignment for ze_context */ + uint32_t _pad; /* Ensure 8-byte alignment for ze_context */ ze_context_handle_t ze_context; } UCS_S_PACKED uct_ze_ipc_cache_hash_key_t; @@ -42,9 +42,8 @@ uct_ze_ipc_cache_hash_func(uct_ze_ipc_cache_hash_key_t key) } -KHASH_INIT(ze_ipc_rem_cache, uct_ze_ipc_cache_hash_key_t, - uct_ze_ipc_cache_t*, 1, uct_ze_ipc_cache_hash_func, - uct_ze_ipc_cache_hash_equal); +KHASH_INIT(ze_ipc_rem_cache, uct_ze_ipc_cache_hash_key_t, uct_ze_ipc_cache_t*, + 1, uct_ze_ipc_cache_hash_func, uct_ze_ipc_cache_hash_equal); typedef struct uct_ze_ipc_remote_cache { @@ -56,13 +55,14 @@ typedef struct uct_ze_ipc_remote_cache { static uct_ze_ipc_remote_cache_t uct_ze_ipc_remote_cache; -static ucs_pgt_dir_t *uct_ze_ipc_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) +static ucs_pgt_dir_t * +uct_ze_ipc_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) { void *ptr; int ret; ret = ucs_posix_memalign(&ptr, - ucs_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), + ucs_max(sizeof(void*), UCS_PGT_ENTRY_MIN_ALIGN), sizeof(ucs_pgt_dir_t), "ze_ipc_cache_pgdir"); return (ret == 0) ? ptr : NULL; } @@ -123,11 +123,10 @@ static void uct_ze_ipc_cache_purge(uct_ze_ipc_cache_t *cache) } -static ucs_status_t -uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, - ze_context_handle_t ze_context, - ze_device_handle_t ze_device, - void **mapped_addr, int *dup_fd) +static ucs_status_t uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + void **mapped_addr, int *dup_fd) { ze_ipc_mem_handle_t local_handle; int remote_fd; @@ -141,7 +140,8 @@ uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, if (key->pid != getpid() && remote_fd > 0 && remote_fd < 65536) { *dup_fd = uct_ze_ipc_dup_fd_from_pid(key->pid, remote_fd); if (*dup_fd < 0) { - ucs_error("failed to duplicate fd %d from pid %d", remote_fd, key->pid); + ucs_error("failed to duplicate fd %d from pid %d", remote_fd, + key->pid); return UCS_ERR_IO_ERROR; } /* Update handle with local fd */ @@ -150,8 +150,8 @@ uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, *dup_fd = -1; } - ret = zeMemOpenIpcHandle(ze_context, ze_device, local_handle, - 0, mapped_addr); + ret = zeMemOpenIpcHandle(ze_context, ze_device, local_handle, 0, + mapped_addr); if (ret != ZE_RESULT_SUCCESS) { ucs_error("zeMemOpenIpcHandle failed with error 0x%x", ret); if (*dup_fd >= 0) { @@ -181,25 +181,25 @@ static void uct_ze_ipc_cache_invalidate_regions(uct_ze_ipc_cache_t *cache, status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); if (status != UCS_OK) { ucs_error("failed to remove address:%p from cache (%s)", - (void *)region->key.address, ucs_status_string(status)); + (void*)region->key.address, ucs_status_string(status)); } status = uct_ze_ipc_close_memhandle(region); if (status != UCS_OK) { ucs_error("failed to close memhandle for base addr:%p (%s)", - (void *)region->key.address, ucs_status_string(status)); + (void*)region->key.address, ucs_status_string(status)); } ucs_free(region); } - ucs_trace("%s: closed memhandles in the range [%p..%p]", - cache->name, from, to); + ucs_trace("%s: closed memhandles in the range [%p..%p]", cache->name, from, + to); } -static ucs_status_t -uct_ze_ipc_get_remote_cache(pid_t pid, ze_context_handle_t ze_context, - uct_ze_ipc_cache_t **cache) +static ucs_status_t uct_ze_ipc_get_remote_cache(pid_t pid, + ze_context_handle_t ze_context, + uct_ze_ipc_cache_t **cache) { ucs_status_t status = UCS_OK; char target_name[64]; @@ -271,7 +271,7 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); if (status != UCS_OK) { ucs_error("failed to remove address:%p from cache (%s)", - (void *)region->key.address, ucs_status_string(status)); + (void*)region->key.address, ucs_status_string(status)); } ucs_assert(region->mapped_addr == mapped_addr); status = uct_ze_ipc_close_memhandle(region); @@ -285,10 +285,8 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, (key, ze_context, ze_device, mapped_addr, dup_fd), - uct_ze_ipc_key_t *key, - ze_context_handle_t ze_context, - ze_device_handle_t ze_device, - void **mapped_addr, int *dup_fd) + uct_ze_ipc_key_t *key, ze_context_handle_t ze_context, + ze_device_handle_t ze_device, void **mapped_addr, int *dup_fd) { uct_ze_ipc_cache_t *cache; ucs_status_t status; @@ -302,18 +300,19 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, } pthread_rwlock_wrlock(&cache->lock); - pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, - &cache->pgtable, key->address); + pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, &cache->pgtable, + key->address); if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); if (ucs_likely((region->key.length == key->length) && - !memcmp(®ion->key.ipc_handle, &key->ipc_handle, + !memcmp(®ion->key.ipc_handle, &key->ipc_handle, sizeof(key->ipc_handle)))) { /* cache hit */ - ucs_trace("%s: ze_ipc cache hit addr:%p size:%lu region:" - UCS_PGT_REGION_FMT, cache->name, (void *)key->address, - key->length, UCS_PGT_REGION_ARG(®ion->super)); + ucs_trace("%s: ze_ipc cache hit addr:%p size:%lu " + "region:" UCS_PGT_REGION_FMT, + cache->name, (void*)key->address, key->length, + UCS_PGT_REGION_ARG(®ion->super)); *mapped_addr = region->mapped_addr; *dup_fd = region->dup_fd; @@ -326,24 +325,25 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, /* remote allocation at this address was freed and a different one * reused the same VA - drop the stale mapping and remap below */ ucs_trace("%s: ze_ipc cache stale mapping addr:%p, invalidating", - cache->name, (void *)key->address); + cache->name, (void*)key->address); status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); if (status != UCS_OK) { ucs_error("failed to remove address:%p from cache (%s)", - (void *)region->key.address, ucs_status_string(status)); + (void*)region->key.address, ucs_status_string(status)); } uct_ze_ipc_close_memhandle(region); ucs_free(region); } - status = uct_ze_ipc_open_memhandle(key, ze_context, ze_device, mapped_addr, dup_fd); + status = uct_ze_ipc_open_memhandle(key, ze_context, ze_device, mapped_addr, + dup_fd); if (ucs_unlikely(status != UCS_OK)) { goto err; } /* create new cache entry */ - ret = ucs_posix_memalign((void **)®ion, - ucs_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), + ret = ucs_posix_memalign((void**)®ion, + ucs_max(sizeof(void*), UCS_PGT_ENTRY_MIN_ALIGN), sizeof(uct_ze_ipc_cache_region_t), "uct_ze_ipc_cache_region"); if (ret != 0) { @@ -353,34 +353,34 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, } region->super.start = ucs_align_down_pow2((uintptr_t)key->address, - UCS_PGT_ADDR_ALIGN); - region->super.end = ucs_align_up_pow2((uintptr_t)key->address + key->length, - UCS_PGT_ADDR_ALIGN); - region->key = *key; + UCS_PGT_ADDR_ALIGN); + region->super.end = ucs_align_up_pow2((uintptr_t)key->address + key->length, + UCS_PGT_ADDR_ALIGN); + region->key = *key; region->mapped_addr = *mapped_addr; region->refcount = 1; region->ze_context = ze_context; region->dup_fd = *dup_fd; - status = UCS_PROFILE_CALL(ucs_pgtable_insert, - &cache->pgtable, ®ion->super); + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); if (status == UCS_ERR_ALREADY_EXISTS) { /* overlapped region means memory freed at source. remove and try insert */ - uct_ze_ipc_cache_invalidate_regions(cache, - (void *)region->super.start, - (void *)region->super.end); - status = UCS_PROFILE_CALL(ucs_pgtable_insert, - &cache->pgtable, ®ion->super); + uct_ze_ipc_cache_invalidate_regions(cache, (void*)region->super.start, + (void*)region->super.end); + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); } if (status != UCS_OK) { - ucs_error("%s: failed to insert region:"UCS_PGT_REGION_FMT" size:%lu :%s", + ucs_error("%s: failed to insert region:" UCS_PGT_REGION_FMT + " size:%lu :%s", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length, ucs_status_string(status)); ucs_free(region); goto err; } - ucs_trace("%s: ze_ipc cache new region:"UCS_PGT_REGION_FMT" size:%lu", + ucs_trace("%s: ze_ipc cache new region:" UCS_PGT_REGION_FMT " size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length); status = UCS_OK; @@ -391,8 +391,8 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, } -ucs_status_t uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, - const char *name) +ucs_status_t +uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name) { ucs_status_t status; uct_ze_ipc_cache_t *cache_desc; @@ -467,18 +467,19 @@ void uct_ze_ipc_purge_cache_by_context(ze_context_handle_t ze_context) } -UCS_STATIC_INIT { +UCS_STATIC_INIT +{ ucs_recursive_spinlock_init(&uct_ze_ipc_remote_cache.lock, 0); kh_init_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); } -UCS_STATIC_CLEANUP { +UCS_STATIC_CLEANUP +{ uct_ze_ipc_cache_t *rem_cache; - kh_foreach_value(&uct_ze_ipc_remote_cache.hash, rem_cache, { - uct_ze_ipc_destroy_cache(rem_cache); - }) - kh_destroy_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); + kh_foreach_value(&uct_ze_ipc_remote_cache.hash, rem_cache, + { uct_ze_ipc_destroy_cache(rem_cache); }) + kh_destroy_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); ucs_recursive_spinlock_destroy(&uct_ze_ipc_remote_cache.lock); } diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.h b/src/uct/ze/ze_ipc/ze_ipc_cache.h index c1e80b8380b..a2367cda9bf 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.h +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.h @@ -13,22 +13,22 @@ #include "ze_ipc_md.h" -typedef struct uct_ze_ipc_cache uct_ze_ipc_cache_t; +typedef struct uct_ze_ipc_cache uct_ze_ipc_cache_t; typedef struct uct_ze_ipc_cache_region uct_ze_ipc_cache_region_t; -typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; +typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; /** * Cache region structure for storing mapped IPC handles */ struct uct_ze_ipc_cache_region { - ucs_pgt_region_t super; /**< Base class - page table region */ - ucs_list_link_t list; /**< List element */ - uct_ze_ipc_key_t key; /**< Remote memory key */ - void *mapped_addr; /**< Local mapped address */ - uint64_t refcount; /**< Track in-flight ops before unmapping*/ - ze_context_handle_t ze_context; /**< Level Zero context */ - int dup_fd; /**< Duplicated file descriptor */ + ucs_pgt_region_t super; /**< Base class - page table region */ + ucs_list_link_t list; /**< List element */ + uct_ze_ipc_key_t key; /**< Remote memory key */ + void *mapped_addr; /**< Local mapped address */ + uint64_t refcount; /**< Track in-flight ops before unmapping*/ + ze_context_handle_t ze_context; /**< Level Zero context */ + int dup_fd; /**< Duplicated file descriptor */ }; @@ -36,9 +36,9 @@ struct uct_ze_ipc_cache_region { * Cache structure for managing IPC handle mappings */ struct uct_ze_ipc_cache { - pthread_rwlock_t lock; /**< Protects the page table */ - ucs_pgtable_t pgtable; /**< Page table to hold the regions */ - char *name; /**< Name for debugging */ + pthread_rwlock_t lock; /**< Protects the page table */ + ucs_pgtable_t pgtable; /**< Page table to hold the regions */ + char *name; /**< Name for debugging */ }; @@ -49,8 +49,8 @@ struct uct_ze_ipc_cache { * @param name Name for the cache (for debugging) * @return UCS_OK on success, error code otherwise */ -ucs_status_t uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, - const char *name); +ucs_status_t +uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name); /** @@ -74,8 +74,7 @@ void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache); ucs_status_t uct_ze_ipc_map_memhandle(uct_ze_ipc_key_t *key, ze_context_handle_t ze_context, ze_device_handle_t ze_device, - void **mapped_addr, - int *dup_fd); + void **mapped_addr, int *dup_fd); /** @@ -95,6 +94,7 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, int dup_fd, int cache_enabled); + /** * Purge and destroy all remote caches keyed by the given local Level Zero * context, so that closed contexts do not leave behind cache entries that diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c index 92f94b1d3bd..71b5388a647 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_ep.c +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -4,7 +4,7 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif #include "ze_ipc_ep.h" @@ -95,12 +95,13 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_ep_t) UCS_CLASS_DEFINE(uct_ze_ipc_ep_t, uct_base_ep_t) -UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t *); +UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t*); UCS_CLASS_DEFINE_DELETE_FUNC(uct_ze_ipc_ep_t, uct_ep_t); -#define uct_ze_ipc_trace_data(_addr, _rkey, _fmt, ...) \ - ucs_trace_data(_fmt " to %"PRIx64"(%+ld)", ## __VA_ARGS__, (_addr), (_rkey)) +#define uct_ze_ipc_trace_data(_addr, _rkey, _fmt, ...) \ + ucs_trace_data(_fmt " to %" PRIx64 "(%+ld)", ##__VA_ARGS__, (_addr), \ + (_rkey)) /* Timeline profiling for performance analysis */ @@ -112,36 +113,44 @@ typedef struct uct_ze_ipc_timeline { struct timespec end; } uct_ze_ipc_timeline_t; -static UCS_F_ALWAYS_INLINE void -uct_ze_ipc_timeline_record(struct timespec *ts) +static UCS_F_ALWAYS_INLINE void uct_ze_ipc_timeline_record(struct timespec *ts) { clock_gettime(CLOCK_MONOTONIC, ts); } static UCS_F_ALWAYS_INLINE void -uct_ze_ipc_timeline_report(const uct_ze_ipc_timeline_t *timeline, - size_t length, unsigned cmd_list_idx) +uct_ze_ipc_timeline_report(const uct_ze_ipc_timeline_t *timeline, size_t length, + unsigned cmd_list_idx) { double ipc_time, event_time, copy_time, other_time, total_time; - ipc_time = (timeline->get_ipc_handle.tv_sec - timeline->start.tv_sec) * 1000.0 + - (timeline->get_ipc_handle.tv_nsec - timeline->start.tv_nsec) / 1000000.0; - event_time = (timeline->event_alloc.tv_sec - timeline->get_ipc_handle.tv_sec) * 1000.0 + - (timeline->event_alloc.tv_nsec - timeline->get_ipc_handle.tv_nsec) / 1000000.0; - copy_time = (timeline->async_copy.tv_sec - timeline->event_alloc.tv_sec) * 1000.0 + - (timeline->async_copy.tv_nsec - timeline->event_alloc.tv_nsec) / 1000000.0; + ipc_time = (timeline->get_ipc_handle.tv_sec - timeline->start.tv_sec) * + 1000.0 + + (timeline->get_ipc_handle.tv_nsec - timeline->start.tv_nsec) / + 1000000.0; + event_time = + (timeline->event_alloc.tv_sec - timeline->get_ipc_handle.tv_sec) * + 1000.0 + + (timeline->event_alloc.tv_nsec - timeline->get_ipc_handle.tv_nsec) / + 1000000.0; + copy_time = (timeline->async_copy.tv_sec - timeline->event_alloc.tv_sec) * + 1000.0 + + (timeline->async_copy.tv_nsec - timeline->event_alloc.tv_nsec) / + 1000000.0; other_time = (timeline->end.tv_sec - timeline->async_copy.tv_sec) * 1000.0 + - (timeline->end.tv_nsec - timeline->async_copy.tv_nsec) / 1000000.0; + (timeline->end.tv_nsec - timeline->async_copy.tv_nsec) / + 1000000.0; total_time = (timeline->end.tv_sec - timeline->start.tv_sec) * 1000.0 + (timeline->end.tv_nsec - timeline->start.tv_nsec) / 1000000.0; - ucs_trace("ze_ipc timeline: total=%.3fms (ipc=%.3fms event=%.3fms copy=%.3fms other=%.3fms) " + ucs_trace("ze_ipc timeline: total=%.3fms (ipc=%.3fms event=%.3fms " + "copy=%.3fms other=%.3fms) " "size=%zu cmd_list=%u", - total_time, ipc_time, event_time, copy_time, other_time, length, cmd_list_idx); + total_time, ipc_time, event_time, copy_time, other_time, length, + cmd_list_idx); } -#define UCT_ZE_IPC_TIMELINE_INIT(_tl) \ - uct_ze_ipc_timeline_t _tl = {{0}} +#define UCT_ZE_IPC_TIMELINE_INIT(_tl) uct_ze_ipc_timeline_t _tl = {{0}} #define UCT_ZE_IPC_TIMELINE_RECORD(_tl, _field) \ if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE)) { \ @@ -168,13 +177,13 @@ int uct_ze_ipc_ep_is_connected(const uct_ep_h tl_ep, static UCS_F_ALWAYS_INLINE ucs_status_t -uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, - const uct_iov_t *iov, uct_rkey_t rkey, - uct_completion_t *comp, int direction) +uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, + uct_rkey_t rkey, uct_completion_t *comp, int direction) { - uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ze_ipc_iface_t); + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_ep->iface, + uct_ze_ipc_iface_t); uct_ze_ipc_ep_t *ep = ucs_derived_of(tl_ep, uct_ze_ipc_ep_t); - uct_ze_ipc_key_t *key = (uct_ze_ipc_key_t *)rkey; + uct_ze_ipc_key_t *key = (uct_ze_ipc_key_t*)rkey; uct_ze_ipc_event_desc_t *event_desc; uct_ze_ipc_queue_desc_t *q_desc; ze_event_handle_t event = NULL; @@ -198,11 +207,12 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, } if (ucs_unlikely((remote_addr < key->address) || - ((offset = remote_addr - key->address) > key->length) || - (iov[0].length > (key->length - offset)))) { - ucs_error("ze_ipc_ep: remote_addr 0x%"PRIx64" length %zu is out of " - "rkey range [0x%lx..0x%lx)", remote_addr, iov[0].length, - key->address, key->address + key->length); + ((offset = remote_addr - key->address) > key->length) || + (iov[0].length > (key->length - offset)))) { + ucs_error("ze_ipc_ep: remote_addr 0x%" PRIx64 " length %zu is out of " + "rkey range [0x%lx..0x%lx)", + remote_addr, iov[0].length, key->address, + key->address + key->length); return UCS_ERR_INVALID_PARAM; } @@ -225,9 +235,10 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, } UCT_ZE_IPC_TIMELINE_RECORD(timeline, get_ipc_handle); - ucs_debug("ze_ipc_ep: IPC handle mapped (cached), mapped_addr=%p", mapped_addr); + ucs_debug("ze_ipc_ep: IPC handle mapped (cached), mapped_addr=%p", + mapped_addr); - mapped_rem_addr = (void *)((uintptr_t)mapped_addr + offset); + mapped_rem_addr = (void*)((uintptr_t)mapped_addr + offset); /* Allocate event from shared pool for performance */ event_index = uct_ze_ipc_alloc_event(iface, &event); @@ -252,7 +263,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, /* Store information for cache-based cleanup and event tracking */ event_desc->event = event; - event_desc->event_pool = NULL; /* Using shared pool, not private */ + event_desc->event_pool = NULL; /* Using shared pool, not private */ event_desc->event_index = event_index; event_desc->dup_fd = local_fd; event_desc->pid = ep->remote_pid; @@ -312,8 +323,10 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, UCT_ZE_IPC_TIMELINE_RECORD(timeline, end); - ucs_trace("zeCommandListAppendMemoryCopy issued (async): cmd_list[%u/%u]=%p dst=%p src=%p len=%zu", - cmd_list_idx, iface->num_cmd_lists, q_desc->cmd_list, dst, src, iov[0].length); + ucs_trace("zeCommandListAppendMemoryCopy issued (async): " + "cmd_list[%u/%u]=%p dst=%p src=%p len=%zu", + cmd_list_idx, iface->num_cmd_lists, q_desc->cmd_list, dst, src, + iov[0].length); UCT_ZE_IPC_TIMELINE_REPORT(timeline, iov[0].length, cmd_list_idx); return UCS_INPROGRESS; @@ -328,10 +341,9 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, } UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_get_zcopy, - (tl_ep, iov, iovcnt, remote_addr, rkey, comp), - uct_ep_h tl_ep, const uct_iov_t *iov, size_t iovcnt, - uint64_t remote_addr, uct_rkey_t rkey, - uct_completion_t *comp) + (tl_ep, iov, iovcnt, remote_addr, rkey, comp), uct_ep_h tl_ep, + const uct_iov_t *iov, size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp) { ucs_status_t status; @@ -340,7 +352,8 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_get_zcopy, status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, UCT_ZE_IPC_GET); if (UCS_STATUS_IS_ERR(status)) { - ucs_error("ze_ipc_ep: GET_ZCOPY failed with status %s", ucs_status_string(status)); + ucs_error("ze_ipc_ep: GET_ZCOPY failed with status %s", + ucs_status_string(status)); return status; } @@ -353,10 +366,9 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_get_zcopy, UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_put_zcopy, - (tl_ep, iov, iovcnt, remote_addr, rkey, comp), - uct_ep_h tl_ep, const uct_iov_t *iov, size_t iovcnt, - uint64_t remote_addr, uct_rkey_t rkey, - uct_completion_t *comp) + (tl_ep, iov, iovcnt, remote_addr, rkey, comp), uct_ep_h tl_ep, + const uct_iov_t *iov, size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp) { ucs_status_t status; @@ -365,7 +377,8 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_put_zcopy, status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, UCT_ZE_IPC_PUT); if (UCS_STATUS_IS_ERR(status)) { - ucs_error("ze_ipc_ep: PUT_ZCOPY failed with status %s", ucs_status_string(status)); + ucs_error("ze_ipc_ep: PUT_ZCOPY failed with status %s", + ucs_status_string(status)); return status; } diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.h b/src/uct/ze/ze_ipc/ze_ipc_ep.h index f65ab68d603..a737dd63e09 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_ep.h +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.h @@ -17,23 +17,21 @@ typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; typedef struct uct_ze_ipc_ep { - uct_base_ep_t super; - pid_t remote_pid; + uct_base_ep_t super; + pid_t remote_pid; } uct_ze_ipc_ep_t; -UCS_CLASS_DECLARE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t *); +UCS_CLASS_DECLARE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t*); UCS_CLASS_DECLARE_DELETE_FUNC(uct_ze_ipc_ep_t, uct_ep_t); -ucs_status_t uct_ze_ipc_ep_get_zcopy(uct_ep_h tl_ep, - const uct_iov_t *iov, size_t iovcnt, - uint64_t remote_addr, uct_rkey_t rkey, - uct_completion_t *comp); +ucs_status_t uct_ze_ipc_ep_get_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, + size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp); -ucs_status_t uct_ze_ipc_ep_put_zcopy(uct_ep_h tl_ep, - const uct_iov_t *iov, size_t iovcnt, - uint64_t remote_addr, uct_rkey_t rkey, - uct_completion_t *comp); +ucs_status_t uct_ze_ipc_ep_put_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, + size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp); int uct_ze_ipc_ep_is_connected(const uct_ep_h tl_ep, const uct_ep_is_connected_params_t *params); diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index 974841d6cb6..51a53e6cf29 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -4,7 +4,7 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif #include "ze_ipc_iface.h" @@ -21,39 +21,31 @@ static ucs_config_field_t uct_ze_ipc_iface_config_table[] = { - {"", "", NULL, - ucs_offsetof(uct_ze_ipc_iface_config_t, super), + {"", "", NULL, ucs_offsetof(uct_ze_ipc_iface_config_t, super), UCS_CONFIG_TYPE_TABLE(uct_iface_config_table)}, {"MAX_POLL", "16", "Max number of event completions to pick during ze events polling", - ucs_offsetof(uct_ze_ipc_iface_config_t, max_poll), - UCS_CONFIG_TYPE_UINT}, + ucs_offsetof(uct_ze_ipc_iface_config_t, max_poll), UCS_CONFIG_TYPE_UINT}, {"MAX_CMD_LISTS", UCS_PP_MAKE_STRING(UCT_ZE_IPC_MAX_PEERS), - "Max number of command lists (upper limit, actual count matches copy engine queues)", + "Max number of command lists (upper limit, actual count matches copy " + "engine queues)", ucs_offsetof(uct_ze_ipc_iface_config_t, max_cmd_lists), UCS_CONFIG_TYPE_UINT}, - {"ENABLE_CACHE", "yes", - "Enable IPC handle caching to improve performance", + {"ENABLE_CACHE", "yes", "Enable IPC handle caching to improve performance", ucs_offsetof(uct_ze_ipc_iface_config_t, enable_cache), UCS_CONFIG_TYPE_BOOL}, - {"BW", "50000MBs", - "Effective p2p memory bandwidth", - ucs_offsetof(uct_ze_ipc_iface_config_t, bandwidth), - UCS_CONFIG_TYPE_BW}, + {"BW", "50000MBs", "Effective p2p memory bandwidth", + ucs_offsetof(uct_ze_ipc_iface_config_t, bandwidth), UCS_CONFIG_TYPE_BW}, - {"LAT", "1.8us", - "Estimated latency", - ucs_offsetof(uct_ze_ipc_iface_config_t, latency), - UCS_CONFIG_TYPE_TIME}, + {"LAT", "1.8us", "Estimated latency", + ucs_offsetof(uct_ze_ipc_iface_config_t, latency), UCS_CONFIG_TYPE_TIME}, - {"OVERHEAD", "4.0us", - "Estimated CPU overhead for transferring GPU memory", - ucs_offsetof(uct_ze_ipc_iface_config_t, overhead), - UCS_CONFIG_TYPE_TIME}, + {"OVERHEAD", "4.0us", "Estimated CPU overhead for transferring GPU memory", + ucs_offsetof(uct_ze_ipc_iface_config_t, overhead), UCS_CONFIG_TYPE_TIME}, {NULL} }; @@ -63,9 +55,8 @@ static ucs_config_field_t uct_ze_ipc_iface_config_table[] = { static void UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t)(uct_iface_t*); -static ucs_status_t -uct_ze_ipc_iface_get_device_address(uct_iface_t *tl_iface, - uct_device_addr_t *addr) +static ucs_status_t uct_ze_ipc_iface_get_device_address(uct_iface_t *tl_iface, + uct_device_addr_t *addr) { *(uint64_t*)addr = ucs_get_system_id(); return UCS_OK; @@ -91,7 +82,7 @@ uct_ze_ipc_iface_is_reachable_v2(const uct_iface_h tl_iface, return 0; } - dev_addr = (uint64_t *)params->device_addr; + dev_addr = (uint64_t*)params->device_addr; remote_pid = *(pid_t*)params->iface_addr; /* Reject same process */ @@ -116,18 +107,17 @@ uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) uct_base_iface_query(&iface->super, iface_attr); - iface_attr->iface_addr_len = sizeof(pid_t); - iface_attr->device_addr_len = sizeof(uint64_t); - iface_attr->ep_addr_len = 0; - iface_attr->max_conn_priv = 0; - iface_attr->cap.flags = UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE | - UCT_IFACE_FLAG_CONNECT_TO_IFACE | - UCT_IFACE_FLAG_PENDING | - UCT_IFACE_FLAG_GET_ZCOPY | - UCT_IFACE_FLAG_PUT_ZCOPY; + iface_attr->iface_addr_len = sizeof(pid_t); + iface_attr->device_addr_len = sizeof(uint64_t); + iface_attr->ep_addr_len = 0; + iface_attr->max_conn_priv = 0; + iface_attr->cap.flags = UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE | + UCT_IFACE_FLAG_CONNECT_TO_IFACE | + UCT_IFACE_FLAG_PENDING | UCT_IFACE_FLAG_GET_ZCOPY | + UCT_IFACE_FLAG_PUT_ZCOPY; - iface_attr->cap.event_flags = UCT_IFACE_FLAG_EVENT_SEND_COMP | - UCT_IFACE_FLAG_EVENT_RECV; + iface_attr->cap.event_flags = UCT_IFACE_FLAG_EVENT_SEND_COMP | + UCT_IFACE_FLAG_EVENT_RECV; iface_attr->cap.put.max_short = 0; iface_attr->cap.put.max_bcopy = 0; @@ -144,11 +134,11 @@ uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) iface_attr->cap.get.align_mtu = iface_attr->cap.get.opt_zcopy_align; iface_attr->cap.get.max_iov = 1; - iface_attr->latency = ucs_linear_func_make(1e-6, 0); - iface_attr->bandwidth.dedicated = 0; - iface_attr->bandwidth.shared = iface->config.bandwidth; - iface_attr->overhead = 7.0e-6; - iface_attr->priority = 0; + iface_attr->latency = ucs_linear_func_make(1e-6, 0); + iface_attr->bandwidth.dedicated = 0; + iface_attr->bandwidth.shared = iface->config.bandwidth; + iface_attr->overhead = 7.0e-6; + iface_attr->priority = 0; return UCS_OK; } @@ -158,7 +148,8 @@ uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) * Allocate an event from the shared event pool * Returns event index on success, -1 on failure */ -int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t *event_p) +int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, + ze_event_handle_t *event_p) { unsigned i, word_idx, bit_idx; uint64_t mask; @@ -207,8 +198,7 @@ int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t *event_p /** * Free an event back to the shared event pool */ -void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, - ze_event_handle_t event, +void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t event, unsigned event_index) { unsigned word_idx, bit_idx; @@ -229,14 +219,13 @@ void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, } } -static unsigned -uct_ze_ipc_iface_progress(uct_iface_h tl_iface) +static unsigned uct_ze_ipc_iface_progress(uct_iface_h tl_iface) { uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); uct_ze_ipc_event_desc_t *event_desc; uct_ze_ipc_queue_desc_t *q_desc; ucs_queue_iter_t iter; - unsigned count = 0; + unsigned count = 0; unsigned max_poll = iface->config.max_poll; ze_result_t ret; @@ -287,13 +276,15 @@ uct_ze_ipc_iface_progress(uct_iface_h tl_iface) if (event_desc->comp != NULL) { uct_invoke_completion(event_desc->comp, (ret == ZE_RESULT_SUCCESS) ? - UCS_OK : UCS_ERR_IO_ERROR); + UCS_OK : + UCS_ERR_IO_ERROR); } /* Free event back to shared pool or destroy private pool */ if (event_desc->event_index != (unsigned)-1) { /* Using shared event pool */ - uct_ze_ipc_free_event(iface, event_desc->event, event_desc->event_index); + uct_ze_ipc_free_event(iface, event_desc->event, + event_desc->event_index); } else { /* Using private event pool (backward compatibility) */ if (event_desc->event != NULL) { @@ -318,9 +309,8 @@ uct_ze_ipc_iface_progress(uct_iface_h tl_iface) } -static ucs_status_t -uct_ze_ipc_iface_flush(uct_iface_h tl_iface, unsigned flags, - uct_completion_t *comp) +static ucs_status_t uct_ze_ipc_iface_flush(uct_iface_h tl_iface, unsigned flags, + uct_completion_t *comp) { uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); unsigned i; @@ -328,7 +318,8 @@ uct_ze_ipc_iface_flush(uct_iface_h tl_iface, unsigned flags, /* Check if all command list queues are empty */ for (i = 0; i < iface->num_cmd_lists; i++) { if (!ucs_queue_is_empty(&iface->queue_desc[i].event_queue)) { - UCT_TL_IFACE_STAT_FLUSH_WAIT(ucs_derived_of(tl_iface, uct_base_iface_t)); + UCT_TL_IFACE_STAT_FLUSH_WAIT( + ucs_derived_of(tl_iface, uct_base_iface_t)); return UCS_INPROGRESS; } } @@ -339,22 +330,24 @@ uct_ze_ipc_iface_flush(uct_iface_h tl_iface, unsigned flags, static uct_iface_ops_t uct_ze_ipc_iface_ops = { - .ep_get_zcopy = uct_ze_ipc_ep_get_zcopy, - .ep_put_zcopy = uct_ze_ipc_ep_put_zcopy, - .ep_pending_add = (uct_ep_pending_add_func_t)ucs_empty_function_return_busy, - .ep_pending_purge = (uct_ep_pending_purge_func_t)ucs_empty_function, - .ep_flush = uct_base_ep_flush, - .ep_fence = uct_base_ep_fence, - .ep_check = (uct_ep_check_func_t)ucs_empty_function_return_unsupported, - .ep_create = UCS_CLASS_NEW_FUNC_NAME(uct_ze_ipc_ep_t), - .ep_destroy = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_ep_t), - .iface_flush = uct_ze_ipc_iface_flush, - .iface_fence = uct_base_iface_fence, + .ep_get_zcopy = uct_ze_ipc_ep_get_zcopy, + .ep_put_zcopy = uct_ze_ipc_ep_put_zcopy, + .ep_pending_add = (uct_ep_pending_add_func_t)ucs_empty_function_return_busy, + .ep_pending_purge = (uct_ep_pending_purge_func_t)ucs_empty_function, + .ep_flush = uct_base_ep_flush, + .ep_fence = uct_base_ep_fence, + .ep_check = (uct_ep_check_func_t)ucs_empty_function_return_unsupported, + .ep_create = UCS_CLASS_NEW_FUNC_NAME(uct_ze_ipc_ep_t), + .ep_destroy = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_ep_t), + .iface_flush = uct_ze_ipc_iface_flush, + .iface_fence = uct_base_iface_fence, .iface_progress_enable = uct_base_iface_progress_enable, .iface_progress_disable = uct_base_iface_progress_disable, .iface_progress = uct_ze_ipc_iface_progress, - .iface_event_fd_get = (uct_iface_event_fd_get_func_t)ucs_empty_function_return_unsupported, - .iface_event_arm = (uct_iface_event_arm_func_t)ucs_empty_function_return_unsupported, + .iface_event_fd_get = (uct_iface_event_fd_get_func_t) + ucs_empty_function_return_unsupported, + .iface_event_arm = (uct_iface_event_arm_func_t) + ucs_empty_function_return_unsupported, .iface_close = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t), .iface_query = uct_ze_ipc_iface_query, .iface_get_device_address = uct_ze_ipc_iface_get_device_address, @@ -406,22 +399,25 @@ uct_ze_ipc_estimate_perf(uct_iface_h tl_iface, uct_perf_attr_t *perf_attr) } static uct_iface_internal_ops_t uct_ze_ipc_iface_internal_ops = { - .iface_query_v2 = uct_iface_base_query_v2, - .iface_estimate_perf = uct_ze_ipc_estimate_perf, - .iface_vfs_refresh = (uct_iface_vfs_refresh_func_t)ucs_empty_function, - .ep_query = (uct_ep_query_func_t)ucs_empty_function_return_unsupported, - .ep_invalidate = (uct_ep_invalidate_func_t)ucs_empty_function_return_unsupported, - .ep_connect_to_ep_v2 = (uct_ep_connect_to_ep_v2_func_t)ucs_empty_function_return_unsupported, - .iface_is_reachable_v2 = uct_ze_ipc_iface_is_reachable_v2, - .ep_is_connected = uct_ze_ipc_ep_is_connected, - .ep_outstanding_purge = (uct_ep_outstanding_purge_func_t)ucs_empty_function_return_unsupported + .iface_query_v2 = uct_iface_base_query_v2, + .iface_estimate_perf = uct_ze_ipc_estimate_perf, + .iface_vfs_refresh = (uct_iface_vfs_refresh_func_t)ucs_empty_function, + .ep_query = (uct_ep_query_func_t)ucs_empty_function_return_unsupported, + .ep_invalidate = (uct_ep_invalidate_func_t) + ucs_empty_function_return_unsupported, + .ep_connect_to_ep_v2 = (uct_ep_connect_to_ep_v2_func_t) + ucs_empty_function_return_unsupported, + .iface_is_reachable_v2 = uct_ze_ipc_iface_is_reachable_v2, + .ep_is_connected = uct_ze_ipc_ep_is_connected, + .ep_outstanding_purge = (uct_ep_outstanding_purge_func_t) + ucs_empty_function_return_unsupported }; /* Find the copy engine queue group ordinal for a device */ -static ucs_status_t -uct_ze_ipc_find_copy_ordinal(ze_device_handle_t device, uint32_t *ordinal_p, - uint32_t *num_queues_p) +static ucs_status_t uct_ze_ipc_find_copy_ordinal(ze_device_handle_t device, + uint32_t *ordinal_p, + uint32_t *num_queues_p) { ze_command_queue_group_properties_t queue_props[16]; uint32_t num_queue_groups = 16; @@ -436,20 +432,24 @@ uct_ze_ipc_find_copy_ordinal(ze_device_handle_t device, uint32_t *ordinal_p, /* Get queue group properties */ ret = zeDeviceGetCommandQueueGroupProperties(device, &num_queue_groups, - queue_props); + queue_props); if (ret != ZE_RESULT_SUCCESS) { - ucs_error("ze_ipc_iface: zeDeviceGetCommandQueueGroupProperties failed: 0x%x", + ucs_error("ze_ipc_iface: zeDeviceGetCommandQueueGroupProperties " + "failed: 0x%x", ret); return UCS_ERR_IO_ERROR; } /* First pass: find dedicated copy engine (COPY flag but NOT COMPUTE) */ for (i = 0; i < num_queue_groups; i++) { - if ((queue_props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) && - !(queue_props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE)) { - *ordinal_p = i; + if ((queue_props[i].flags & + ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) && + !(queue_props[i].flags & + ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE)) { + *ordinal_p = i; *num_queues_p = queue_props[i].numQueues; - ucs_debug("ze_ipc_iface: using dedicated copy engine queue group %u (numQueues=%u)", + ucs_debug("ze_ipc_iface: using dedicated copy engine queue group " + "%u (numQueues=%u)", i, queue_props[i].numQueues); return UCS_OK; } @@ -458,9 +458,10 @@ uct_ze_ipc_find_copy_ordinal(ze_device_handle_t device, uint32_t *ordinal_p, /* Second pass: find any queue group that supports COPY */ for (i = 0; i < num_queue_groups; i++) { if (queue_props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) { - *ordinal_p = i; + *ordinal_p = i; *num_queues_p = queue_props[i].numQueues; - ucs_debug("ze_ipc_iface: using copy-capable queue group %u (numQueues=%u)", + ucs_debug("ze_ipc_iface: using copy-capable queue group %u " + "(numQueues=%u)", i, queue_props[i].numQueues); return UCS_OK; } @@ -479,7 +480,7 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, ze_command_queue_desc_t queue_desc = {}; ze_event_pool_desc_t event_pool_desc; uint32_t copy_ordinal = 0; - uint32_t num_queues = 0; + uint32_t num_queues = 0; ucs_status_t status; ze_result_t ret; unsigned i; @@ -490,17 +491,18 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_ze_ipc_iface_ops, &uct_ze_ipc_iface_internal_ops, md, worker, - params, tl_config - UCS_STATS_ARG(params->stats_root) - UCS_STATS_ARG(UCT_ZE_IPC_TL_NAME)); + params, + tl_config UCS_STATS_ARG(params->stats_root) + UCS_STATS_ARG(UCT_ZE_IPC_TL_NAME)); - self->ze_context = ze_md->ze_context; - self->ze_device = ze_md->ze_device; - self->config = *config; - self->next_cmd_list = 0; + self->ze_context = ze_md->ze_context; + self->ze_device = ze_md->ze_device; + self->config = *config; + self->next_cmd_list = 0; /* Find copy engine queue group ordinal and available queue count */ - status = uct_ze_ipc_find_copy_ordinal(self->ze_device, ©_ordinal, &num_queues); + status = uct_ze_ipc_find_copy_ordinal(self->ze_device, ©_ordinal, + &num_queues); if (status != UCS_OK) { return status; } @@ -521,19 +523,24 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, if (num_queues == 0) { /* No copy engines available, use single command list on compute engine */ self->num_cmd_lists = 1; - ucs_info("ze_ipc_iface: no dedicated copy engines found, using 1 command list " - "on queue group %u (compute engine fallback)", copy_ordinal); + ucs_info("ze_ipc_iface: no dedicated copy engines found, using 1 " + "command list " + "on queue group %u (compute engine fallback)", + copy_ordinal); } else { /* Match command list count to available Copy Engine queues */ self->num_cmd_lists = ucs_min(num_queues, config->max_cmd_lists); if (self->num_cmd_lists < num_queues) { - ucs_info("ze_ipc_iface: limiting command lists to %u (hardware has %u copy queues, " - "config limit is %u)", self->num_cmd_lists, num_queues, - config->max_cmd_lists); + ucs_info("ze_ipc_iface: limiting command lists to %u (hardware has " + "%u copy queues, " + "config limit is %u)", + self->num_cmd_lists, num_queues, config->max_cmd_lists); } else { - ucs_info("ze_ipc_iface: creating %u command list(s) to match %u copy engine queue(s) " - "on queue group %u", self->num_cmd_lists, num_queues, copy_ordinal); + ucs_info("ze_ipc_iface: creating %u command list(s) to match %u " + "copy engine queue(s) " + "on queue group %u", + self->num_cmd_lists, num_queues, copy_ordinal); } } @@ -544,7 +551,8 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, return UCS_ERR_INVALID_PARAM; } if (self->num_cmd_lists > UCT_ZE_IPC_MAX_PEERS) { - ucs_error("ze_ipc_iface: computed num_cmd_lists (%u) exceeds maximum (%u)", + ucs_error("ze_ipc_iface: computed num_cmd_lists (%u) exceeds maximum " + "(%u)", self->num_cmd_lists, UCT_ZE_IPC_MAX_PEERS); return UCS_ERR_INVALID_PARAM; } @@ -569,16 +577,18 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, queue_desc.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC; queue_desc.ordinal = copy_ordinal; queue_desc.mode = ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS; - queue_desc.index = 0; /* MUST be 0 for immediate command lists */ + queue_desc.index = 0; /* MUST be 0 for immediate command lists */ queue_desc.flags = 0; queue_desc.priority = ZE_COMMAND_QUEUE_PRIORITY_NORMAL; for (i = 0; i < self->num_cmd_lists; i++) { ret = zeCommandListCreateImmediate(self->ze_context, self->ze_device, - &queue_desc, &self->queue_desc[i].cmd_list); + &queue_desc, + &self->queue_desc[i].cmd_list); if (ret != ZE_RESULT_SUCCESS) { unsigned j; - ucs_error("ze_ipc_iface: zeCommandListCreateImmediate[%u] failed with error 0x%x", + ucs_error("ze_ipc_iface: zeCommandListCreateImmediate[%u] failed " + "with error 0x%x", i, ret); /* Clean up previously created command lists */ for (j = 0; j < i; j++) { @@ -604,7 +614,7 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, * - Event reuse via bitmap tracking * - Typical pool size: 1024 events (configurable) */ - self->event_pool_size = 1024; /* TODO: make this configurable */ + self->event_pool_size = 1024; /* TODO: make this configurable */ event_pool_desc.stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC; event_pool_desc.pNext = NULL; @@ -614,7 +624,8 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, ret = zeEventPoolCreate(self->ze_context, &event_pool_desc, 1, &self->ze_device, &self->ze_event_pool); if (ret != ZE_RESULT_SUCCESS) { - ucs_error("ze_ipc_iface: zeEventPoolCreate failed with error 0x%x", ret); + ucs_error("ze_ipc_iface: zeEventPoolCreate failed with error 0x%x", + ret); /* Clean up command lists */ for (i = 0; i < self->num_cmd_lists; i++) { zeCommandListDestroy(self->queue_desc[i].cmd_list); @@ -623,8 +634,10 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, } /* Initialize event bitmap for tracking free events */ - bitmap_size = (self->event_pool_size + 63) / 64; /* Round up to 64-bit words */ - self->event_bitmap = ucs_calloc(bitmap_size, sizeof(uint64_t), "ze_ipc_event_bitmap"); + bitmap_size = (self->event_pool_size + 63) / + 64; /* Round up to 64-bit words */ + self->event_bitmap = ucs_calloc(bitmap_size, sizeof(uint64_t), + "ze_ipc_event_bitmap"); if (self->event_bitmap == NULL) { ucs_error("ze_ipc_iface: failed to allocate event bitmap"); zeEventPoolDestroy(self->ze_event_pool); @@ -666,14 +679,13 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) /* Clean up outstanding events from all command list queues */ for (i = 0; i < self->num_cmd_lists; i++) { while (!ucs_queue_is_empty(&self->queue_desc[i].event_queue)) { - event_desc = ucs_queue_pull_elem_non_empty(&self->queue_desc[i].event_queue, - uct_ze_ipc_event_desc_t, - queue); + event_desc = ucs_queue_pull_elem_non_empty( + &self->queue_desc[i].event_queue, uct_ze_ipc_event_desc_t, + queue); if (event_desc->mapped_addr != NULL) { uct_ze_ipc_unmap_memhandle(event_desc->pid, event_desc->address, event_desc->mapped_addr, - self->ze_context, - event_desc->dup_fd, + self->ze_context, event_desc->dup_fd, self->config.enable_cache); } if (event_desc->event != NULL) { @@ -708,7 +720,8 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) static ucs_status_t -uct_ze_ipc_query_devices(uct_md_h uct_md, uct_tl_device_resource_t **tl_devices_p, +uct_ze_ipc_query_devices(uct_md_h uct_md, + uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p) { return uct_ze_base_query_devices_common(uct_md, UCT_DEVICE_TYPE_ACC, @@ -717,10 +730,11 @@ uct_ze_ipc_query_devices(uct_md_h uct_md, uct_tl_device_resource_t **tl_devices_ UCS_CLASS_DEFINE(uct_ze_ipc_iface_t, uct_base_iface_t); -UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_iface_t, uct_iface_t, uct_md_h, uct_worker_h, - const uct_iface_params_t*, const uct_iface_config_t*); +UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_iface_t, uct_iface_t, uct_md_h, + uct_worker_h, const uct_iface_params_t*, + const uct_iface_config_t*); static UCS_CLASS_DEFINE_DELETE_FUNC(uct_ze_ipc_iface_t, uct_iface_t); -UCT_TL_DEFINE(&uct_ze_ipc_component, ze_ipc, - uct_ze_ipc_query_devices, uct_ze_ipc_iface_t, "ZE_IPC_", - uct_ze_ipc_iface_config_table, uct_ze_ipc_iface_config_t); +UCT_TL_DEFINE(&uct_ze_ipc_component, ze_ipc, uct_ze_ipc_query_devices, + uct_ze_ipc_iface_t, "ZE_IPC_", uct_ze_ipc_iface_config_table, + uct_ze_ipc_iface_config_t); diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.h b/src/uct/ze/ze_ipc/ze_ipc_iface.h index d09d5320e62..50f2ee39468 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.h +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.h @@ -20,12 +20,12 @@ typedef struct uct_ze_ipc_iface_config { uct_iface_config_t super; - unsigned max_poll; /* query attempts w.o success */ - unsigned max_cmd_lists; /* max number of command lists for parallel progress */ - int enable_cache; /* enable/disable ipc handle cache */ - double bandwidth; /* estimated bandwidth */ - double latency; /* estimated latency */ - double overhead; /* estimated CPU overhead */ + unsigned max_poll; /* query attempts w.o success */ + unsigned max_cmd_lists; /* max number of command lists for parallel progress */ + int enable_cache; /* enable/disable ipc handle cache */ + double bandwidth; /* estimated bandwidth */ + double latency; /* estimated latency */ + double overhead; /* estimated CPU overhead */ } uct_ze_ipc_iface_config_t; @@ -34,42 +34,44 @@ typedef struct uct_ze_ipc_iface_config { * Similar to CUDA IPC's uct_cuda_queue_desc_t design */ typedef struct uct_ze_ipc_queue_desc { - ucs_queue_elem_t queue; /* element in active queue list */ - ze_command_list_handle_t cmd_list; /* immediate command list */ - ucs_queue_head_t event_queue; /* queue of outstanding events */ + ucs_queue_elem_t queue; /* element in active queue list */ + ze_command_list_handle_t cmd_list; /* immediate command list */ + ucs_queue_head_t event_queue; /* queue of outstanding events */ } uct_ze_ipc_queue_desc_t; typedef struct uct_ze_ipc_iface { - uct_base_iface_t super; - ze_context_handle_t ze_context; - ze_device_handle_t ze_device; - uct_ze_ipc_iface_config_t config; + uct_base_iface_t super; + ze_context_handle_t ze_context; + ze_device_handle_t ze_device; + uct_ze_ipc_iface_config_t config; /* Multi-command-list support for parallel progress */ - uct_ze_ipc_queue_desc_t queue_desc[UCT_ZE_IPC_MAX_PEERS]; /* array of queue descriptors */ - ucs_queue_head_t active_queue; /* queue of active queue descriptors */ - unsigned num_cmd_lists; /* actual number of command lists */ - unsigned next_cmd_list; /* round-robin index for load balancing */ + uct_ze_ipc_queue_desc_t + queue_desc[UCT_ZE_IPC_MAX_PEERS]; /* array of queue descriptors */ + ucs_queue_head_t active_queue; /* queue of active queue descriptors */ + unsigned num_cmd_lists; /* actual number of command lists */ + unsigned next_cmd_list; /* round-robin index for load balancing */ /* Pre-allocated event pool for performance optimization */ - ze_event_pool_handle_t ze_event_pool; /* shared event pool for all operations */ - unsigned event_pool_size; /* number of events in the pool */ - ucs_spinlock_t event_lock; /* lock for event allocation */ - uint64_t *event_bitmap; /* bitmap to track free events */ + ze_event_pool_handle_t + ze_event_pool; /* shared event pool for all operations */ + unsigned event_pool_size; /* number of events in the pool */ + ucs_spinlock_t event_lock; /* lock for event allocation */ + uint64_t *event_bitmap; /* bitmap to track free events */ } uct_ze_ipc_iface_t; typedef struct uct_ze_ipc_event_desc { - ze_event_handle_t event; - ze_event_pool_handle_t event_pool; /* NULL when using shared pool */ - void *mapped_addr; - uct_completion_t *comp; - ucs_queue_elem_t queue; - int dup_fd; /* duplicated fd to close, or -1 if none */ - pid_t pid; /* remote process id for cache lookup */ - uintptr_t address; /* remote base address for cache lookup */ - unsigned event_index; /* index in shared event pool, or -1 if using private pool */ + ze_event_handle_t event; + ze_event_pool_handle_t event_pool; /* NULL when using shared pool */ + void *mapped_addr; + uct_completion_t *comp; + ucs_queue_elem_t queue; + int dup_fd; /* duplicated fd to close, or -1 if none */ + pid_t pid; /* remote process id for cache lookup */ + uintptr_t address; /* remote base address for cache lookup */ + unsigned event_index; /* index in shared event pool, or -1 if using private pool */ } uct_ze_ipc_event_desc_t; @@ -80,7 +82,8 @@ typedef struct uct_ze_ipc_event_desc { * @param event_p Pointer to store the allocated event handle * @return Event index on success, -1 on failure */ -int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t *event_p); +int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, + ze_event_handle_t *event_p); /** * Free an event back to the shared event pool diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index 9327ef04a9b..e83255cd080 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -4,7 +4,7 @@ */ #ifdef HAVE_CONFIG_H -# include "config.h" +#include "config.h" #endif #include "ze_ipc_md.h" @@ -25,11 +25,11 @@ /* Helper function to compute a simple checksum of IPC handle for verification */ static uint32_t uct_ze_ipc_handle_checksum(const ze_ipc_mem_handle_t *handle) { - const unsigned char *bytes = (const unsigned char *)handle; - uint32_t sum = 0; + const unsigned char *bytes = (const unsigned char*)handle; + uint32_t sum = 0; for (size_t i = 0; i < sizeof(ze_ipc_mem_handle_t); i++) { sum += bytes[i]; - sum = (sum << 1) | (sum >> 31); /* rotate left */ + sum = (sum << 1) | (sum >> 31); /* rotate left */ } return sum; } @@ -40,36 +40,31 @@ static void uct_ze_ipc_print_handle(const char *prefix, const ze_ipc_mem_handle_t *handle, uintptr_t addr, size_t len, int dev_num) { - const unsigned char *bytes = (const unsigned char *)handle; - uint32_t checksum = uct_ze_ipc_handle_checksum(handle); + const unsigned char *bytes = (const unsigned char*)handle; + uint32_t checksum = uct_ze_ipc_handle_checksum(handle); /* Print first 16 bytes of handle + checksum for quick comparison */ ucs_info("%s: addr=0x%lx len=%zu dev=%d checksum=0x%08x " "handle[0-15]=%02x%02x%02x%02x%02x%02x%02x%02x" "%02x%02x%02x%02x%02x%02x%02x%02x", - prefix, (unsigned long)addr, len, dev_num, checksum, - bytes[0], bytes[1], bytes[2], bytes[3], - bytes[4], bytes[5], bytes[6], bytes[7], - bytes[8], bytes[9], bytes[10], bytes[11], - bytes[12], bytes[13], bytes[14], bytes[15]); + prefix, (unsigned long)addr, len, dev_num, checksum, bytes[0], + bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], + bytes[7], bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], + bytes[13], bytes[14], bytes[15]); } static ucs_config_field_t uct_ze_ipc_md_config_table[] = { - {"", "", NULL, - ucs_offsetof(uct_ze_ipc_md_config_t, super), + {"", "", NULL, ucs_offsetof(uct_ze_ipc_md_config_t, super), UCS_CONFIG_TYPE_TABLE(uct_md_config_table)}, - {"DEVICE_ORDINAL", "0", - "Ordinal of the GPU device to use for IPC.", - ucs_offsetof(uct_ze_ipc_md_config_t, device_ordinal), - UCS_CONFIG_TYPE_INT}, + {"DEVICE_ORDINAL", "0", "Ordinal of the GPU device to use for IPC.", + ucs_offsetof(uct_ze_ipc_md_config_t, device_ordinal), UCS_CONFIG_TYPE_INT}, {NULL} }; -static ucs_status_t -uct_ze_ipc_md_query(uct_md_h md, uct_md_attr_v2_t *md_attr) +static ucs_status_t uct_ze_ipc_md_query(uct_md_h md, uct_md_attr_v2_t *md_attr) { uct_md_base_md_query(md_attr); md_attr->rkey_packed_size = sizeof(uct_ze_ipc_key_t); @@ -99,14 +94,13 @@ uct_ze_ipc_mkey_pack(uct_md_h uct_md, uct_mem_h memh, void *address, } -static ucs_status_t -uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, size_t length, - uct_ze_ipc_key_t *key) +static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, + size_t length, uct_ze_ipc_key_t *key) { ze_memory_allocation_properties_t props = { .stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES }; - ze_device_handle_t alloc_device = NULL; + ze_device_handle_t alloc_device = NULL; void *base_address; size_t alloc_size; ze_result_t ret; @@ -115,7 +109,8 @@ uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, size_t length, /* Get memory allocation properties to verify this is ZE device memory * Also get the device where memory was allocated */ - ret = zeMemGetAllocProperties(md->ze_context, address, &props, &alloc_device); + ret = zeMemGetAllocProperties(md->ze_context, address, &props, + &alloc_device); if ((ret != ZE_RESULT_SUCCESS) || (props.type == ZE_MEMORY_TYPE_UNKNOWN)) { ucs_error("failed to get allocation properties for %p", address); return UCS_ERR_INVALID_ADDR; @@ -165,7 +160,8 @@ uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, uct_ze_ipc_key_t *key; ucs_status_t status; - ucs_info("ze_ipc_md: mem_reg called address=%p length=%zu", address, length); + ucs_info("ze_ipc_md: mem_reg called address=%p length=%zu", address, + length); key = ucs_malloc(sizeof(*key), "uct_ze_ipc_key_t"); if (key == NULL) { @@ -181,7 +177,8 @@ uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, return status; } - ucs_info("ze_ipc_md: mem_reg succeeded address=%p key->address=0x%lx key->length=%zu", + ucs_info("ze_ipc_md: mem_reg succeeded address=%p key->address=0x%lx " + "key->length=%zu", address, (unsigned long)key->address, key->length); *memh_p = key; @@ -215,7 +212,7 @@ uct_ze_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, const uct_rkey_unpack_params_t *params, uct_rkey_t *rkey_p, void **handle_p) { - uct_ze_ipc_key_t *packed = (uct_ze_ipc_key_t *)rkey_buffer; + uct_ze_ipc_key_t *packed = (uct_ze_ipc_key_t*)rkey_buffer; uct_ze_ipc_key_t *key; /* Print handle info for verification - compare with mkey_pack output */ @@ -236,12 +233,11 @@ uct_ze_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, } -static ucs_status_t -uct_ze_ipc_rkey_release(uct_component_t *component, uct_rkey_t rkey, - void *handle) +static ucs_status_t uct_ze_ipc_rkey_release(uct_component_t *component, + uct_rkey_t rkey, void *handle) { ucs_assert(handle == NULL); - ucs_free((void *)rkey); + ucs_free((void*)rkey); return UCS_OK; } @@ -262,18 +258,24 @@ static ucs_status_t uct_ze_ipc_md_open(uct_component_h component, const char *md_name, const uct_md_config_t *uct_md_config, uct_md_h *md_p) { - static uct_md_ops_t md_ops = { + static uct_md_ops_t md_ops = { .close = uct_ze_ipc_md_close, .query = uct_ze_ipc_md_query, - .mem_alloc = (uct_md_mem_alloc_func_t)ucs_empty_function_return_unsupported, - .mem_free = (uct_md_mem_free_func_t)ucs_empty_function_return_unsupported, - .mem_advise = (uct_md_mem_advise_func_t)ucs_empty_function_return_unsupported, + .mem_alloc = (uct_md_mem_alloc_func_t) + ucs_empty_function_return_unsupported, + .mem_free = (uct_md_mem_free_func_t) + ucs_empty_function_return_unsupported, + .mem_advise = (uct_md_mem_advise_func_t) + ucs_empty_function_return_unsupported, .mem_reg = uct_ze_ipc_mem_reg, .mem_dereg = uct_ze_ipc_mem_dereg, - .mem_query = (uct_md_mem_query_func_t)ucs_empty_function_return_unsupported, + .mem_query = (uct_md_mem_query_func_t) + ucs_empty_function_return_unsupported, .mkey_pack = uct_ze_ipc_mkey_pack, - .mem_attach = (uct_md_mem_attach_func_t)ucs_empty_function_return_unsupported, - .detect_memory_type = (uct_md_detect_memory_type_func_t)ucs_empty_function_return_unsupported, + .mem_attach = (uct_md_mem_attach_func_t) + ucs_empty_function_return_unsupported, + .detect_memory_type = (uct_md_detect_memory_type_func_t) + ucs_empty_function_return_unsupported, }; uct_ze_ipc_md_config_t *config = ucs_derived_of(uct_md_config, uct_ze_ipc_md_config_t); @@ -318,21 +320,24 @@ uct_ze_ipc_md_open(uct_component_h component, const char *md_name, uct_component_t uct_ze_ipc_component = { .query_md_resources = uct_ze_base_query_md_resources, .md_open = uct_ze_ipc_md_open, - .cm_open = (uct_component_cm_open_func_t)ucs_empty_function_return_unsupported, + .cm_open = (uct_component_cm_open_func_t) + ucs_empty_function_return_unsupported, .rkey_unpack = uct_ze_ipc_rkey_unpack, - .rkey_ptr = (uct_component_rkey_ptr_func_t)ucs_empty_function_return_unsupported, + .rkey_ptr = (uct_component_rkey_ptr_func_t) + ucs_empty_function_return_unsupported, .rkey_release = uct_ze_ipc_rkey_release, .rkey_compare = uct_base_rkey_compare, .name = "ze_ipc", - .md_config = { - .name = "ZE-IPC memory domain", - .prefix = "ZE_IPC_", - .table = uct_ze_ipc_md_config_table, - .size = sizeof(uct_ze_ipc_md_config_t), - }, - .cm_config = UCS_CONFIG_EMPTY_GLOBAL_LIST_ENTRY, - .tl_list = UCT_COMPONENT_TL_LIST_INITIALIZER(&uct_ze_ipc_component), - .flags = 0, - .md_vfs_init = (uct_component_md_vfs_init_func_t)ucs_empty_function + .md_config = + { + .name = "ZE-IPC memory domain", + .prefix = "ZE_IPC_", + .table = uct_ze_ipc_md_config_table, + .size = sizeof(uct_ze_ipc_md_config_t), + }, + .cm_config = UCS_CONFIG_EMPTY_GLOBAL_LIST_ENTRY, + .tl_list = UCT_COMPONENT_TL_LIST_INITIALIZER(&uct_ze_ipc_component), + .flags = 0, + .md_vfs_init = (uct_component_md_vfs_init_func_t)ucs_empty_function }; UCT_COMPONENT_REGISTER(&uct_ze_ipc_component); diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.h b/src/uct/ze/ze_ipc/ze_ipc_md.h index f816749fc0f..b040a6840b8 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.h +++ b/src/uct/ze/ze_ipc/ze_ipc_md.h @@ -17,9 +17,9 @@ extern uct_component_t uct_ze_ipc_component; * @brief ze ipc MD descriptor */ typedef struct uct_ze_ipc_md { - uct_md_t super; /**< Domain info */ + uct_md_t super; /**< Domain info */ ze_context_handle_t ze_context; /**< Level Zero context */ - ze_device_handle_t ze_device; /**< Level Zero device */ + ze_device_handle_t ze_device; /**< Level Zero device */ } uct_ze_ipc_md_t; @@ -37,10 +37,10 @@ typedef struct uct_ze_ipc_md_config { */ typedef struct uct_ze_ipc_key { ze_ipc_mem_handle_t ipc_handle; /**< IPC memory handle */ - pid_t pid; /**< Remote process ID (for cache) */ - uintptr_t address; /**< Base address of the allocation */ - size_t length; /**< Size of the allocation */ - int dev_num; /**< GPU device number */ + pid_t pid; /**< Remote process ID (for cache) */ + uintptr_t address; /**< Base address of the allocation */ + size_t length; /**< Size of the allocation */ + int dev_num; /**< GPU device number */ } uct_ze_ipc_key_t; diff --git a/test/gtest/uct/ze/test_ze_base.cc b/test/gtest/uct/ze/test_ze_base.cc index 6c05acb6146..112a8185e7d 100644 --- a/test/gtest/uct/ze/test_ze_base.cc +++ b/test/gtest/uct/ze/test_ze_base.cc @@ -77,8 +77,8 @@ UCS_TEST_F(test_ze_base, get_device_ordinal_unknown_handle) { } /* Bogus pointer that cannot be a registered device handle. */ - ze_device_handle_t bogus = - reinterpret_cast(uintptr_t(0xdead)); + ze_device_handle_t bogus = reinterpret_cast( + uintptr_t(0xdead)); EXPECT_EQ(-1, uct_ze_base_get_device_ordinal(bogus)); EXPECT_EQ(-1, uct_ze_base_get_device_ordinal(NULL)); } diff --git a/test/gtest/uct/ze/test_ze_copy_md.cc b/test/gtest/uct/ze/test_ze_copy_md.cc index cad68b03610..57c8de29b60 100644 --- a/test/gtest/uct/ze/test_ze_copy_md.cc +++ b/test/gtest/uct/ze/test_ze_copy_md.cc @@ -19,7 +19,8 @@ extern "C" { */ class test_ze_copy_md : public ucs::test { protected: - void SetUp() override { + void SetUp() override + { ucs::test::SetUp(); if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { UCS_TEST_SKIP_R("Level Zero runtime not available"); @@ -29,13 +30,14 @@ class test_ze_copy_md : public ucs::test { } } - static uct_md_h open_first_md() { + static uct_md_h open_first_md() + { uct_md_resource_desc_t *resources = NULL; - unsigned num = 0; - ucs_status_t status; + unsigned num = 0; + ucs_status_t status; - status = uct_ze_copy_component.query_md_resources(&uct_ze_copy_component, - &resources, &num); + status = uct_ze_copy_component.query_md_resources( + &uct_ze_copy_component, &resources, &num); EXPECT_UCS_OK(status); if (num == 0) { ucs_free(resources); @@ -48,8 +50,8 @@ class test_ze_copy_md : public ucs::test { EXPECT_UCS_OK(status); uct_md_h md = NULL; - status = uct_md_open(&uct_ze_copy_component, resources[0].md_name, - md_config, &md); + status = uct_md_open(&uct_ze_copy_component, resources[0].md_name, + md_config, &md); uct_config_release(md_config); ucs_free(resources); @@ -65,10 +67,11 @@ UCS_TEST_F(test_ze_copy_md, component_is_registered) { UCS_TEST_F(test_ze_copy_md, query_md_resources_returns_at_least_one) { uct_md_resource_desc_t *resources = NULL; - unsigned num = 0; + unsigned num = 0; - EXPECT_UCS_OK(uct_ze_copy_component.query_md_resources( - &uct_ze_copy_component, &resources, &num)); + EXPECT_UCS_OK( + uct_ze_copy_component.query_md_resources(&uct_ze_copy_component, + &resources, &num)); EXPECT_GE(num, 1u); ucs_free(resources); } @@ -81,7 +84,7 @@ UCS_TEST_F(test_ze_copy_md, open_close_md) { } auto *ze_md = ucs_derived_of(md, uct_ze_copy_md_t); - EXPECT_TRUE(ze_md->ze_device != NULL); + EXPECT_TRUE(ze_md->ze_device != NULL); EXPECT_TRUE(ze_md->ze_context != NULL); uct_md_close(md); @@ -115,14 +118,14 @@ UCS_TEST_F(test_ze_copy_md, mem_alloc_free_device) { UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); } - size_t length = 4096; - void *addr = NULL; - uct_mem_h memh = NULL; + size_t length = 4096; + void *addr = NULL; + uct_mem_h memh = NULL; - ucs_status_t status = - md->ops->mem_alloc(md, &length, &addr, UCS_MEMORY_TYPE_ZE_DEVICE, - UCS_SYS_DEVICE_ID_UNKNOWN, 0, - "test_ze_copy", &memh); + ucs_status_t status = md->ops->mem_alloc(md, &length, &addr, + UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); if (status == UCS_ERR_UNSUPPORTED) { UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); } @@ -144,13 +147,13 @@ UCS_TEST_F(test_ze_copy_md, detect_memory_type_device) { UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); } - size_t length = 4096; - void *addr = NULL; - uct_mem_h memh = NULL; - ucs_status_t status = - md->ops->mem_alloc(md, &length, &addr, UCS_MEMORY_TYPE_ZE_DEVICE, - UCS_SYS_DEVICE_ID_UNKNOWN, 0, - "test_ze_copy", &memh); + size_t length = 4096; + void *addr = NULL; + uct_mem_h memh = NULL; + ucs_status_t status = md->ops->mem_alloc(md, &length, &addr, + UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); if (status == UCS_ERR_UNSUPPORTED) { UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); } @@ -174,22 +177,22 @@ UCS_TEST_F(test_ze_copy_md, mem_query_base_and_length) { UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); } - size_t length = 8192; - void *addr = NULL; - uct_mem_h memh = NULL; - ucs_status_t status = - md->ops->mem_alloc(md, &length, &addr, UCS_MEMORY_TYPE_ZE_DEVICE, - UCS_SYS_DEVICE_ID_UNKNOWN, 0, - "test_ze_copy", &memh); + size_t length = 8192; + void *addr = NULL; + uct_mem_h memh = NULL; + ucs_status_t status = md->ops->mem_alloc(md, &length, &addr, + UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); if (status == UCS_ERR_UNSUPPORTED) { UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); } ASSERT_UCS_OK(status); uct_md_mem_attr_v2_t mem_attr = {}; - mem_attr.field_mask = UCT_MD_MEM_ATTR_V2_FIELD_MEM_TYPE | - UCT_MD_MEM_ATTR_V2_FIELD_BASE_ADDRESS | - UCT_MD_MEM_ATTR_V2_FIELD_ALLOC_LENGTH; + mem_attr.field_mask = UCT_MD_MEM_ATTR_V2_FIELD_MEM_TYPE | + UCT_MD_MEM_ATTR_V2_FIELD_BASE_ADDRESS | + UCT_MD_MEM_ATTR_V2_FIELD_ALLOC_LENGTH; EXPECT_UCS_OK(md->ops->mem_query(md, addr, length, &mem_attr)); EXPECT_EQ(UCS_MEMORY_TYPE_ZE_DEVICE, mem_attr.mem_type); EXPECT_TRUE(mem_attr.base_address != NULL); diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc index 938bad66e51..c8d65deac22 100644 --- a/test/gtest/uct/ze/test_ze_ipc_cache.cc +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -20,7 +20,8 @@ extern "C" { */ class test_ze_ipc_cache : public ucs::test { protected: - void SetUp() override { + void SetUp() override + { ucs::test::SetUp(); if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { UCS_TEST_SKIP_R("Level Zero runtime not available"); @@ -92,7 +93,7 @@ UCS_TEST_F(test_ze_ipc_cache, create_destroy_repeated) { UCS_TEST_F(test_ze_ipc_cache, multiple_instances) { - const int N = 8; + const int N = 8; uct_ze_ipc_cache_t *caches[N] = {}; for (int i = 0; i < N; ++i) { @@ -186,8 +187,8 @@ UCS_TEST_F(test_ze_ipc_cache, purge_by_context_then_remap) { } auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); - const size_t size = 4096; - void *ptr = NULL; + const size_t size = 4096; + void *ptr = NULL; ze_device_mem_alloc_desc_t dev_desc = {}; dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; diff --git a/test/gtest/uct/ze/test_ze_ipc_md.cc b/test/gtest/uct/ze/test_ze_ipc_md.cc index 18c39bf4dc9..f562da81844 100644 --- a/test/gtest/uct/ze/test_ze_ipc_md.cc +++ b/test/gtest/uct/ze/test_ze_ipc_md.cc @@ -20,7 +20,8 @@ extern "C" { */ class test_ze_ipc_md : public ucs::test { protected: - void SetUp() override { + void SetUp() override + { ucs::test::SetUp(); if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { UCS_TEST_SKIP_R("Level Zero runtime not available"); @@ -31,29 +32,28 @@ class test_ze_ipc_md : public ucs::test { } /* Query MD resources via the component API used by the rest of UCT. */ - static ucs_status_t query_md_resources(uct_md_resource_desc_t **resources_p, - unsigned *num_p) + static ucs_status_t + query_md_resources(uct_md_resource_desc_t **resources_p, unsigned *num_p) { uct_component_attr_t attr; - ucs_status_t status; + ucs_status_t status; attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT; - status = uct_component_query(&uct_ze_ipc_component, &attr); + status = uct_component_query(&uct_ze_ipc_component, &attr); if (status != UCS_OK) { return status; } - unsigned count = attr.md_resource_count; - uct_md_resource_desc_t *res = (uct_md_resource_desc_t *) - ucs_calloc(count, sizeof(uct_md_resource_desc_t), - "ze_ipc_md_resources"); + unsigned count = attr.md_resource_count; + uct_md_resource_desc_t *res = (uct_md_resource_desc_t*)ucs_calloc( + count, sizeof(uct_md_resource_desc_t), "ze_ipc_md_resources"); if ((res == NULL) && (count > 0)) { return UCS_ERR_NO_MEMORY; } attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES; attr.md_resources = res; - status = uct_component_query(&uct_ze_ipc_component, &attr); + status = uct_component_query(&uct_ze_ipc_component, &attr); if (status != UCS_OK) { ucs_free(res); return status; @@ -64,9 +64,10 @@ class test_ze_ipc_md : public ucs::test { return UCS_OK; } - static uct_md_h open_first_md() { + static uct_md_h open_first_md() + { uct_md_resource_desc_t *resources = NULL; - unsigned num = 0; + unsigned num = 0; if (query_md_resources(&resources, &num) != UCS_OK) { return NULL; @@ -85,8 +86,8 @@ class test_ze_ipc_md : public ucs::test { } uct_md_h md = NULL; - status = uct_md_open(&uct_ze_ipc_component, resources[0].md_name, - md_config, &md); + status = uct_md_open(&uct_ze_ipc_component, resources[0].md_name, + md_config, &md); uct_config_release(md_config); ucs_free(resources); @@ -102,7 +103,7 @@ UCS_TEST_F(test_ze_ipc_md, component_is_registered) { UCS_TEST_F(test_ze_ipc_md, query_md_resources_returns_at_least_one) { uct_md_resource_desc_t *resources = NULL; - unsigned num = 0; + unsigned num = 0; EXPECT_UCS_OK(query_md_resources(&resources, &num)); EXPECT_GE(num, 1u); @@ -118,7 +119,7 @@ UCS_TEST_F(test_ze_ipc_md, open_close_md) { /* The post-merge ze_ipc_md must populate device + context. */ auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); - EXPECT_TRUE(ze_md->ze_device != NULL); + EXPECT_TRUE(ze_md->ze_device != NULL); EXPECT_TRUE(ze_md->ze_context != NULL); uct_md_close(md); @@ -156,9 +157,9 @@ UCS_TEST_F(test_ze_ipc_md, mem_reg_pack_dereg_device) { } auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); - const size_t size = 4096; - void *dev_ptr = NULL; - ze_device_mem_alloc_desc_t dev_desc = {}; + const size_t size = 4096; + void *dev_ptr = NULL; + ze_device_mem_alloc_desc_t dev_desc = {}; dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; ASSERT_EQ(ZE_RESULT_SUCCESS, From f8ec33bda77fd499d395e04596fd3e4eeec1d72b Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 09:27:42 -0700 Subject: [PATCH 25/35] AUTHORS: add Tushar Gohad Signed-off-by: Tushar Gohad --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 149e0d3056b..78f1ed59459 100644 --- a/AUTHORS +++ b/AUTHORS @@ -127,6 +127,7 @@ Swen Boehm Thomas Vegas Tomer Gilad Tony Curtis +Tushar Gohad Tzafrir Cohen Valentin Petrov Wenbin Lu From c8572bbfcfbd61cb36450f47dac086473a11e299 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 12:23:25 -0700 Subject: [PATCH 26/35] UCT/ZE_IPC: require a quiesced context when purging the cache uct_ze_ipc_purge_cache_by_context() closes mappings regardless of refcount and frees the cache descriptors, which is only sound if nothing is still using them. That precondition was never written down. It is not an extra constraint: the sole caller is md_close(), which destroys the ZE context on the next line, so an in-flight copy is dead either way. Honoring refcount would be worse - the mapping would survive to be closed against a destroyed context. Document that. The purge test now covers both paths: the quiesced sequence md_close() actually produces, and a purge with a reference outstanding. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_cache.h | 13 ++++++++--- test/gtest/uct/ze/test_ze_ipc_cache.cc | 30 +++++++++++++++++--------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.h b/src/uct/ze/ze_ipc/ze_ipc_cache.h index a2367cda9bf..139785da239 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.h +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.h @@ -94,12 +94,19 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, int dup_fd, int cache_enabled); - /** - * Purge and destroy all remote caches keyed by the given local Level Zero - * context, so that closed contexts do not leave behind cache entries that + * Purge and free all remote cache entries keyed by the given local Level Zero + * context, so that a closed context does not leave behind cache entries that * would later be used with a destroyed context. * + * Mappings are closed regardless of refcount, and cache descriptors are freed. + * The caller must therefore have quiesced this context: all operations issued + * against it must have completed, and no thread may enter map/unmap for it + * concurrently. This matches the UCT requirement that a memory domain have no + * outstanding operations at uct_md_close(), which is the only caller - and it is + * not an extra constraint, since md_close() destroys the context immediately + * afterwards and would invalidate any in-flight copy anyway. + * * @param ze_context Level Zero context being closed */ void uct_ze_ipc_purge_cache_by_context(ze_context_handle_t ze_context); diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc index c8d65deac22..87b948b70c2 100644 --- a/test/gtest/uct/ze/test_ze_ipc_cache.cc +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -203,20 +203,30 @@ UCS_TEST_F(test_ze_ipc_cache, purge_by_context_then_remap) { ASSERT_UCS_OK(md->ops->mkey_pack(md, memh, ptr, size, NULL, &key)); void *mapped = NULL; - int dup_fd = -1; - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, - ze_md->ze_device, &mapped, - &dup_fd)); + int dup_fd = -1; + /* quiesced purge, which is what md_close() guarantees: drop the reference + * first, then purge */ + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, + ze_md->ze_device, &mapped, &dup_fd)); + ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped, + ze_md->ze_context, dup_fd, 1)); uct_ze_ipc_purge_cache_by_context(ze_md->ze_context); - /* the purged region must not be referenced again; a fresh map/unmap - * must succeed cleanly */ + /* purge freed the cache descriptor as well, so this must rebuild it and go + * through a fresh open rather than reuse a freed region */ ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, - ze_md->ze_device, &mapped, - &dup_fd)); - uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped, ze_md->ze_context, - dup_fd, 0); + ze_md->ze_device, &mapped, &dup_fd)); + ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped, + ze_md->ze_context, dup_fd, 0)); + + /* purging with a reference still outstanding is deliberately destructive: + * md_close() destroys the context immediately afterwards, so the mapping + * cannot be left for the in-flight operation to reclaim. Nothing may call + * unmap for this mapping afterwards - the region is already gone. */ + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, + ze_md->ze_device, &mapped, &dup_fd)); + uct_ze_ipc_purge_cache_by_context(ze_md->ze_context); uct_md_mem_dereg_params_t dereg_params = {}; dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; From 25a01741147a3be8c699f584283e3cce66dba1a8 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 12:24:45 -0700 Subject: [PATCH 27/35] UCT/ZE_IPC: defer closing mappings that still have in-flight operations When a remote VA is reused, map_memhandle() drops the stale region from the page table and closes its mapping immediately - even if an earlier operation is still copying through it. The copy then runs against closed memory, and its unmap tripped the ucs_assert() on the page table lookup because the entry now belongs to the new allocation. Move such regions to a per-cache orphan list instead, keyed for lookup by mapped address since the page table entry is gone, and let the last unmap close them. uct_ze_ipc_cache_invalidate_regions() had the same problem and now shares the path. Factor out uct_ze_ipc_close_mapping() so the fd is closed even when zeMemCloseIpcHandle() fails, which leaked one dup'd fd per failure. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_cache.c | 118 ++++++++++++++++++++----- src/uct/ze/ze_ipc/ze_ipc_cache.h | 5 +- test/gtest/uct/ze/test_ze_ipc_cache.cc | 13 +-- 3 files changed, 107 insertions(+), 29 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index 72080e6be6e..62c68618aa5 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -88,22 +88,64 @@ uct_ze_ipc_cache_region_collect_callback(const ucs_pgtable_t *pgtable, } -static ucs_status_t -uct_ze_ipc_close_memhandle(uct_ze_ipc_cache_region_t *region) +static ucs_status_t uct_ze_ipc_close_mapping(ze_context_handle_t ze_context, + void *mapped_addr, int dup_fd) { ze_result_t ret; - ret = zeMemCloseIpcHandle(region->ze_context, region->mapped_addr); + ret = zeMemCloseIpcHandle(ze_context, mapped_addr); + if (dup_fd >= 0) { + close(dup_fd); + } + if (ret != ZE_RESULT_SUCCESS) { ucs_warn("zeMemCloseIpcHandle failed with error 0x%x", ret); return UCS_ERR_IO_ERROR; } - if (region->dup_fd >= 0) { - close(region->dup_fd); + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_close_memhandle(uct_ze_ipc_cache_region_t *region) +{ + return uct_ze_ipc_close_mapping(region->ze_context, region->mapped_addr, + region->dup_fd); +} + + +/* + * Drop a region that has already been unlinked from the page table. If an + * operation is still in flight on the mapping, closing it here would pull the + * memory out from under the running copy, so keep it on the orphan list and let + * the matching uct_ze_ipc_unmap_memhandle() close it. + */ +static void uct_ze_ipc_cache_region_put(uct_ze_ipc_cache_t *cache, + uct_ze_ipc_cache_region_t *region) +{ + if (region->refcount > 0) { + ucs_list_add_tail(&cache->orphan_list, ®ion->list); + return; } - return UCS_OK; + uct_ze_ipc_close_memhandle(region); + ucs_free(region); +} + + +static uct_ze_ipc_cache_region_t * +uct_ze_ipc_cache_find_orphan(uct_ze_ipc_cache_t *cache, void *mapped_addr) +{ + uct_ze_ipc_cache_region_t *region; + + ucs_list_for_each(region, &cache->orphan_list, list) { + if (region->mapped_addr == mapped_addr) { + return region; + } + } + + return NULL; } @@ -119,6 +161,17 @@ static void uct_ze_ipc_cache_purge(uct_ze_ipc_cache_t *cache) uct_ze_ipc_close_memhandle(region); ucs_free(region); } + + /* the context backing these mappings is going away, so orphans cannot be + * left for their in-flight operations to reclaim: closing them later would + * pass a destroyed context to zeMemCloseIpcHandle(). This is only safe + * because the caller must be quiesced - see uct_ze_ipc_purge_cache_by_context() */ + ucs_list_for_each_safe(region, tmp, &cache->orphan_list, list) { + ucs_list_del(®ion->list); + uct_ze_ipc_close_memhandle(region); + ucs_free(region); + } + ucs_trace("%s: ze ipc cache purged", cache->name); } @@ -184,13 +237,8 @@ static void uct_ze_ipc_cache_invalidate_regions(uct_ze_ipc_cache_t *cache, (void*)region->key.address, ucs_status_string(status)); } - status = uct_ze_ipc_close_memhandle(region); - if (status != UCS_OK) { - ucs_error("failed to close memhandle for base addr:%p (%s)", - (void*)region->key.address, ucs_status_string(status)); - } - - ucs_free(region); + ucs_list_del(®ion->list); + uct_ze_ipc_cache_region_put(cache, region); } ucs_trace("%s: closed memhandles in the range [%p..%p]", cache->name, from, to); @@ -249,6 +297,7 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, uct_ze_ipc_cache_t *cache; ucs_pgt_region_t *pgt_region; uct_ze_ipc_cache_region_t *region; + int orphaned; status = uct_ze_ipc_get_remote_cache(pid, ze_context, &cache); if (status != UCS_OK) { @@ -258,8 +307,25 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, /* use write lock because cache maybe modified */ pthread_rwlock_wrlock(&cache->lock); pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, &cache->pgtable, address); - ucs_assert(pgt_region != NULL); - region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + region = NULL; + if (pgt_region != NULL) { + region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + } + + /* the cached region may have been replaced by a remap of a reused remote + * address while this operation was in flight, in which case the mapping it + * used was moved to the orphan list */ + orphaned = (region == NULL) || (region->mapped_addr != mapped_addr); + if (orphaned) { + region = uct_ze_ipc_cache_find_orphan(cache, mapped_addr); + } + + if (region == NULL) { + pthread_rwlock_unlock(&cache->lock); + ucs_error("%s: no ze_ipc cache region for mapped address %p", + cache->name, mapped_addr); + return UCS_ERR_NO_ELEM; + } ucs_assert(region->refcount >= 1); region->refcount--; @@ -267,13 +333,18 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, /* * check refcount to see if an in-flight transfer is using the same mapping */ - if (!region->refcount && !cache_enabled) { - status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); - if (status != UCS_OK) { - ucs_error("failed to remove address:%p from cache (%s)", - (void*)region->key.address, ucs_status_string(status)); + if ((region->refcount == 0) && (orphaned || !cache_enabled)) { + if (orphaned) { + ucs_list_del(®ion->list); + } else { + status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_error("failed to remove address:%p from cache (%s)", + (void*)region->key.address, + ucs_status_string(status)); + } } - ucs_assert(region->mapped_addr == mapped_addr); + status = uct_ze_ipc_close_memhandle(region); ucs_free(region); } @@ -331,8 +402,7 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, ucs_error("failed to remove address:%p from cache (%s)", (void*)region->key.address, ucs_status_string(status)); } - uct_ze_ipc_close_memhandle(region); - ucs_free(region); + uct_ze_ipc_cache_region_put(cache, region); } status = uct_ze_ipc_open_memhandle(key, ze_context, ze_device, mapped_addr, @@ -424,6 +494,8 @@ uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name) goto err_destroy_rwlock; } + ucs_list_head_init(&cache_desc->orphan_list); + *cache = cache_desc; return UCS_OK; diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.h b/src/uct/ze/ze_ipc/ze_ipc_cache.h index 139785da239..c5bab708fe4 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.h +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.h @@ -36,8 +36,11 @@ struct uct_ze_ipc_cache_region { * Cache structure for managing IPC handle mappings */ struct uct_ze_ipc_cache { - pthread_rwlock_t lock; /**< Protects the page table */ + pthread_rwlock_t lock; /**< Protects the page table and orphan list */ ucs_pgtable_t pgtable; /**< Page table to hold the regions */ + ucs_list_link_t orphan_list; /**< Regions dropped from the page table + while still referenced by in-flight + operations */ char *name; /**< Name for debugging */ }; diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc index 87b948b70c2..1eae87f6188 100644 --- a/test/gtest/uct/ze/test_ze_ipc_cache.cc +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -155,11 +155,14 @@ UCS_TEST_F(test_ze_ipc_cache, stale_address_is_remapped) { EXPECT_NE(mapped_a, mapped_b) << "cache returned a's stale mapping for a different allocation"; - /* the second map_memhandle() already invalidated and freed a's region - * (both keys alias the same address), so only b's mapping is still - * live in the cache to unmap */ - uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, - ze_md->ze_context, dup_fd_b, 0); + /* the second map_memhandle() unlinked a's region from the pgtable, but a + * still holds a reference, so its mapping must survive on the orphan list + * and be reclaimed by its own unmap - looked up by mapped address, since + * the pgtable entry for this address now belongs to b */ + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_a.pid, key_a.address, mapped_a, + ze_md->ze_context, dup_fd_a, 0)); + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, + ze_md->ze_context, dup_fd_b, 0)); uct_md_mem_dereg_params_t dereg_params = {}; dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; From 0aaf37f65e22e0dbde7c5c0982b26bca93fa1363 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 12:25:11 -0700 Subject: [PATCH 28/35] UCT/ZE_IPC: close the mapping when caching it fails map_memhandle() opened the IPC handle and then jumped to err: if the region allocation or the page table insert failed, so the mapping and its dup'd fd were leaked and the caller got back a status with *mapped_addr still set. Add an err_close: path that closes the mapping and clears the output arguments. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_cache.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index 62c68618aa5..8db6d0b864f 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -419,7 +419,7 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, if (ret != 0) { ucs_warn("failed to allocate uct_ze_ipc_cache region"); status = UCS_ERR_NO_MEMORY; - goto err; + goto err_close; } region->super.start = ucs_align_down_pow2((uintptr_t)key->address, @@ -447,14 +447,19 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length, ucs_status_string(status)); ucs_free(region); - goto err; + goto err_close; } ucs_trace("%s: ze_ipc cache new region:" UCS_PGT_REGION_FMT " size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length); - status = UCS_OK; + pthread_rwlock_unlock(&cache->lock); + return UCS_OK; +err_close: + uct_ze_ipc_close_mapping(ze_context, *mapped_addr, *dup_fd); + *mapped_addr = NULL; + *dup_fd = -1; err: pthread_rwlock_unlock(&cache->lock); return status; From 5d0b1ea0841797a649b4fc73c84608a808ec4a0a Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 20 Aug 2026 12:25:49 -0700 Subject: [PATCH 29/35] UCT/ZE_IPC: discriminate cached mappings by allocation and process Comparing the packed IPC handle to decide whether a cached mapping still refers to the same remote allocation does not work: on Linux the handle carries an fd number from the exporting process, and fds are reused as eagerly as virtual addresses, so a freed-and-reallocated peer buffer at the same address can produce a byte-identical handle and hit in the cache. Pack ze_memory_allocation_properties_t.id and compare it with the length instead. That id is a per-process counter, so it does not survive PID reuse: once a peer exits and a new process takes over its PID, the new process's first allocation can report the same pid, address, id and length as the dead peer's, and the cache hands back the dead mapping. Pack the peer's process creation time as the generation that separates the two, as UCT keepalive already does, and fail mkey_pack if it cannot be read rather than export a key the importer would have to distrust. Both discriminators live in the cache-hit predicate, so a restarted peer reuses the existing per-(pid, context) cache and invalidates the stale region on first touch. Keying the cache hash on the generation instead would strand every dead peer's mappings until md_close. Note the id is Intel compute runtime behavior - ZE defines it as an allocation identifier without specifying its uniqueness scope. This changes the packed rkey layout, so both peers must run the same build. That is not a compatibility break yet, since the transport is not released, but the layout should get a version before it is. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_cache.c | 15 +++- src/uct/ze/ze_ipc/ze_ipc_md.c | 20 ++++- src/uct/ze/ze_ipc/ze_ipc_md.h | 5 ++ test/gtest/uct/ze/test_ze_ipc_cache.cc | 104 ++++++++++++++++++++++--- 4 files changed, 128 insertions(+), 16 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index 8db6d0b864f..a43a3003e1f 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -376,9 +376,18 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); - if (ucs_likely((region->key.length == key->length) && - !memcmp(®ion->key.ipc_handle, &key->ipc_handle, - sizeof(key->ipc_handle)))) { + /* the IPC handle itself cannot discriminate here: on Linux it carries + * an fd number in the exporting process, and fd numbers are reused as + * eagerly as virtual addresses. The driver allocation id is a better + * discriminator, though ZE only defines it as an allocation identifier + * and does not specify its uniqueness scope; the Intel compute runtime + * increments it per process. Since that counter restarts in a fresh + * process, pid alone does not separate a dead peer from a new one that + * reused its PID - the creation time supplies that generation. */ + if (ucs_likely( + (region->key.alloc_id == key->alloc_id) && + (region->key.proc_create_time == key->proc_create_time) && + (region->key.length == key->length))) { /* cache hit */ ucs_trace("%s: ze_ipc cache hit addr:%p size:%lu " "region:" UCS_PGT_REGION_FMT, diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index e83255cd080..f7c6d0d325e 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -103,6 +103,7 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, ze_device_handle_t alloc_device = NULL; void *base_address; size_t alloc_size; + unsigned long proc_create_time; ze_result_t ret; ucs_status_t status; int dev_ordinal; @@ -132,6 +133,15 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, return UCS_ERR_INVALID_ADDR; } + /* Resolve this before exporting the handle: without it the importer cannot + * tell our allocations from those of a dead process whose PID we reused, + * and exporting a key it must then distrust is worse than failing here */ + proc_create_time = ucs_sys_get_proc_create_time(getpid()); + if (proc_create_time == 0) { + ucs_error("failed to get process creation time for pid %d", getpid()); + return UCS_ERR_IO_ERROR; + } + /* Get IPC handle for the memory */ status = UCT_ZE_FUNC_LOG_ERR( zeMemGetIpcHandle(md->ze_context, base_address, &key->ipc_handle)); @@ -140,10 +150,12 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, return status; } - key->pid = getpid(); - key->address = (uintptr_t)base_address; - key->length = alloc_size; - key->dev_num = dev_ordinal; + key->pid = getpid(); + key->address = (uintptr_t)base_address; + key->length = alloc_size; + key->dev_num = dev_ordinal; + key->alloc_id = props.id; + key->proc_create_time = proc_create_time; ucs_trace("packed IPC handle for %p base=%p len=%zu dev=%d pid=%d", address, base_address, alloc_size, dev_ordinal, key->pid); diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.h b/src/uct/ze/ze_ipc/ze_ipc_md.h index b040a6840b8..1d6bdc48374 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.h +++ b/src/uct/ze/ze_ipc/ze_ipc_md.h @@ -37,6 +37,11 @@ typedef struct uct_ze_ipc_md_config { */ typedef struct uct_ze_ipc_key { ze_ipc_mem_handle_t ipc_handle; /**< IPC memory handle */ + uint64_t alloc_id; /**< Driver allocation id, distinguishes + allocations that reuse a VA */ + uint64_t proc_create_time; /**< Remote process creation time, + distinguishes processes that + reuse a PID */ pid_t pid; /**< Remote process ID (for cache) */ uintptr_t address; /**< Base address of the allocation */ size_t length; /**< Size of the allocation */ diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc index 1eae87f6188..ff1b98ed33e 100644 --- a/test/gtest/uct/ze/test_ze_ipc_cache.cc +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -28,8 +28,10 @@ class test_ze_ipc_cache : public ucs::test { } } - static uct_md_h open_first_md() { + static uct_md_h open_first_md() + { uct_component_attr_t attr; + attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT; if (uct_component_query(&uct_ze_ipc_component, &attr) != UCS_OK) { return NULL; @@ -40,15 +42,15 @@ class test_ze_ipc_cache : public ucs::test { return NULL; } - uct_md_resource_desc_t *res = (uct_md_resource_desc_t *) - ucs_calloc(count, sizeof(uct_md_resource_desc_t), - "ze_ipc_md_resources"); + uct_md_resource_desc_t *res = (uct_md_resource_desc_t*)ucs_calloc( + count, sizeof(uct_md_resource_desc_t), "ze_ipc_md_resources"); if (res == NULL) { return NULL; } attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES; attr.md_resources = res; + ucs_status_t status = uct_component_query(&uct_ze_ipc_component, &attr); if (status != UCS_OK) { ucs_free(res); @@ -109,7 +111,7 @@ UCS_TEST_F(test_ze_ipc_cache, multiple_instances) { /* * Simulates a peer freeing an allocation and a later, different allocation * reusing the same VA: two distinct real IPC keys are forced to report the - * same base address. The cache must detect the IPC-handle mismatch on the + * same base address. The cache must detect the allocation-id mismatch on the * second map and remap, rather than returning the first mapping. */ UCS_TEST_F(test_ze_ipc_cache, stale_address_is_remapped) { @@ -119,9 +121,11 @@ UCS_TEST_F(test_ze_ipc_cache, stale_address_is_remapped) { } auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); - const size_t size = 4096; - void *ptr_a = NULL, *ptr_b = NULL; + const size_t size = 4096; + void *ptr_a = NULL; + void *ptr_b = NULL; ze_device_mem_alloc_desc_t dev_desc = {}; + dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; ASSERT_EQ(ZE_RESULT_SUCCESS, @@ -142,8 +146,10 @@ UCS_TEST_F(test_ze_ipc_cache, stale_address_is_remapped) { /* simulate allocation B being reported at the VA allocation A used */ key_b.address = key_a.address; - void *mapped_a = NULL, *mapped_b = NULL; - int dup_fd_a = -1, dup_fd_b = -1; + void *mapped_a = NULL; + void *mapped_b = NULL; + int dup_fd_a = -1; + int dup_fd_b = -1; ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_a, ze_md->ze_context, ze_md->ze_device, &mapped_a, @@ -177,6 +183,85 @@ UCS_TEST_F(test_ze_ipc_cache, stale_address_is_remapped) { } +/* + * A peer can exit and a new process can reuse its PID. alloc_id is a + * per-process counter, so the restarted peer's first allocation can report the + * same (pid, address, alloc_id, length) as the dead peer's - every field the + * cache compares on a hit. Two distinct allocations are forced to agree on all + * four; the cache must still not hand back the first mapping. + */ +UCS_TEST_F(test_ze_ipc_cache, pid_reuse_is_remapped) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + + const size_t size = 4096; + void *ptr_a = NULL; + void *ptr_b = NULL; + ze_device_mem_alloc_desc_t dev_desc = {}; + + dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, + ze_md->ze_device, &ptr_a)); + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, + ze_md->ze_device, &ptr_b)); + + uct_mem_h memh_a = NULL, memh_b = NULL; + ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_a, size, NULL, &memh_a)); + ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_b, size, NULL, &memh_b)); + + uct_ze_ipc_key_t key_a = {}, key_b = {}; + ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_a, ptr_a, size, NULL, &key_a)); + ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_b, ptr_b, size, NULL, &key_b)); + + /* a restarted peer on a reused PID: same VA, same length, and its + * per-process allocation counter has handed out the same id again. Only the + * process creation time differs, which is the whole point */ + key_b.address = key_a.address; + key_b.alloc_id = key_a.alloc_id; + key_b.proc_create_time = key_a.proc_create_time + 1; + ASSERT_EQ(key_a.pid, key_b.pid); + ASSERT_EQ(key_a.length, key_b.length); + ASSERT_NE(0u, key_a.proc_create_time); + + void *mapped_a = NULL; + void *mapped_b = NULL; + int dup_fd_a = -1; + int dup_fd_b = -1; + + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_a, ze_md->ze_context, + ze_md->ze_device, &mapped_a, + &dup_fd_a)); + ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_b, ze_md->ze_context, + ze_md->ze_device, &mapped_b, + &dup_fd_b)); + + EXPECT_NE(mapped_a, mapped_b) + << "cache returned the dead peer's mapping after PID reuse"; + + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_a.pid, key_a.address, mapped_a, + ze_md->ze_context, dup_fd_a, 0)); + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, + ze_md->ze_context, dup_fd_b, 0)); + + uct_md_mem_dereg_params_t dereg_params = {}; + dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; + dereg_params.memh = memh_a; + EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); + dereg_params.memh = memh_b; + EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); + + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_a)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_b)); + uct_md_close(md); +} + + /* * uct_ze_ipc_purge_cache_by_context() must drop cached regions for a * closed context without leaving a dangling pgtable entry: mapping the @@ -193,6 +278,7 @@ UCS_TEST_F(test_ze_ipc_cache, purge_by_context_then_remap) { const size_t size = 4096; void *ptr = NULL; ze_device_mem_alloc_desc_t dev_desc = {}; + dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; ASSERT_EQ(ZE_RESULT_SUCCESS, From 0f87bd9021873ce0f9cac2d6094e00d3068ee838 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Tue, 25 Aug 2026 01:22:56 -0700 Subject: [PATCH 30/35] TEST/UCT/ZE_IPC: use mock mapping operations in cache tests The cache identity and purge tests only need to observe mapping lifecycle. Opening real Level Zero IPC handles makes them hardware dependent and exposes the runtime dma-buf descriptor leak, so they cannot run reliably in CI. Add internal open and close operations to the cache, retain Level Zero as the default production implementation, and use deterministic callbacks in the cache fixture. This keeps the allocation, process generation, and purge checks active without requiring a GPU. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_cache.c | 68 +++-- src/uct/ze/ze_ipc/ze_ipc_cache.h | 31 ++- src/uct/ze/ze_ipc/ze_ipc_ep.c | 2 +- test/gtest/uct/ze/test_ze_ipc_cache.cc | 350 ++++++++----------------- 4 files changed, 182 insertions(+), 269 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index a43a3003e1f..4639fc7931c 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -55,6 +55,11 @@ typedef struct uct_ze_ipc_remote_cache { static uct_ze_ipc_remote_cache_t uct_ze_ipc_remote_cache; +static ucs_status_t +uct_ze_ipc_create_cache_with_ops(uct_ze_ipc_cache_t **cache, const char *name, + const uct_ze_ipc_cache_ops_t *ops); + + static ucs_pgt_dir_t * uct_ze_ipc_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) { @@ -108,10 +113,11 @@ static ucs_status_t uct_ze_ipc_close_mapping(ze_context_handle_t ze_context, static ucs_status_t -uct_ze_ipc_close_memhandle(uct_ze_ipc_cache_region_t *region) +uct_ze_ipc_close_memhandle(uct_ze_ipc_cache_t *cache, + uct_ze_ipc_cache_region_t *region) { - return uct_ze_ipc_close_mapping(region->ze_context, region->mapped_addr, - region->dup_fd); + return cache->ops.close(region->ze_context, region->mapped_addr, + region->dup_fd); } @@ -129,7 +135,7 @@ static void uct_ze_ipc_cache_region_put(uct_ze_ipc_cache_t *cache, return; } - uct_ze_ipc_close_memhandle(region); + uct_ze_ipc_close_memhandle(cache, region); ucs_free(region); } @@ -158,7 +164,7 @@ static void uct_ze_ipc_cache_purge(uct_ze_ipc_cache_t *cache) ucs_pgtable_purge(&cache->pgtable, uct_ze_ipc_cache_region_collect_callback, ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { - uct_ze_ipc_close_memhandle(region); + uct_ze_ipc_close_memhandle(cache, region); ucs_free(region); } @@ -168,7 +174,7 @@ static void uct_ze_ipc_cache_purge(uct_ze_ipc_cache_t *cache) * because the caller must be quiesced - see uct_ze_ipc_purge_cache_by_context() */ ucs_list_for_each_safe(region, tmp, &cache->orphan_list, list) { ucs_list_del(®ion->list); - uct_ze_ipc_close_memhandle(region); + uct_ze_ipc_close_memhandle(cache, region); ucs_free(region); } @@ -218,6 +224,12 @@ static ucs_status_t uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, } +static const uct_ze_ipc_cache_ops_t uct_ze_ipc_cache_default_ops = { + .open = uct_ze_ipc_open_memhandle, + .close = uct_ze_ipc_close_mapping +}; + + static void uct_ze_ipc_cache_invalidate_regions(uct_ze_ipc_cache_t *cache, void *from, void *to) { @@ -245,9 +257,10 @@ static void uct_ze_ipc_cache_invalidate_regions(uct_ze_ipc_cache_t *cache, } -static ucs_status_t uct_ze_ipc_get_remote_cache(pid_t pid, - ze_context_handle_t ze_context, - uct_ze_ipc_cache_t **cache) +static ucs_status_t +uct_ze_ipc_get_remote_cache(pid_t pid, ze_context_handle_t ze_context, + const uct_ze_ipc_cache_ops_t *ops, + uct_ze_ipc_cache_t **cache) { ucs_status_t status = UCS_OK; char target_name[64]; @@ -267,7 +280,9 @@ static ucs_status_t uct_ze_ipc_get_remote_cache(pid_t pid, (khret == UCS_KH_PUT_BUCKET_CLEAR)) { ucs_snprintf_safe(target_name, sizeof(target_name), "dest:%d:%p", key.pid, key.ze_context); - status = uct_ze_ipc_create_cache(cache, target_name); + status = uct_ze_ipc_create_cache_with_ops( + cache, target_name, + (ops == NULL) ? &uct_ze_ipc_cache_default_ops : ops); if (status != UCS_OK) { kh_del(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, khiter); ucs_error("could not create ze ipc cache: %s", @@ -299,7 +314,7 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, uct_ze_ipc_cache_region_t *region; int orphaned; - status = uct_ze_ipc_get_remote_cache(pid, ze_context, &cache); + status = uct_ze_ipc_get_remote_cache(pid, ze_context, NULL, &cache); if (status != UCS_OK) { return status; } @@ -345,7 +360,7 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, } } - status = uct_ze_ipc_close_memhandle(region); + status = uct_ze_ipc_close_memhandle(cache, region); ucs_free(region); } @@ -355,9 +370,11 @@ ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, - (key, ze_context, ze_device, mapped_addr, dup_fd), + (key, ze_context, ze_device, ops, mapped_addr, dup_fd), uct_ze_ipc_key_t *key, ze_context_handle_t ze_context, - ze_device_handle_t ze_device, void **mapped_addr, int *dup_fd) + ze_device_handle_t ze_device, + const uct_ze_ipc_cache_ops_t *ops, void **mapped_addr, + int *dup_fd) { uct_ze_ipc_cache_t *cache; ucs_status_t status; @@ -365,7 +382,7 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, uct_ze_ipc_cache_region_t *region; int ret; - status = uct_ze_ipc_get_remote_cache(key->pid, ze_context, &cache); + status = uct_ze_ipc_get_remote_cache(key->pid, ze_context, ops, &cache); if (status != UCS_OK) { return status; } @@ -414,8 +431,7 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, uct_ze_ipc_cache_region_put(cache, region); } - status = uct_ze_ipc_open_memhandle(key, ze_context, ze_device, mapped_addr, - dup_fd); + status = cache->ops.open(key, ze_context, ze_device, mapped_addr, dup_fd); if (ucs_unlikely(status != UCS_OK)) { goto err; } @@ -466,7 +482,7 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, return UCS_OK; err_close: - uct_ze_ipc_close_mapping(ze_context, *mapped_addr, *dup_fd); + cache->ops.close(ze_context, *mapped_addr, *dup_fd); *mapped_addr = NULL; *dup_fd = -1; err: @@ -475,13 +491,16 @@ UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, } -ucs_status_t -uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name) +static ucs_status_t +uct_ze_ipc_create_cache_with_ops(uct_ze_ipc_cache_t **cache, const char *name, + const uct_ze_ipc_cache_ops_t *ops) { ucs_status_t status; uct_ze_ipc_cache_t *cache_desc; int ret; + ucs_assert((ops != NULL) && (ops->open != NULL) && (ops->close != NULL)); + cache_desc = ucs_malloc(sizeof(uct_ze_ipc_cache_t), "uct_ze_ipc_cache_t"); if (cache_desc == NULL) { ucs_error("failed to allocate memory for ze_ipc cache"); @@ -509,6 +528,7 @@ uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name) } ucs_list_head_init(&cache_desc->orphan_list); + cache_desc->ops = *ops; *cache = cache_desc; return UCS_OK; @@ -521,6 +541,14 @@ uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name) } +ucs_status_t +uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name) +{ + return uct_ze_ipc_create_cache_with_ops(cache, name, + &uct_ze_ipc_cache_default_ops); +} + + void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache) { uct_ze_ipc_cache_purge(cache); diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.h b/src/uct/ze/ze_ipc/ze_ipc_cache.h index c5bab708fe4..ad8bd0df8c1 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.h +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.h @@ -18,6 +18,22 @@ typedef struct uct_ze_ipc_cache_region uct_ze_ipc_cache_region_t; typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; +typedef ucs_status_t +(*uct_ze_ipc_cache_open_func_t)(uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + void **mapped_addr, int *dup_fd); + +typedef ucs_status_t +(*uct_ze_ipc_cache_close_func_t)(ze_context_handle_t ze_context, + void *mapped_addr, int dup_fd); + +typedef struct uct_ze_ipc_cache_ops { + uct_ze_ipc_cache_open_func_t open; + uct_ze_ipc_cache_close_func_t close; +} uct_ze_ipc_cache_ops_t; + + /** * Cache region structure for storing mapped IPC handles */ @@ -36,12 +52,13 @@ struct uct_ze_ipc_cache_region { * Cache structure for managing IPC handle mappings */ struct uct_ze_ipc_cache { - pthread_rwlock_t lock; /**< Protects the page table and orphan list */ - ucs_pgtable_t pgtable; /**< Page table to hold the regions */ - ucs_list_link_t orphan_list; /**< Regions dropped from the page table - while still referenced by in-flight - operations */ - char *name; /**< Name for debugging */ + pthread_rwlock_t lock; /**< Protects the page table and orphan list */ + ucs_pgtable_t pgtable; /**< Page table to hold the regions */ + ucs_list_link_t orphan_list; /**< Regions dropped from the page table + while still referenced by in-flight + operations */ + uct_ze_ipc_cache_ops_t ops; /**< Mapping operations */ + char *name; /**< Name for debugging */ }; @@ -70,6 +87,7 @@ void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache); * @param key Remote memory key containing IPC handle * @param ze_context Level Zero context * @param ze_device Level Zero device + * @param ops Mapping operations, or NULL to use Level Zero * @param mapped_addr Pointer to store the mapped address * @param dup_fd Pointer to store the duplicated file descriptor * @return UCS_OK on success, error code otherwise @@ -77,6 +95,7 @@ void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache); ucs_status_t uct_ze_ipc_map_memhandle(uct_ze_ipc_key_t *key, ze_context_handle_t ze_context, ze_device_handle_t ze_device, + const uct_ze_ipc_cache_ops_t *ops, void **mapped_addr, int *dup_fd); diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c index 71b5388a647..af15f031638 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_ep.c +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -228,7 +228,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, /* Use cache to map IPC handle */ status = uct_ze_ipc_map_memhandle(key, iface->ze_context, remote_device, - &mapped_addr, &local_fd); + NULL, &mapped_addr, &local_fd); if (status != UCS_OK) { ucs_error("ze_ipc_ep: uct_ze_ipc_map_memhandle failed"); return status; diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc index ff1b98ed33e..bb07430580e 100644 --- a/test/gtest/uct/ze/test_ze_ipc_cache.cc +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -6,73 +6,91 @@ #include extern "C" { -#include -#include #include -#include } +#include + /* - * Lifecycle tests for the ze_ipc cache. The cache is a simple pgtable- - * based handle map with no LRU; we only validate that it can be created - * and destroyed cleanly, including back-to-back and multiple instances. + * The cache tests use synthetic mapping operations to cover cache identity and + * lifecycle behavior without requiring a Level Zero device. */ class test_ze_ipc_cache : public ucs::test { protected: + struct cache_ops_state { + unsigned open_count; + unsigned close_count; + uintptr_t next_mapped_addr; + }; + void SetUp() override { ucs::test::SetUp(); - if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { - UCS_TEST_SKIP_R("Level Zero runtime not available"); - } + m_state.open_count = 0; + m_state.close_count = 0; + m_state.next_mapped_addr = 0x10000000ul; + } + + void TearDown() override + { + uct_ze_ipc_purge_cache_by_context(context()); + ucs::test::TearDown(); + } + + static ucs_status_t open_memhandle(uct_ze_ipc_key_t *, + ze_context_handle_t ze_context, + ze_device_handle_t, + void **mapped_addr, int *dup_fd) + { + cache_ops_state *state = reinterpret_cast(ze_context); + + *mapped_addr = reinterpret_cast(state->next_mapped_addr); + state->next_mapped_addr += 0x10000; + state->open_count++; + *dup_fd = -1; + return UCS_OK; + } + + static ucs_status_t close_memhandle(ze_context_handle_t ze_context, + void *, int) + { + cache_ops_state *state = reinterpret_cast(ze_context); + + state->close_count++; + return UCS_OK; + } + + ze_context_handle_t context() + { + return reinterpret_cast(&m_state); + } + + uct_ze_ipc_key_t make_key(uint64_t alloc_id = 1, + uint64_t proc_create_time = 1) + { + uct_ze_ipc_key_t key = {}; + + key.pid = getpid(); + key.address = 0x20000000ul; + key.length = 4096; + key.alloc_id = alloc_id; + key.proc_create_time = proc_create_time; + return key; } - static uct_md_h open_first_md() + ucs_status_t map(uct_ze_ipc_key_t *key, void **mapped_addr, int *dup_fd) { - uct_component_attr_t attr; - - attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT; - if (uct_component_query(&uct_ze_ipc_component, &attr) != UCS_OK) { - return NULL; - } - - unsigned count = attr.md_resource_count; - if (count == 0) { - return NULL; - } - - uct_md_resource_desc_t *res = (uct_md_resource_desc_t*)ucs_calloc( - count, sizeof(uct_md_resource_desc_t), "ze_ipc_md_resources"); - if (res == NULL) { - return NULL; - } - - attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES; - attr.md_resources = res; - - ucs_status_t status = uct_component_query(&uct_ze_ipc_component, &attr); - if (status != UCS_OK) { - ucs_free(res); - return NULL; - } - - uct_md_config_t *md_config = NULL; - status = uct_md_config_read(&uct_ze_ipc_component, NULL, NULL, - &md_config); - if (status != UCS_OK) { - ucs_free(res); - return NULL; - } - - uct_md_h md = NULL; - status = uct_md_open(&uct_ze_ipc_component, res[0].md_name, md_config, - &md); - uct_config_release(md_config); - ucs_free(res); - - return (status == UCS_OK) ? md : NULL; + static const uct_ze_ipc_cache_ops_t ops = { + open_memhandle, + close_memhandle + }; + + return uct_ze_ipc_map_memhandle(key, context(), NULL, &ops, mapped_addr, + dup_fd); } + + cache_ops_state m_state; }; @@ -108,220 +126,68 @@ UCS_TEST_F(test_ze_ipc_cache, multiple_instances) { } -/* - * Simulates a peer freeing an allocation and a later, different allocation - * reusing the same VA: two distinct real IPC keys are forced to report the - * same base address. The cache must detect the allocation-id mismatch on the - * second map and remap, rather than returning the first mapping. - */ -UCS_TEST_F(test_ze_ipc_cache, stale_address_is_remapped) { - uct_md_h md = open_first_md(); - if (md == NULL) { - UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); - } - auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); - - const size_t size = 4096; - void *ptr_a = NULL; - void *ptr_b = NULL; - ze_device_mem_alloc_desc_t dev_desc = {}; - - dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; - - ASSERT_EQ(ZE_RESULT_SUCCESS, - zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, - ze_md->ze_device, &ptr_a)); - ASSERT_EQ(ZE_RESULT_SUCCESS, - zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, - ze_md->ze_device, &ptr_b)); - - uct_mem_h memh_a = NULL, memh_b = NULL; - ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_a, size, NULL, &memh_a)); - ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_b, size, NULL, &memh_b)); - - uct_ze_ipc_key_t key_a = {}, key_b = {}; - ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_a, ptr_a, size, NULL, &key_a)); - ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_b, ptr_b, size, NULL, &key_b)); - - /* simulate allocation B being reported at the VA allocation A used */ - key_b.address = key_a.address; - +UCS_TEST_F(test_ze_ipc_cache, stale_allocation_is_remapped) { + uct_ze_ipc_key_t key_a = make_key(1); + uct_ze_ipc_key_t key_b = make_key(2); void *mapped_a = NULL; void *mapped_b = NULL; int dup_fd_a = -1; int dup_fd_b = -1; - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_a, ze_md->ze_context, - ze_md->ze_device, &mapped_a, - &dup_fd_a)); - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_b, ze_md->ze_context, - ze_md->ze_device, &mapped_b, - &dup_fd_b)); + ASSERT_UCS_OK(map(&key_a, &mapped_a, &dup_fd_a)); + ASSERT_UCS_OK(map(&key_b, &mapped_b, &dup_fd_b)); - EXPECT_NE(mapped_a, mapped_b) - << "cache returned a's stale mapping for a different allocation"; + EXPECT_EQ(2u, m_state.open_count); - /* the second map_memhandle() unlinked a's region from the pgtable, but a - * still holds a reference, so its mapping must survive on the orphan list - * and be reclaimed by its own unmap - looked up by mapped address, since - * the pgtable entry for this address now belongs to b */ EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_a.pid, key_a.address, mapped_a, - ze_md->ze_context, dup_fd_a, 0)); + context(), dup_fd_a, 0)); EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, - ze_md->ze_context, dup_fd_b, 0)); - - uct_md_mem_dereg_params_t dereg_params = {}; - dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; - dereg_params.memh = memh_a; - EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); - dereg_params.memh = memh_b; - EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); - - EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_a)); - EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_b)); - uct_md_close(md); + context(), dup_fd_b, 0)); + EXPECT_EQ(2u, m_state.close_count); } -/* - * A peer can exit and a new process can reuse its PID. alloc_id is a - * per-process counter, so the restarted peer's first allocation can report the - * same (pid, address, alloc_id, length) as the dead peer's - every field the - * cache compares on a hit. Two distinct allocations are forced to agree on all - * four; the cache must still not hand back the first mapping. - */ UCS_TEST_F(test_ze_ipc_cache, pid_reuse_is_remapped) { - uct_md_h md = open_first_md(); - if (md == NULL) { - UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); - } - auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); - - const size_t size = 4096; - void *ptr_a = NULL; - void *ptr_b = NULL; - ze_device_mem_alloc_desc_t dev_desc = {}; - - dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; - - ASSERT_EQ(ZE_RESULT_SUCCESS, - zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, - ze_md->ze_device, &ptr_a)); - ASSERT_EQ(ZE_RESULT_SUCCESS, - zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, - ze_md->ze_device, &ptr_b)); - - uct_mem_h memh_a = NULL, memh_b = NULL; - ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_a, size, NULL, &memh_a)); - ASSERT_UCS_OK(md->ops->mem_reg(md, ptr_b, size, NULL, &memh_b)); - - uct_ze_ipc_key_t key_a = {}, key_b = {}; - ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_a, ptr_a, size, NULL, &key_a)); - ASSERT_UCS_OK(md->ops->mkey_pack(md, memh_b, ptr_b, size, NULL, &key_b)); - - /* a restarted peer on a reused PID: same VA, same length, and its - * per-process allocation counter has handed out the same id again. Only the - * process creation time differs, which is the whole point */ - key_b.address = key_a.address; - key_b.alloc_id = key_a.alloc_id; - key_b.proc_create_time = key_a.proc_create_time + 1; - ASSERT_EQ(key_a.pid, key_b.pid); - ASSERT_EQ(key_a.length, key_b.length); - ASSERT_NE(0u, key_a.proc_create_time); - + uct_ze_ipc_key_t key_a = make_key(1, 1); + uct_ze_ipc_key_t key_b = make_key(1, 2); void *mapped_a = NULL; void *mapped_b = NULL; int dup_fd_a = -1; int dup_fd_b = -1; - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_a, ze_md->ze_context, - ze_md->ze_device, &mapped_a, - &dup_fd_a)); - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key_b, ze_md->ze_context, - ze_md->ze_device, &mapped_b, - &dup_fd_b)); + ASSERT_UCS_OK(map(&key_a, &mapped_a, &dup_fd_a)); + ASSERT_UCS_OK(map(&key_b, &mapped_b, &dup_fd_b)); - EXPECT_NE(mapped_a, mapped_b) - << "cache returned the dead peer's mapping after PID reuse"; + EXPECT_EQ(2u, m_state.open_count); EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_a.pid, key_a.address, mapped_a, - ze_md->ze_context, dup_fd_a, 0)); + context(), dup_fd_a, 0)); EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, - ze_md->ze_context, dup_fd_b, 0)); - - uct_md_mem_dereg_params_t dereg_params = {}; - dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; - dereg_params.memh = memh_a; - EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); - dereg_params.memh = memh_b; - EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); - - EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_a)); - EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr_b)); - uct_md_close(md); + context(), dup_fd_b, 0)); + EXPECT_EQ(2u, m_state.close_count); } -/* - * uct_ze_ipc_purge_cache_by_context() must drop cached regions for a - * closed context without leaving a dangling pgtable entry: mapping the - * same key again afterwards must go through a fresh open, not reuse a - * freed region. - */ UCS_TEST_F(test_ze_ipc_cache, purge_by_context_then_remap) { - uct_md_h md = open_first_md(); - if (md == NULL) { - UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); - } - auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); - - const size_t size = 4096; - void *ptr = NULL; - ze_device_mem_alloc_desc_t dev_desc = {}; - - dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; - - ASSERT_EQ(ZE_RESULT_SUCCESS, - zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, - ze_md->ze_device, &ptr)); - - uct_mem_h memh = NULL; - ASSERT_UCS_OK(md->ops->mem_reg(md, ptr, size, NULL, &memh)); - - uct_ze_ipc_key_t key = {}; - ASSERT_UCS_OK(md->ops->mkey_pack(md, memh, ptr, size, NULL, &key)); - - void *mapped = NULL; - int dup_fd = -1; - - /* quiesced purge, which is what md_close() guarantees: drop the reference - * first, then purge */ - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, - ze_md->ze_device, &mapped, &dup_fd)); - ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped, - ze_md->ze_context, dup_fd, 1)); - uct_ze_ipc_purge_cache_by_context(ze_md->ze_context); - - /* purge freed the cache descriptor as well, so this must rebuild it and go - * through a fresh open rather than reuse a freed region */ - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, - ze_md->ze_device, &mapped, &dup_fd)); - ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped, - ze_md->ze_context, dup_fd, 0)); - - /* purging with a reference still outstanding is deliberately destructive: - * md_close() destroys the context immediately afterwards, so the mapping - * cannot be left for the in-flight operation to reclaim. Nothing may call - * unmap for this mapping afterwards - the region is already gone. */ - ASSERT_UCS_OK(uct_ze_ipc_map_memhandle(&key, ze_md->ze_context, - ze_md->ze_device, &mapped, &dup_fd)); - uct_ze_ipc_purge_cache_by_context(ze_md->ze_context); - - uct_md_mem_dereg_params_t dereg_params = {}; - dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; - dereg_params.memh = memh; - EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); - - EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, ptr)); - uct_md_close(md); + uct_ze_ipc_key_t key = make_key(); + void *mapped_a = NULL; + void *mapped_b = NULL; + int dup_fd_a = -1; + int dup_fd_b = -1; + + ASSERT_UCS_OK(map(&key, &mapped_a, &dup_fd_a)); + ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped_a, + context(), dup_fd_a, 1)); + EXPECT_EQ(1u, m_state.open_count); + EXPECT_EQ(0u, m_state.close_count); + + uct_ze_ipc_purge_cache_by_context(context()); + EXPECT_EQ(1u, m_state.close_count); + + ASSERT_UCS_OK(map(&key, &mapped_b, &dup_fd_b)); + EXPECT_EQ(2u, m_state.open_count); + + ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped_b, + context(), dup_fd_b, 0)); + EXPECT_EQ(2u, m_state.close_count); } From 0156ae45f3f4ccdb3ff74d85fe3a8b59012df811 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Thu, 1 Oct 2026 13:45:26 -0700 Subject: [PATCH 31/35] UCT/ZE: Fix review comments on IPC device, events and logging Open the IPC handle on the iface device, whose command lists run the copy. The rkey carried the exporter's device ordinal in dev_num and the importer used it to pick a local device, which names the wrong GPU whenever the two processes enumerate GPUs in different orders. Remove dev_num from the key together with uct_ze_base_get_device_ordinal. That helper matched only root-device handles, so registering memory allocated on a sub-device failed with UCS_ERR_INVALID_ADDR. When the shared event pool is exhausted, give the operation a private one-event pool rather than failing with UCS_ERR_NO_RESOURCE. ep_pending_add cannot queue the operation, so UCP would retry the send in a tight loop. The shared pool size is now configurable with UCX_ZE_IPC_EVENT_POOL_SIZE (default 1024). Stop advertising EVENT_SEND_COMP and EVENT_RECV. The iface has no event fd and its event_arm returns UCS_ERR_UNSUPPORTED, which ucp_worker_iface_check_events_do treats as fatal ("failed to arm iface"); ucx_perftest aborted that way at teardown. Duplicate the fd from a remote IPC handle whenever it is non-negative and reject a negative one. The old (0, 65536) bound skipped the dup for fd 0 and for fds of 65536 and above. Remove the IPC handle dump and checksum helpers and the hand-rolled timeline profiling. Drop the per-registration and per-query info logs and move the iface init logs to debug. Report the actual cause in the ze_copy missing-device error and indent Makefile.am with spaces. Rename the get_device_by_ordinal_round_trip test and drop the tests of the removed helper. Signed-off-by: Tushar Gohad --- src/uct/ze/Makefile.am | 16 +-- src/uct/ze/base/ze_base.c | 20 ---- src/uct/ze/base/ze_base.h | 3 - src/uct/ze/copy/ze_copy_iface.c | 6 +- src/uct/ze/ze_ipc/ze_ipc_cache.c | 8 +- src/uct/ze/ze_ipc/ze_ipc_ep.c | 120 +++------------------- src/uct/ze/ze_ipc/ze_ipc_iface.c | 163 +++++++++++++++++------------- src/uct/ze/ze_ipc/ze_ipc_iface.h | 23 +++-- src/uct/ze/ze_ipc/ze_ipc_md.c | 69 +------------ src/uct/ze/ze_ipc/ze_ipc_md.h | 1 - test/gtest/uct/ze/test_ze_base.cc | 21 +--- 11 files changed, 143 insertions(+), 307 deletions(-) diff --git a/src/uct/ze/Makefile.am b/src/uct/ze/Makefile.am index 02224208788..988ab6a3a3b 100644 --- a/src/uct/ze/Makefile.am +++ b/src/uct/ze/Makefile.am @@ -27,20 +27,20 @@ libuct_ze_la_SOURCES = \ noinst_HEADERS += \ copy/ze_copy_md.h \ copy/ze_copy_iface.h \ - copy/ze_copy_ep.h \ + copy/ze_copy_ep.h \ ze_ipc/ze_ipc_md.h \ - ze_ipc/ze_ipc_iface.h \ - ze_ipc/ze_ipc_ep.h \ - ze_ipc/ze_ipc_cache.h + ze_ipc/ze_ipc_iface.h \ + ze_ipc/ze_ipc_ep.h \ + ze_ipc/ze_ipc_cache.h libuct_ze_la_SOURCES += \ copy/ze_copy_md.c \ copy/ze_copy_iface.c \ - copy/ze_copy_ep.c \ + copy/ze_copy_ep.c \ ze_ipc/ze_ipc_md.c \ - ze_ipc/ze_ipc_iface.c \ - ze_ipc/ze_ipc_ep.c \ - ze_ipc/ze_ipc_cache.c + ze_ipc/ze_ipc_iface.c \ + ze_ipc/ze_ipc_ep.c \ + ze_ipc/ze_ipc_cache.c PKG_CONFIG_NAME=ze diff --git a/src/uct/ze/base/ze_base.c b/src/uct/ze/base/ze_base.c index c664447bb6d..b975715aace 100644 --- a/src/uct/ze/base/ze_base.c +++ b/src/uct/ze/base/ze_base.c @@ -374,23 +374,6 @@ const uct_ze_subdevice_t *uct_ze_base_get_subdevice_by_global_id(int global_id) return &ze_subdevices[global_id]; } -int uct_ze_base_get_device_ordinal(ze_device_handle_t device) -{ - int i; - - if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { - return -1; - } - - for (i = 0; i < uct_ze_base.num_devices; i++) { - if (uct_ze_base.devices[i].root_device == device) { - return i; - } - } - - return -1; /* not found */ -} - ze_device_handle_t uct_ze_base_get_device(int ordinal) { if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { @@ -558,9 +541,6 @@ uct_ze_base_query_devices_common(uct_md_h md, uct_device_type_t dev_type, uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p) { - ucs_info("ze_base: query_devices_common for md component=%s type=%d", - md->component->name, dev_type); - return uct_single_device_resource(md, md->component->name, dev_type, UCS_SYS_DEVICE_ID_UNKNOWN, tl_devices_p, num_tl_devices_p); diff --git a/src/uct/ze/base/ze_base.h b/src/uct/ze/base/ze_base.h index 26166f02c05..d72c9d8c3d4 100644 --- a/src/uct/ze/base/ze_base.h +++ b/src/uct/ze/base/ze_base.h @@ -65,9 +65,6 @@ ze_device_handle_t uct_ze_base_get_device_handle_from_subdevice( ucs_sys_device_t uct_ze_base_get_sys_dev_from_handle(ze_device_handle_t device); -/* Get device ordinal from device handle, returns -1 if not found */ -int uct_ze_base_get_device_ordinal(ze_device_handle_t device); - /* Get root device handle by ordinal, returns NULL if out of range */ ze_device_handle_t uct_ze_base_get_device(int ordinal); diff --git a/src/uct/ze/copy/ze_copy_iface.c b/src/uct/ze/copy/ze_copy_iface.c index 8b7e65e1e06..770162106fa 100644 --- a/src/uct/ze/copy/ze_copy_iface.c +++ b/src/uct/ze/copy/ze_copy_iface.c @@ -259,7 +259,7 @@ static UCS_CLASS_INIT_FUNC(uct_ze_copy_iface_t, uct_md_h md, /* Use the device configured in the MD */ device = ze_md->ze_device; if (device == NULL) { - ucs_error("ze_copy_iface: uct_ze_base_get_device(0) returned NULL"); + ucs_error("ze_copy_iface: memory domain has no device"); return UCS_ERR_NO_DEVICE; } @@ -280,8 +280,8 @@ static UCS_CLASS_INIT_FUNC(uct_ze_copy_iface_t, uct_md_h md, self->ze_cmdl = cmdl; self->id = ucs_generate_uuid((uintptr_t)self); - ucs_info("ze_copy_iface: initialized iface device=%p cmdq=%p cmdl=%p", - device, cmdq, cmdl); + ucs_debug("ze_copy_iface: initialized iface on device %p cmdq %p cmdl %p", + device, cmdq, cmdl); return UCS_OK; } diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index 4639fc7931c..ef26d88a573 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -196,7 +196,13 @@ static ucs_status_t uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, remote_fd = *(int*)local_handle.data; /* Duplicate the file descriptor from remote process */ - if (key->pid != getpid() && remote_fd > 0 && remote_fd < 65536) { + if (key->pid != getpid()) { + if (remote_fd < 0) { + ucs_error("invalid fd %d in ipc handle from pid %d", remote_fd, + key->pid); + return UCS_ERR_INVALID_PARAM; + } + *dup_fd = uct_ze_ipc_dup_fd_from_pid(key->pid, remote_fd); if (*dup_fd < 0) { ucs_error("failed to duplicate fd %d from pid %d", remote_fd, diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c index af15f031638..7fb3190546b 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_ep.c +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -11,7 +11,6 @@ #include "ze_ipc_iface.h" #include "ze_ipc_md.h" #include "ze_ipc_cache.h" -#include #include #include @@ -24,9 +23,6 @@ #include #include -#include -#include - #define UCT_ZE_IPC_PUT 0 #define UCT_ZE_IPC_GET 1 @@ -104,65 +100,6 @@ UCS_CLASS_DEFINE_DELETE_FUNC(uct_ze_ipc_ep_t, uct_ep_t); (_rkey)) -/* Timeline profiling for performance analysis */ -typedef struct uct_ze_ipc_timeline { - struct timespec start; - struct timespec get_ipc_handle; - struct timespec event_alloc; - struct timespec async_copy; - struct timespec end; -} uct_ze_ipc_timeline_t; - -static UCS_F_ALWAYS_INLINE void uct_ze_ipc_timeline_record(struct timespec *ts) -{ - clock_gettime(CLOCK_MONOTONIC, ts); -} - -static UCS_F_ALWAYS_INLINE void -uct_ze_ipc_timeline_report(const uct_ze_ipc_timeline_t *timeline, size_t length, - unsigned cmd_list_idx) -{ - double ipc_time, event_time, copy_time, other_time, total_time; - - ipc_time = (timeline->get_ipc_handle.tv_sec - timeline->start.tv_sec) * - 1000.0 + - (timeline->get_ipc_handle.tv_nsec - timeline->start.tv_nsec) / - 1000000.0; - event_time = - (timeline->event_alloc.tv_sec - timeline->get_ipc_handle.tv_sec) * - 1000.0 + - (timeline->event_alloc.tv_nsec - timeline->get_ipc_handle.tv_nsec) / - 1000000.0; - copy_time = (timeline->async_copy.tv_sec - timeline->event_alloc.tv_sec) * - 1000.0 + - (timeline->async_copy.tv_nsec - timeline->event_alloc.tv_nsec) / - 1000000.0; - other_time = (timeline->end.tv_sec - timeline->async_copy.tv_sec) * 1000.0 + - (timeline->end.tv_nsec - timeline->async_copy.tv_nsec) / - 1000000.0; - total_time = (timeline->end.tv_sec - timeline->start.tv_sec) * 1000.0 + - (timeline->end.tv_nsec - timeline->start.tv_nsec) / 1000000.0; - - ucs_trace("ze_ipc timeline: total=%.3fms (ipc=%.3fms event=%.3fms " - "copy=%.3fms other=%.3fms) " - "size=%zu cmd_list=%u", - total_time, ipc_time, event_time, copy_time, other_time, length, - cmd_list_idx); -} - -#define UCT_ZE_IPC_TIMELINE_INIT(_tl) uct_ze_ipc_timeline_t _tl = {{0}} - -#define UCT_ZE_IPC_TIMELINE_RECORD(_tl, _field) \ - if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE)) { \ - uct_ze_ipc_timeline_record(&(_tl)._field); \ - } - -#define UCT_ZE_IPC_TIMELINE_REPORT(_tl, _len, _idx) \ - if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE)) { \ - uct_ze_ipc_timeline_report(&(_tl), (_len), (_idx)); \ - } - - int uct_ze_ipc_ep_is_connected(const uct_ep_h tl_ep, const uct_ep_is_connected_params_t *params) { @@ -186,8 +123,6 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, uct_ze_ipc_key_t *key = (uct_ze_ipc_key_t*)rkey; uct_ze_ipc_event_desc_t *event_desc; uct_ze_ipc_queue_desc_t *q_desc; - ze_event_handle_t event = NULL; - ze_device_handle_t remote_device; void *mapped_addr = NULL; void *mapped_rem_addr; void *dst, *src; @@ -195,11 +130,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, ze_result_t ret; ucs_status_t status; int local_fd; - int event_index = -1; unsigned cmd_list_idx; - UCT_ZE_IPC_TIMELINE_INIT(timeline); - - UCT_ZE_IPC_TIMELINE_RECORD(timeline, start); if (ucs_unlikely(iov[0].length == 0)) { ucs_trace_data("Zero length request: skip it"); @@ -216,55 +147,40 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, return UCS_ERR_INVALID_PARAM; } - /* Resolve the local device matching the remote allocation's device - * ordinal - on multi-GPU nodes this may differ from the iface's own - * device */ - remote_device = uct_ze_base_get_device(key->dev_num); - if (remote_device == NULL) { - ucs_error("ze_ipc_ep: failed to get local device for ordinal %d", - key->dev_num); - return UCS_ERR_NO_DEVICE; - } - - /* Use cache to map IPC handle */ - status = uct_ze_ipc_map_memhandle(key, iface->ze_context, remote_device, + /* Open the handle on the device whose command lists perform the copy; + * the exporter's device ordinal need not name the same device here */ + status = uct_ze_ipc_map_memhandle(key, iface->ze_context, iface->ze_device, NULL, &mapped_addr, &local_fd); if (status != UCS_OK) { ucs_error("ze_ipc_ep: uct_ze_ipc_map_memhandle failed"); return status; } - UCT_ZE_IPC_TIMELINE_RECORD(timeline, get_ipc_handle); ucs_debug("ze_ipc_ep: IPC handle mapped (cached), mapped_addr=%p", mapped_addr); mapped_rem_addr = (void*)((uintptr_t)mapped_addr + offset); - /* Allocate event from shared pool for performance */ - event_index = uct_ze_ipc_alloc_event(iface, &event); - if (event_index < 0) { - ucs_error("failed to allocate event from shared pool"); - uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, - iface->ze_context, local_fd, - iface->config.enable_cache); - return UCS_ERR_NO_RESOURCE; - } - /* Allocate event descriptor */ event_desc = ucs_malloc(sizeof(*event_desc), "uct_ze_ipc_event_desc_t"); if (event_desc == NULL) { ucs_error("failed to allocate event descriptor"); - uct_ze_ipc_free_event(iface, event, event_index); uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, iface->ze_context, local_fd, iface->config.enable_cache); return UCS_ERR_NO_MEMORY; } + status = uct_ze_ipc_alloc_event(iface, event_desc); + if (status != UCS_OK) { + ucs_free(event_desc); + uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, + iface->ze_context, local_fd, + iface->config.enable_cache); + return status; + } + /* Store information for cache-based cleanup and event tracking */ - event_desc->event = event; - event_desc->event_pool = NULL; /* Using shared pool, not private */ - event_desc->event_index = event_index; event_desc->dup_fd = local_fd; event_desc->pid = ep->remote_pid; event_desc->address = key->address; @@ -278,8 +194,6 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, src = mapped_rem_addr; } - UCT_ZE_IPC_TIMELINE_RECORD(timeline, event_alloc); - /* * Select command list using round-robin scheduling * Similar to CUDA IPC's stream selection: key->stream_id % iface->config.max_streams @@ -301,14 +215,13 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, * - Supporting multiple outstanding operations */ ret = zeCommandListAppendMemoryCopy(q_desc->cmd_list, dst, src, - iov[0].length, event, 0, NULL); + iov[0].length, event_desc->event, 0, + NULL); if (ret != ZE_RESULT_SUCCESS) { ucs_error("zeCommandListAppendMemoryCopy failed with error 0x%x", ret); goto err_cleanup; } - UCT_ZE_IPC_TIMELINE_RECORD(timeline, async_copy); - /* Store event info for progress tracking */ event_desc->mapped_addr = mapped_addr; event_desc->comp = comp; @@ -321,18 +234,15 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, /* Push event to this command list's event queue */ ucs_queue_push(&q_desc->event_queue, &event_desc->queue); - UCT_ZE_IPC_TIMELINE_RECORD(timeline, end); - ucs_trace("zeCommandListAppendMemoryCopy issued (async): " "cmd_list[%u/%u]=%p dst=%p src=%p len=%zu", cmd_list_idx, iface->num_cmd_lists, q_desc->cmd_list, dst, src, iov[0].length); - UCT_ZE_IPC_TIMELINE_REPORT(timeline, iov[0].length, cmd_list_idx); return UCS_INPROGRESS; err_cleanup: - uct_ze_ipc_free_event(iface, event, event_index); + uct_ze_ipc_free_event(iface, event_desc); ucs_free(event_desc); uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, iface->ze_context, local_fd, diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index 51a53e6cf29..7d306bf8d8c 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -38,6 +38,12 @@ static ucs_config_field_t uct_ze_ipc_iface_config_table[] = { ucs_offsetof(uct_ze_ipc_iface_config_t, enable_cache), UCS_CONFIG_TYPE_BOOL}, + {"EVENT_POOL_SIZE", "1024", + "Number of events in the shared event pool. Operations in flight beyond " + "this number use a slower per-operation event pool", + ucs_offsetof(uct_ze_ipc_iface_config_t, event_pool_size), + UCS_CONFIG_TYPE_UINT}, + {"BW", "50000MBs", "Effective p2p memory bandwidth", ucs_offsetof(uct_ze_ipc_iface_config_t, bandwidth), UCS_CONFIG_TYPE_BW}, @@ -116,9 +122,6 @@ uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) UCT_IFACE_FLAG_PENDING | UCT_IFACE_FLAG_GET_ZCOPY | UCT_IFACE_FLAG_PUT_ZCOPY; - iface_attr->cap.event_flags = UCT_IFACE_FLAG_EVENT_SEND_COMP | - UCT_IFACE_FLAG_EVENT_RECV; - iface_attr->cap.put.max_short = 0; iface_attr->cap.put.max_bcopy = 0; iface_attr->cap.put.min_zcopy = 0; @@ -144,17 +147,32 @@ uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) } -/** - * Allocate an event from the shared event pool - * Returns event index on success, -1 on failure - */ -int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, - ze_event_handle_t *event_p) +static ucs_status_t uct_ze_ipc_create_event(ze_event_pool_handle_t event_pool, + unsigned index, + ze_event_handle_t *event_p) { + ze_event_desc_t event_desc = { + .stype = ZE_STRUCTURE_TYPE_EVENT_DESC, + .index = index, + .signal = ZE_EVENT_SCOPE_FLAG_HOST, + .wait = ZE_EVENT_SCOPE_FLAG_HOST + }; + + return UCT_ZE_FUNC_LOG_ERR(zeEventCreate(event_pool, &event_desc, event_p)); +} + + +ucs_status_t uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc) +{ + ze_event_pool_desc_t pool_desc = { + .stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, + .flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE, + .count = 1 + }; unsigned i, word_idx, bit_idx; uint64_t mask; - ze_event_desc_t event_desc; - ze_result_t ret; + ucs_status_t status; ucs_spin_lock(&iface->event_lock); @@ -169,54 +187,69 @@ int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, iface->event_bitmap[word_idx] |= mask; ucs_spin_unlock(&iface->event_lock); - /* Create event from shared pool */ - event_desc.stype = ZE_STRUCTURE_TYPE_EVENT_DESC; - event_desc.pNext = NULL; - event_desc.index = i; - event_desc.signal = ZE_EVENT_SCOPE_FLAG_HOST; - event_desc.wait = ZE_EVENT_SCOPE_FLAG_HOST; - - ret = zeEventCreate(iface->ze_event_pool, &event_desc, event_p); - if (ret != ZE_RESULT_SUCCESS) { - ucs_error("zeEventCreate failed with error 0x%x", ret); + status = uct_ze_ipc_create_event(iface->ze_event_pool, i, + &event_desc->event); + if (status != UCS_OK) { /* Mark as free again */ ucs_spin_lock(&iface->event_lock); iface->event_bitmap[word_idx] &= ~mask; ucs_spin_unlock(&iface->event_lock); - return -1; + return status; } - return i; + event_desc->event_pool = NULL; + event_desc->event_index = i; + return UCS_OK; } } ucs_spin_unlock(&iface->event_lock); - ucs_warn("ze_ipc: event pool exhausted (size=%u)", iface->event_pool_size); - return -1; + + /* ep_pending_add cannot queue the operation, so UCS_ERR_NO_RESOURCE here + * would make UCP retry the send in a tight loop */ + ucs_trace("ze_ipc: event pool of %u events exhausted, using a private " + "event pool", + iface->event_pool_size); + + status = UCT_ZE_FUNC_LOG_ERR( + zeEventPoolCreate(iface->ze_context, &pool_desc, 1, + &iface->ze_device, &event_desc->event_pool)); + if (status != UCS_OK) { + return status; + } + + status = uct_ze_ipc_create_event(event_desc->event_pool, 0, + &event_desc->event); + if (status != UCS_OK) { + zeEventPoolDestroy(event_desc->event_pool); + return status; + } + + event_desc->event_index = (unsigned)-1; + return UCS_OK; } -/** - * Free an event back to the shared event pool - */ -void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t event, - unsigned event_index) + +void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc) { unsigned word_idx, bit_idx; uint64_t mask; - if (event != NULL) { - zeEventDestroy(event); + zeEventDestroy(event_desc->event); + + if (event_desc->event_pool != NULL) { + zeEventPoolDestroy(event_desc->event_pool); + return; } - if (event_index != (unsigned)-1) { - word_idx = event_index / 64; - bit_idx = event_index % 64; - mask = 1ULL << bit_idx; + word_idx = event_desc->event_index / 64; + bit_idx = event_desc->event_index % 64; + mask = 1ULL << bit_idx; - ucs_spin_lock(&iface->event_lock); - iface->event_bitmap[word_idx] &= ~mask; - ucs_spin_unlock(&iface->event_lock); - } + ucs_spin_lock(&iface->event_lock); + iface->event_bitmap[word_idx] &= ~mask; + ucs_spin_unlock(&iface->event_lock); } static unsigned uct_ze_ipc_iface_progress(uct_iface_h tl_iface) @@ -280,20 +313,7 @@ static unsigned uct_ze_ipc_iface_progress(uct_iface_h tl_iface) UCS_ERR_IO_ERROR); } - /* Free event back to shared pool or destroy private pool */ - if (event_desc->event_index != (unsigned)-1) { - /* Using shared event pool */ - uct_ze_ipc_free_event(iface, event_desc->event, - event_desc->event_index); - } else { - /* Using private event pool (backward compatibility) */ - if (event_desc->event != NULL) { - zeEventDestroy(event_desc->event); - } - if (event_desc->event_pool != NULL) { - zeEventPoolDestroy(event_desc->event_pool); - } - } + uct_ze_ipc_free_event(iface, event_desc); ucs_free(event_desc); count++; @@ -523,24 +543,21 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, if (num_queues == 0) { /* No copy engines available, use single command list on compute engine */ self->num_cmd_lists = 1; - ucs_info("ze_ipc_iface: no dedicated copy engines found, using 1 " - "command list " - "on queue group %u (compute engine fallback)", - copy_ordinal); + ucs_debug("ze_ipc_iface: no dedicated copy engines found, using 1 " + "command list on queue group %u (compute engine fallback)", + copy_ordinal); } else { /* Match command list count to available Copy Engine queues */ self->num_cmd_lists = ucs_min(num_queues, config->max_cmd_lists); if (self->num_cmd_lists < num_queues) { - ucs_info("ze_ipc_iface: limiting command lists to %u (hardware has " - "%u copy queues, " - "config limit is %u)", - self->num_cmd_lists, num_queues, config->max_cmd_lists); + ucs_debug("ze_ipc_iface: limiting command lists to %u (hardware " + "has %u copy queues, config limit is %u)", + self->num_cmd_lists, num_queues, config->max_cmd_lists); } else { - ucs_info("ze_ipc_iface: creating %u command list(s) to match %u " - "copy engine queue(s) " - "on queue group %u", - self->num_cmd_lists, num_queues, copy_ordinal); + ucs_debug("ze_ipc_iface: creating %u command list(s) to match %u " + "copy engine queue(s) on queue group %u", + self->num_cmd_lists, num_queues, copy_ordinal); } } @@ -556,6 +573,10 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, self->num_cmd_lists, UCT_ZE_IPC_MAX_PEERS); return UCS_ERR_INVALID_PARAM; } + if (config->event_pool_size == 0) { + ucs_error("ze_ipc_iface: EVENT_POOL_SIZE must be at least 1"); + return UCS_ERR_INVALID_PARAM; + } /* Initialize active queue for command lists with pending operations */ ucs_queue_head_init(&self->active_queue); @@ -614,7 +635,7 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, * - Event reuse via bitmap tracking * - Typical pool size: 1024 events (configurable) */ - self->event_pool_size = 1024; /* TODO: make this configurable */ + self->event_pool_size = config->event_pool_size; event_pool_desc.stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC; event_pool_desc.pNext = NULL; @@ -659,10 +680,10 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, return status; } - ucs_info("ze_ipc_iface: initialized iface for device %p context %p " - "(pid=%d, copy_ordinal=%u, num_cmd_lists=%u, event_pool_size=%u)", - self->ze_device, self->ze_context, getpid(), copy_ordinal, - self->num_cmd_lists, self->event_pool_size); + ucs_debug("ze_ipc_iface: initialized iface for device %p context %p pid %d " + "copy ordinal %u, %u command lists, %u events", + self->ze_device, self->ze_context, getpid(), copy_ordinal, + self->num_cmd_lists, self->event_pool_size); return UCS_OK; } diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.h b/src/uct/ze/ze_ipc/ze_ipc_iface.h index 50f2ee39468..190d61f090f 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.h +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.h @@ -23,6 +23,7 @@ typedef struct uct_ze_ipc_iface_config { unsigned max_poll; /* query attempts w.o success */ unsigned max_cmd_lists; /* max number of command lists for parallel progress */ int enable_cache; /* enable/disable ipc handle cache */ + unsigned event_pool_size; /* number of events in the shared event pool */ double bandwidth; /* estimated bandwidth */ double latency; /* estimated latency */ double overhead; /* estimated CPU overhead */ @@ -76,24 +77,24 @@ typedef struct uct_ze_ipc_event_desc { /** - * Allocate an event from the shared event pool + * Allocate an event from the shared event pool, or from a private one-event + * pool when the shared pool is exhausted * * @param iface Pointer to ze_ipc interface - * @param event_p Pointer to store the allocated event handle - * @return Event index on success, -1 on failure + * @param event_desc Descriptor to fill with the event, its pool and index + * @return UCS_OK on success, error status on Level Zero failure */ -int uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, - ze_event_handle_t *event_p); +ucs_status_t uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc); /** - * Free an event back to the shared event pool + * Release an event allocated by uct_ze_ipc_alloc_event * - * @param iface Pointer to ze_ipc interface - * @param event Event handle to destroy - * @param event_index Index of the event in the pool + * @param iface Pointer to ze_ipc interface + * @param event_desc Descriptor filled by uct_ze_ipc_alloc_event */ -void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, ze_event_handle_t event, - unsigned event_index); +void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc); #endif diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index f7c6d0d325e..1fa8675e8a6 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -22,38 +22,6 @@ #include -/* Helper function to compute a simple checksum of IPC handle for verification */ -static uint32_t uct_ze_ipc_handle_checksum(const ze_ipc_mem_handle_t *handle) -{ - const unsigned char *bytes = (const unsigned char*)handle; - uint32_t sum = 0; - for (size_t i = 0; i < sizeof(ze_ipc_mem_handle_t); i++) { - sum += bytes[i]; - sum = (sum << 1) | (sum >> 31); /* rotate left */ - } - return sum; -} - - -/* Helper function to print IPC handle bytes for debugging */ -static void uct_ze_ipc_print_handle(const char *prefix, - const ze_ipc_mem_handle_t *handle, - uintptr_t addr, size_t len, int dev_num) -{ - const unsigned char *bytes = (const unsigned char*)handle; - uint32_t checksum = uct_ze_ipc_handle_checksum(handle); - - /* Print first 16 bytes of handle + checksum for quick comparison */ - ucs_info("%s: addr=0x%lx len=%zu dev=%d checksum=0x%08x " - "handle[0-15]=%02x%02x%02x%02x%02x%02x%02x%02x" - "%02x%02x%02x%02x%02x%02x%02x%02x", - prefix, (unsigned long)addr, len, dev_num, checksum, bytes[0], - bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], - bytes[7], bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], - bytes[13], bytes[14], bytes[15]); -} - - static ucs_config_field_t uct_ze_ipc_md_config_table[] = { {"", "", NULL, ucs_offsetof(uct_ze_ipc_md_config_t, super), UCS_CONFIG_TYPE_TABLE(uct_md_config_table)}, @@ -85,11 +53,6 @@ uct_ze_ipc_mkey_pack(uct_md_h uct_md, uct_mem_h memh, void *address, uct_ze_ipc_key_t *key = memh; *packed = *key; - - /* Print handle info for verification - compare with rkey_unpack output */ - uct_ze_ipc_print_handle("PACK(sender)", &packed->ipc_handle, - packed->address, packed->length, packed->dev_num); - return UCS_OK; } @@ -100,31 +63,19 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, ze_memory_allocation_properties_t props = { .stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES }; - ze_device_handle_t alloc_device = NULL; void *base_address; size_t alloc_size; unsigned long proc_create_time; ze_result_t ret; ucs_status_t status; - int dev_ordinal; - /* Get memory allocation properties to verify this is ZE device memory - * Also get the device where memory was allocated */ - ret = zeMemGetAllocProperties(md->ze_context, address, &props, - &alloc_device); + /* Get memory allocation properties to verify this is ZE device memory */ + ret = zeMemGetAllocProperties(md->ze_context, address, &props, NULL); if ((ret != ZE_RESULT_SUCCESS) || (props.type == ZE_MEMORY_TYPE_UNKNOWN)) { ucs_error("failed to get allocation properties for %p", address); return UCS_ERR_INVALID_ADDR; } - /* Get device ordinal from the device handle */ - dev_ordinal = uct_ze_base_get_device_ordinal(alloc_device); - if (dev_ordinal < 0) { - ucs_error("failed to get device ordinal for device %p (ptr=%p)", - (void*)alloc_device, address); - return UCS_ERR_INVALID_ADDR; - } - /* Get the base address and allocation size */ ret = zeMemGetAddressRange(md->ze_context, address, &base_address, &alloc_size); @@ -153,12 +104,11 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, key->pid = getpid(); key->address = (uintptr_t)base_address; key->length = alloc_size; - key->dev_num = dev_ordinal; key->alloc_id = props.id; key->proc_create_time = proc_create_time; - ucs_trace("packed IPC handle for %p base=%p len=%zu dev=%d pid=%d", address, - base_address, alloc_size, dev_ordinal, key->pid); + ucs_trace("packed ipc handle for %p base %p length %zu pid %d", address, + base_address, alloc_size, key->pid); return UCS_OK; } @@ -172,9 +122,6 @@ uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, uct_ze_ipc_key_t *key; ucs_status_t status; - ucs_info("ze_ipc_md: mem_reg called address=%p length=%zu", address, - length); - key = ucs_malloc(sizeof(*key), "uct_ze_ipc_key_t"); if (key == NULL) { ucs_error("ze_ipc_md: failed to allocate memory for uct_ze_ipc_key_t"); @@ -189,10 +136,6 @@ uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, return status; } - ucs_info("ze_ipc_md: mem_reg succeeded address=%p key->address=0x%lx " - "key->length=%zu", - address, (unsigned long)key->address, key->length); - *memh_p = key; return UCS_OK; } @@ -227,10 +170,6 @@ uct_ze_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, uct_ze_ipc_key_t *packed = (uct_ze_ipc_key_t*)rkey_buffer; uct_ze_ipc_key_t *key; - /* Print handle info for verification - compare with mkey_pack output */ - uct_ze_ipc_print_handle("UNPACK(receiver)", &packed->ipc_handle, - packed->address, packed->length, packed->dev_num); - key = ucs_malloc(sizeof(uct_ze_ipc_key_t), "uct_ze_ipc_key_t"); if (key == NULL) { ucs_error("ze_ipc_md: failed to allocate memory for uct_ze_ipc_key_t"); diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.h b/src/uct/ze/ze_ipc/ze_ipc_md.h index 1d6bdc48374..209308c3cc1 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.h +++ b/src/uct/ze/ze_ipc/ze_ipc_md.h @@ -45,7 +45,6 @@ typedef struct uct_ze_ipc_key { pid_t pid; /**< Remote process ID (for cache) */ uintptr_t address; /**< Base address of the allocation */ size_t length; /**< Size of the allocation */ - int dev_num; /**< GPU device number */ } uct_ze_ipc_key_t; diff --git a/test/gtest/uct/ze/test_ze_base.cc b/test/gtest/uct/ze/test_ze_base.cc index 112a8185e7d..0901d16cba8 100644 --- a/test/gtest/uct/ze/test_ze_base.cc +++ b/test/gtest/uct/ze/test_ze_base.cc @@ -38,7 +38,7 @@ UCS_TEST_F(test_ze_base, get_num_devices_consistent) { } -UCS_TEST_F(test_ze_base, get_device_by_ordinal_round_trip) { +UCS_TEST_F(test_ze_base, get_device_by_ordinal) { if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { UCS_TEST_SKIP_R("Level Zero runtime not available"); } @@ -49,11 +49,7 @@ UCS_TEST_F(test_ze_base, get_device_by_ordinal_round_trip) { } for (int i = 0; i < n; ++i) { - ze_device_handle_t dev = uct_ze_base_get_device(i); - ASSERT_TRUE(dev != NULL) << "ordinal " << i; - - /* round-trip: handle -> ordinal must give same index back */ - EXPECT_EQ(i, uct_ze_base_get_device_ordinal(dev)); + EXPECT_TRUE(uct_ze_base_get_device(i) != NULL) << "ordinal " << i; } } @@ -69,16 +65,3 @@ UCS_TEST_F(test_ze_base, get_device_out_of_range) { EXPECT_TRUE(uct_ze_base_get_device(n) == NULL); EXPECT_TRUE(uct_ze_base_get_device(n + 100) == NULL); } - - -UCS_TEST_F(test_ze_base, get_device_ordinal_unknown_handle) { - if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { - UCS_TEST_SKIP_R("Level Zero runtime not available"); - } - - /* Bogus pointer that cannot be a registered device handle. */ - ze_device_handle_t bogus = reinterpret_cast( - uintptr_t(0xdead)); - EXPECT_EQ(-1, uct_ze_base_get_device_ordinal(bogus)); - EXPECT_EQ(-1, uct_ze_base_get_device_ordinal(NULL)); -} From 407266f1a456296612a69a2122f769f1a4311d5a Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Fri, 2 Oct 2026 22:55:32 -0700 Subject: [PATCH 32/35] TEST/UCT/ZE: Add cross-process ze_ipc RMA test The existing ze_ipc RMA tests keep both entities in one process, so the transport takes its same-pid path and never exercises pidfd_getfd() or the cross-process reachability check. Fork and exec the gtest binary as an importer that unpacks the exporter's rkey and runs put_zcopy and get_zcopy against the exporter's buffer for several rounds, with and without the IPC cache. The exporter checks each put and fills the buffer for the following get. Signed-off-by: Tushar Gohad --- test/gtest/uct/ze/test_ze_ipc_rma.cc | 342 ++++++++++++++++++++++++++- 1 file changed, 337 insertions(+), 5 deletions(-) diff --git a/test/gtest/uct/ze/test_ze_ipc_rma.cc b/test/gtest/uct/ze/test_ze_ipc_rma.cc index 460313c7662..7cf4b3fd915 100644 --- a/test/gtest/uct/ze/test_ze_ipc_rma.cc +++ b/test/gtest/uct/ze/test_ze_ipc_rma.cc @@ -7,15 +7,20 @@ extern "C" { #include +#include } +#include +#include +#include +#include +#include + /* - * Endpoint-level RMA tests for ze_ipc. Both entities live in this process, - * so the transport takes its same-pid loopback path (no pidfd_getfd fd - * duplication), but the iface open/connect/put_zcopy/get_zcopy/flush and - * cache map/unmap paths are exercised the same way they would be across - * processes. + * Endpoint-level RMA tests for ze_ipc with both entities in this process. The + * transport takes its same-pid path, so these tests skip the pidfd_getfd fd + * duplication and the reachability check; test_ze_ipc_rma_xproc covers them. */ class test_ze_ipc_rma : public uct_test { protected: @@ -115,3 +120,330 @@ UCS_TEST_P(test_ze_ipc_rma, put_zcopy_rejects_out_of_range_rkey) } _UCT_INSTANTIATE_TEST_CASE(test_ze_ipc_rma, ze_ipc) + + +/* + * Cross-process RMA. The test process exports a device buffer, and an exec'd + * copy of this gtest binary running the same test imports it and issues PUT + * and GET. Only the control socket survives the exec, so the importer can + * reach the exporter's dmabuf fd only through pidfd_getfd. + */ +class test_ze_ipc_rma_xproc : + private ucs::clear_dontcopy_regions, + public uct_test { +protected: + enum { + MSG_PUT_DONE, + MSG_GET_READY, + MSG_DONE + }; + + struct op_msg { + uint32_t op; + uint32_t round; + }; + + struct exporter_info { + pid_t pid; + uint64_t address; + uint32_t iface_addr_len; + uint32_t dev_addr_len; + uint32_t rkey_len; + }; + + struct peer_process { + pid_t pid = -1; + int fd = -1; + + ~peer_process() + { + if (fd >= 0) { + close(fd); + } + if (pid > 0) { + kill(pid, SIGKILL); + waitpid(pid, NULL, 0); + } + } + }; + + static const char *PEER_FD_ENV; + static const int PEER_FD = 3; + static const int TIMEOUT_MS = 60000; + static const unsigned NUM_ROUNDS = 3; + static const size_t LENGTH = UCS_MBYTE; + + static uint64_t put_seed(unsigned round) + { + return 0x1000 + round; + } + + static uint64_t get_seed(unsigned round) + { + return 0x2000 + round; + } + + static void send_msg(int fd, const void *buffer, size_t length) + { + ssize_t ret = send(fd, buffer, length, MSG_NOSIGNAL); + + if (ret != (ssize_t)length) { + UCS_TEST_ABORT("send() returned " << ret << ": " + << strerror(errno)); + } + } + + static size_t recv_msg(int fd, void *buffer, size_t max_length) + { + struct pollfd pfd = {fd, POLLIN, 0}; + ssize_t ret; + + do { + ret = poll(&pfd, 1, TIMEOUT_MS * ucs::test_time_multiplier()); + } while ((ret < 0) && (errno == EINTR)); + + if (ret != 1) { + UCS_TEST_ABORT("no message from peer process: poll() returned " + << ret); + } + + ret = recv(fd, buffer, max_length, 0); + if (ret <= 0) { + UCS_TEST_ABORT("peer process closed the socket: recv() returned " + << ret); + } + + return ret; + } + + static void send_op(int fd, uint32_t op, uint32_t round) + { + op_msg msg = {op, round}; + + send_msg(fd, &msg, sizeof(msg)); + } + + static void recv_op(int fd, uint32_t op, uint32_t round) + { + op_msg msg = {}; + size_t length; + + length = recv_msg(fd, &msg, sizeof(msg)); + if ((length != sizeof(msg)) || (msg.op != op) || (msg.round != round)) { + UCS_TEST_ABORT("got message op " << msg.op << " round " << msg.round + << ", expected op " << op + << " round " << round); + } + } + + void pack_exporter_info(const entity &e, const mapped_buffer &buffer, + std::vector &info) + { + exporter_info hdr; + uct_md_mkey_pack_params_t pack_params = {}; + + hdr.pid = getpid(); + hdr.address = buffer.addr(); + hdr.iface_addr_len = e.iface_attr().iface_addr_len; + hdr.dev_addr_len = e.iface_attr().device_addr_len; + hdr.rkey_len = e.md_attr().rkey_packed_size; + + info.resize(sizeof(hdr) + hdr.iface_addr_len + hdr.dev_addr_len + + hdr.rkey_len); + memcpy(&info[0], &hdr, sizeof(hdr)); + + char *iface_addr = &info[sizeof(hdr)]; + char *dev_addr = iface_addr + hdr.iface_addr_len; + char *rkey_buf = dev_addr + hdr.dev_addr_len; + + ASSERT_UCS_OK(uct_iface_get_address(e.iface(), + (uct_iface_addr_t*)iface_addr)); + ASSERT_UCS_OK( + uct_iface_get_device_address(e.iface(), + (uct_device_addr_t*)dev_addr)); + ASSERT_UCS_OK(uct_md_mkey_pack_v2(e.md(), buffer.memh(), buffer.ptr(), + buffer.length(), &pack_params, + rkey_buf)); + } + + void start_importer(peer_process &peer) + { + const ::testing::TestInfo *test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + std::string filter = std::string("--gtest_filter=") + + test_info->test_suite_name() + "." + + test_info->name(); + std::string repeat = "--gtest_repeat=1"; + std::string exe = "/proc/self/exe"; + std::string fd_env = std::string(PEER_FD_ENV) + "=" + + std::to_string(PEER_FD); + std::vector argv = {&exe[0], &filter[0], &repeat[0], NULL}; + std::vector envp; + std::vector close_fds; + int fds[2]; + + /* Sharding would make the child skip its only test and exit 0 */ + for (char **var = environ; *var != NULL; ++var) { + if (strncmp(*var, "GTEST_TOTAL_SHARDS=", 19) && + strncmp(*var, "GTEST_SHARD_INDEX=", 18)) { + envp.push_back(*var); + } + } + envp.push_back(&fd_env[0]); + envp.push_back(NULL); + + ASSERT_EQ(0, socketpair(AF_UNIX, SOCK_SEQPACKET, 0, fds)); + for (int fd : ucs::get_open_fds()) { + if (fd > PEER_FD) { + close_fds.push_back(fd); + } + } + + peer.pid = fork(); + if (peer.pid == 0) { + /* Async-signal-safe calls only until execve() */ + if (dup2(fds[1], PEER_FD) == PEER_FD) { + for (int fd : close_fds) { + close(fd); + } + execve(exe.c_str(), argv.data(), envp.data()); + } + _exit(127); + } + + close(fds[1]); + peer.fd = fds[0]; + ASSERT_GT(peer.pid, 0) << "fork() failed: " << strerror(errno); + } + + void run_exporter() + { + entity *e = uct_test::create_entity(0); + m_entities.push_back(e); + + mapped_buffer buffer(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); + std::vector info; + peer_process peer; + int status; + + ASSERT_NO_FATAL_FAILURE(pack_exporter_info(*e, buffer, info)); + ASSERT_NO_FATAL_FAILURE(start_importer(peer)); + send_msg(peer.fd, info.data(), info.size()); + + for (unsigned round = 0; round < NUM_ROUNDS; ++round) { + recv_op(peer.fd, MSG_PUT_DONE, round); + buffer.pattern_check(put_seed(round)); + buffer.pattern_fill(get_seed(round)); + send_op(peer.fd, MSG_GET_READY, round); + } + recv_op(peer.fd, MSG_DONE, NUM_ROUNDS); + + ASSERT_EQ(peer.pid, waitpid(peer.pid, &status, 0)); + peer.pid = -1; + EXPECT_TRUE(WIFEXITED(status) && (WEXITSTATUS(status) == 0)) + << "importer exit status " << status; + } + + void run_importer(int fd) + { + std::vector info(UCS_KBYTE * 4); + uct_iface_is_reachable_params_t params = {}; + uct_ep_params_t ep_params = {}; + ucs::handle ep; + uct_rkey_bundle_t rkey; + exporter_info hdr; + char info_str[256]; + size_t length; + + length = recv_msg(fd, info.data(), info.size()); + ASSERT_GE(length, sizeof(hdr)); + memcpy(&hdr, info.data(), sizeof(hdr)); + ASSERT_EQ(sizeof(hdr) + hdr.iface_addr_len + hdr.dev_addr_len + + hdr.rkey_len, + length); + + const void *iface_addr = &info[sizeof(hdr)]; + const void *dev_addr = &info[sizeof(hdr) + hdr.iface_addr_len]; + const void *rkey_buf = + &info[sizeof(hdr) + hdr.iface_addr_len + hdr.dev_addr_len]; + + entity *e = uct_test::create_entity(0); + m_entities.push_back(e); + + ASSERT_NE(getpid(), hdr.pid); + ASSERT_EQ(e->iface_attr().iface_addr_len, hdr.iface_addr_len); + ASSERT_EQ(e->iface_attr().device_addr_len, hdr.dev_addr_len); + ASSERT_EQ(hdr.pid, *(const pid_t*)iface_addr); + + params.field_mask = UCT_IFACE_IS_REACHABLE_FIELD_DEVICE_ADDR | + UCT_IFACE_IS_REACHABLE_FIELD_IFACE_ADDR | + UCT_IFACE_IS_REACHABLE_FIELD_INFO_STRING | + UCT_IFACE_IS_REACHABLE_FIELD_INFO_STRING_LENGTH; + params.device_addr = (const uct_device_addr_t*)dev_addr; + params.iface_addr = (const uct_iface_addr_t*)iface_addr; + params.info_string = info_str; + params.info_string_length = sizeof(info_str); + info_str[0] = '\0'; + ASSERT_TRUE(uct_iface_is_reachable_v2(e->iface(), ¶ms)) << info_str; + + ep_params.field_mask = UCT_EP_PARAM_FIELD_IFACE | + UCT_EP_PARAM_FIELD_DEV_ADDR | + UCT_EP_PARAM_FIELD_IFACE_ADDR; + ep_params.iface = e->iface(); + ep_params.dev_addr = (const uct_device_addr_t*)dev_addr; + ep_params.iface_addr = (const uct_iface_addr_t*)iface_addr; + UCS_TEST_CREATE_HANDLE(uct_ep_h, ep, uct_ep_destroy, uct_ep_create, + &ep_params); + + ASSERT_UCS_OK(uct_rkey_unpack(GetParam()->component, rkey_buf, &rkey)); + ASSERT_EQ(hdr.pid, ((const uct_ze_ipc_key_t*)rkey.rkey)->pid); + + mapped_buffer sendbuf(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); + + for (unsigned round = 0; round < NUM_ROUNDS; ++round) { + sendbuf.pattern_fill(put_seed(round)); + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_put_zcopy(ep, sendbuf.iov(), 1, + hdr.address, rkey.rkey, + NULL)); + e->flush(); + send_op(fd, MSG_PUT_DONE, round); + + recv_op(fd, MSG_GET_READY, round); + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_get_zcopy(ep, recvbuf.iov(), 1, + hdr.address, rkey.rkey, + NULL)); + e->flush(); + recvbuf.pattern_check(get_seed(round)); + } + + ASSERT_UCS_OK(uct_rkey_release(GetParam()->component, &rkey)); + send_op(fd, MSG_DONE, NUM_ROUNDS); + } + + void test_put_get() + { + const char *peer_fd = getenv(PEER_FD_ENV); + + if (peer_fd == NULL) { + run_exporter(); + } else { + run_importer(atoi(peer_fd)); + } + } +}; + +const char *test_ze_ipc_rma_xproc::PEER_FD_ENV = "UCT_ZE_IPC_TEST_PEER_FD"; + + +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get) +{ + test_put_get(); +} + +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_no_cache, "ZE_IPC_ENABLE_CACHE=n") +{ + test_put_get(); +} + +_UCT_INSTANTIATE_TEST_CASE(test_ze_ipc_rma_xproc, ze_ipc) From 2188836b9c0dd2f3e668a7cc87fd19db728c779e Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Fri, 2 Oct 2026 22:55:51 -0700 Subject: [PATCH 33/35] UCT/ZE: Put the IPC handle at registration and pack a dup of its fd While a zeMemGetIpcHandle() handle is held, the Level Zero driver (seen on libze-intel-gpu1 26.27.39122.11) closes the handle's dmabuf fd on the first command-list operation that a device other than the allocation's device issues on the buffer, and a later zeMemPutIpcHandle() closes the same fd number again. ze_ipc packed that fd number into the rkey and held the handle until dereg, so a peer's pidfd_getfd() failed with EBADF or duplicated an unrelated file that had reused the number, and the deferred put could close that file in our process. Duplicate the fd right after getting the handle, return the handle at once and pack the duplicate; dereg closes it. The duplicate keeps the dmabuf alive for importers, and no driver handle stays outstanding for the driver to close under us. Add a cross-process test that fills the exported buffer from a second device before the importer opens it. It is skipped when fewer than two devices are visible. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_md.c | 31 ++++++++++++++----- test/gtest/uct/ze/test_ze_ipc_rma.cc | 46 ++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index 1fa8675e8a6..60be56c3a2b 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -68,6 +69,7 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, unsigned long proc_create_time; ze_result_t ret; ucs_status_t status; + int fd; /* Get memory allocation properties to verify this is ZE device memory */ ret = zeMemGetAllocProperties(md->ze_context, address, &props, NULL); @@ -101,6 +103,26 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, return status; } + /* Peers get our own duplicate of the fd and the handle goes back at once: + * while a handle is held, the driver may close its fd when another device + * accesses the allocation, and then close that fd number again on put */ + fd = fcntl(*(int*)key->ipc_handle.data, F_DUPFD_CLOEXEC, 0); + if (fd < 0) { + ucs_error("failed to duplicate IPC handle fd %d for %p: %m", + *(int*)key->ipc_handle.data, address); + } + + ret = zeMemPutIpcHandle(md->ze_context, key->ipc_handle); + if (ret != ZE_RESULT_SUCCESS) { + ucs_warn("zeMemPutIpcHandle failed with error 0x%x", ret); + } + + if (fd < 0) { + return UCS_ERR_IO_ERROR; + } + + *(int*)key->ipc_handle.data = fd; + key->pid = getpid(); key->address = (uintptr_t)base_address; key->length = alloc_size; @@ -144,19 +166,12 @@ uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, static ucs_status_t uct_ze_ipc_mem_dereg(uct_md_h uct_md, const uct_md_mem_dereg_params_t *params) { - uct_ze_ipc_md_t *md = ucs_derived_of(uct_md, uct_ze_ipc_md_t); uct_ze_ipc_key_t *key; - ze_result_t ret; UCT_MD_MEM_DEREG_CHECK_PARAMS(params, 0); key = params->memh; - - ret = zeMemPutIpcHandle(md->ze_context, key->ipc_handle); - if (ret != ZE_RESULT_SUCCESS) { - ucs_warn("zeMemPutIpcHandle failed with error 0x%x", ret); - } - + close(*(int*)key->ipc_handle.data); ucs_free(key); return UCS_OK; } diff --git a/test/gtest/uct/ze/test_ze_ipc_rma.cc b/test/gtest/uct/ze/test_ze_ipc_rma.cc index 7cf4b3fd915..3503755df43 100644 --- a/test/gtest/uct/ze/test_ze_ipc_rma.cc +++ b/test/gtest/uct/ze/test_ze_ipc_rma.cc @@ -316,7 +316,35 @@ class test_ze_ipc_rma_xproc : ASSERT_GT(peer.pid, 0) << "fork() failed: " << strerror(errno); } - void run_exporter() + /* The driver closes the fd of an outstanding IPC handle when a device + * other than the allocation's accesses the buffer */ + static void fill_from_other_device(const mapped_buffer &buffer) + { + ze_context_desc_t context_desc = {}; + ze_command_queue_desc_t queue_desc = {}; + ze_command_list_handle_t cmdlist; + ze_context_handle_t context; + uint8_t value = 0; + + context_desc.stype = ZE_STRUCTURE_TYPE_CONTEXT_DESC; + queue_desc.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC; + queue_desc.mode = ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS; + + ASSERT_EQ(ZE_RESULT_SUCCESS, zeContextCreate(uct_ze_base_get_driver(), + &context_desc, &context)); + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeCommandListCreateImmediate(context, + uct_ze_base_get_device(1), + &queue_desc, &cmdlist)); + EXPECT_EQ(ZE_RESULT_SUCCESS, + zeCommandListAppendMemoryFill(cmdlist, buffer.ptr(), &value, + sizeof(value), buffer.length(), + NULL, 0, NULL)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListDestroy(cmdlist)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeContextDestroy(context)); + } + + void run_exporter(bool other_device) { entity *e = uct_test::create_entity(0); m_entities.push_back(e); @@ -328,6 +356,9 @@ class test_ze_ipc_rma_xproc : ASSERT_NO_FATAL_FAILURE(pack_exporter_info(*e, buffer, info)); ASSERT_NO_FATAL_FAILURE(start_importer(peer)); + if (other_device) { + ASSERT_NO_FATAL_FAILURE(fill_from_other_device(buffer)); + } send_msg(peer.fd, info.data(), info.size()); for (unsigned round = 0; round < NUM_ROUNDS; ++round) { @@ -421,12 +452,12 @@ class test_ze_ipc_rma_xproc : send_op(fd, MSG_DONE, NUM_ROUNDS); } - void test_put_get() + void test_put_get(bool other_device = false) { const char *peer_fd = getenv(PEER_FD_ENV); if (peer_fd == NULL) { - run_exporter(); + run_exporter(other_device); } else { run_importer(atoi(peer_fd)); } @@ -446,4 +477,13 @@ UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_no_cache, "ZE_IPC_ENABLE_CACHE=n") test_put_get(); } +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_after_other_device_access) +{ + if (uct_ze_base_get_num_devices() < 2) { + UCS_TEST_SKIP_R("needs two Level Zero devices"); + } + + test_put_get(true); +} + _UCT_INSTANTIATE_TEST_CASE(test_ze_ipc_rma_xproc, ze_ipc) From ad674767178f03d418c86313c50cfa4c8a356fc6 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Fri, 2 Oct 2026 23:59:11 -0700 Subject: [PATCH 34/35] UCT/ZE: Fix ep flush, progress order and md ops uct_base_ep_flush returns UCS_OK while copies are still in flight, so UCP could complete a flush before the data had landed. ep_flush now queues a marker behind the events of each active command list, as the cuda transports do, and invokes the caller's completion once every marker has reached the head of its queue. Closing the iface with a flush still pending frees the flush without completing it. Progress now completes each command list's events in posting order, so a marker completes only after the copies posted before it. It stops at the first event that has not signaled instead of extracting the list and pushing it back. An event leaves its queue only after its completion has run, so a completion that posts on the same command list no longer adds the list to the active queue a second time. iface_flush rejects a completion, like the cuda transports. Also from review: unmap uses the rkey's pid, matching the map; the md defines mem_elem_pack and mem_elem_release; the cache hash key drops its manual pad and packing; ze_copy traces follow the logging style; and the gtest Makefile has a single HAVE_ZE block. Signed-off-by: Tushar Gohad --- src/uct/ze/copy/ze_copy_ep.c | 8 +- src/uct/ze/ze_ipc/ze_ipc_cache.c | 4 +- src/uct/ze/ze_ipc/ze_ipc_ep.c | 9 +- src/uct/ze/ze_ipc/ze_ipc_iface.c | 210 ++++++++++++++++++++------- src/uct/ze/ze_ipc/ze_ipc_iface.h | 6 + src/uct/ze/ze_ipc/ze_ipc_md.c | 4 + test/gtest/Makefile.am | 15 +- test/gtest/uct/ze/test_ze_ipc_rma.cc | 83 +++++++++++ 8 files changed, 261 insertions(+), 78 deletions(-) diff --git a/src/uct/ze/copy/ze_copy_ep.c b/src/uct/ze/copy/ze_copy_ep.c index f74cb6edd6e..635a163bb12 100644 --- a/src/uct/ze/copy/ze_copy_ep.c +++ b/src/uct/ze/copy/ze_copy_ep.c @@ -50,8 +50,8 @@ ucs_status_t uct_ze_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr, ze_result_t ret; void *src, *dst; - ucs_trace("ze_copy_ep: zcopy %s remote_addr=%p local_buf=%p size=%zu", - is_put ? "PUT" : "GET", (void*)remote_addr, iov->buffer, size); + ucs_trace("ze_copy_ep: zcopy %s remote_addr %p local_buf %p size %zu", + is_put ? "put" : "get", (void*)remote_addr, iov->buffer, size); if (is_put) { src = iov->buffer; @@ -147,7 +147,7 @@ ucs_status_t uct_ze_copy_ep_put_short(uct_ep_h tl_ep, const void *buffer, }; ucs_status_t status; - ucs_trace("ze_copy_ep: PUT_SHORT buffer=%p length=%u remote_addr=%p", + ucs_trace("ze_copy_ep: put_short buffer %p length %u remote_addr %p", buffer, length, (void*)remote_addr); status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 1); @@ -169,7 +169,7 @@ ucs_status_t uct_ze_copy_ep_get_short(uct_ep_h tl_ep, void *buffer, }; ucs_status_t status; - ucs_trace("ze_copy_ep: GET_SHORT buffer=%p length=%u remote_addr=%p", + ucs_trace("ze_copy_ep: get_short buffer %p length %u remote_addr %p", buffer, length, (void*)remote_addr); status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 0); diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index ef26d88a573..c726959d350 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -22,9 +22,8 @@ typedef struct uct_ze_ipc_cache_hash_key { pid_t pid; - uint32_t _pad; /* Ensure 8-byte alignment for ze_context */ ze_context_handle_t ze_context; -} UCS_S_PACKED uct_ze_ipc_cache_hash_key_t; +} uct_ze_ipc_cache_hash_key_t; static UCS_F_ALWAYS_INLINE int @@ -278,7 +277,6 @@ uct_ze_ipc_get_remote_cache(pid_t pid, ze_context_handle_t ze_context, key.ze_context = ze_context; key.pid = pid; - key._pad = 0; khiter = kh_put(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, key, &khret); diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c index 7fb3190546b..f1e87a6114b 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_ep.c +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -119,7 +119,6 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, { uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ze_ipc_iface_t); - uct_ze_ipc_ep_t *ep = ucs_derived_of(tl_ep, uct_ze_ipc_ep_t); uct_ze_ipc_key_t *key = (uct_ze_ipc_key_t*)rkey; uct_ze_ipc_event_desc_t *event_desc; uct_ze_ipc_queue_desc_t *q_desc; @@ -165,7 +164,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, event_desc = ucs_malloc(sizeof(*event_desc), "uct_ze_ipc_event_desc_t"); if (event_desc == NULL) { ucs_error("failed to allocate event descriptor"); - uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, + uct_ze_ipc_unmap_memhandle(key->pid, key->address, mapped_addr, iface->ze_context, local_fd, iface->config.enable_cache); return UCS_ERR_NO_MEMORY; @@ -174,7 +173,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, status = uct_ze_ipc_alloc_event(iface, event_desc); if (status != UCS_OK) { ucs_free(event_desc); - uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, + uct_ze_ipc_unmap_memhandle(key->pid, key->address, mapped_addr, iface->ze_context, local_fd, iface->config.enable_cache); return status; @@ -182,7 +181,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, /* Store information for cache-based cleanup and event tracking */ event_desc->dup_fd = local_fd; - event_desc->pid = ep->remote_pid; + event_desc->pid = key->pid; event_desc->address = key->address; /* Set up source and destination based on direction */ @@ -244,7 +243,7 @@ uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, err_cleanup: uct_ze_ipc_free_event(iface, event_desc); ucs_free(event_desc); - uct_ze_ipc_unmap_memhandle(ep->remote_pid, key->address, mapped_addr, + uct_ze_ipc_unmap_memhandle(key->pid, key->address, mapped_addr, iface->ze_context, local_fd, iface->config.enable_cache); return UCS_ERR_IO_ERROR; diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c index 7d306bf8d8c..c25eb04f789 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.c +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -252,76 +252,85 @@ void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, ucs_spin_unlock(&iface->event_lock); } -static unsigned uct_ze_ipc_iface_progress(uct_iface_h tl_iface) +/* Completes events in posting order, so that a flush marker completes only + * after the completions of the copies posted before it. An event leaves the + * queue only after its completion runs, so a completion that posts on the same + * command list does not find the queue empty and add it to the active queue + * a second time */ +static unsigned +uct_ze_ipc_progress_event_queue(uct_ze_ipc_iface_t *iface, + ucs_queue_head_t *event_queue, + unsigned max_events) { - uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + unsigned count = 0; uct_ze_ipc_event_desc_t *event_desc; - uct_ze_ipc_queue_desc_t *q_desc; - ucs_queue_iter_t iter; - unsigned count = 0; - unsigned max_poll = iface->config.max_poll; + ucs_status_t status; ze_result_t ret; - /* Early exit if no active queues */ - if (ucs_queue_is_empty(&iface->active_queue)) { - return 0; - } - - /* - * Progress all active command list queues - * Similar to CUDA IPC's uct_cuda_base_progress_event_queue - */ - ucs_queue_for_each_extract(q_desc, &iface->active_queue, queue, 1) { - ucs_queue_for_each_safe(event_desc, iter, &q_desc->event_queue, queue) { - /* Check if we've reached max_poll limit */ - if (count >= max_poll) { - /* Put queue back and exit */ - ucs_queue_push(&iface->active_queue, &q_desc->queue); - return count; - } - + while ((count < max_events) && !ucs_queue_is_empty(event_queue)) { + event_desc = ucs_queue_head_elem_non_empty(event_queue, + uct_ze_ipc_event_desc_t, + queue); + if (event_desc->event == NULL) { + /* A flush marker is done once it reaches the head */ + ret = ZE_RESULT_SUCCESS; + } else { ret = zeEventQueryStatus(event_desc->event); if (ret == ZE_RESULT_NOT_READY) { - continue; + break; } + if (ret != ZE_RESULT_SUCCESS) { ucs_error("zeEventQueryStatus failed with error 0x%x", ret); } + } - ucs_queue_del_iter(&q_desc->event_queue, iter); - - /* Unmap IPC handle using cache */ - if (event_desc->mapped_addr != NULL) { - ucs_status_t status; - status = uct_ze_ipc_unmap_memhandle(event_desc->pid, - event_desc->address, - event_desc->mapped_addr, - iface->ze_context, - event_desc->dup_fd, - iface->config.enable_cache); - if (status != UCS_OK) { - ucs_warn("failed to unmap IPC handle addr:%p", - event_desc->mapped_addr); - } + /* Unmap IPC handle using cache */ + if (event_desc->mapped_addr != NULL) { + status = uct_ze_ipc_unmap_memhandle(event_desc->pid, + event_desc->address, + event_desc->mapped_addr, + iface->ze_context, + event_desc->dup_fd, + iface->config.enable_cache); + if (status != UCS_OK) { + ucs_warn("failed to unmap IPC handle addr:%p", + event_desc->mapped_addr); } + } - /* Invoke completion callback */ - if (event_desc->comp != NULL) { - uct_invoke_completion(event_desc->comp, - (ret == ZE_RESULT_SUCCESS) ? - UCS_OK : - UCS_ERR_IO_ERROR); - } + if (event_desc->comp != NULL) { + uct_invoke_completion(event_desc->comp, + (ret == ZE_RESULT_SUCCESS) ? + UCS_OK : + UCS_ERR_IO_ERROR); + } + ucs_queue_pull_non_empty(event_queue); + if (event_desc->event != NULL) { uct_ze_ipc_free_event(iface, event_desc); - ucs_free(event_desc); - - count++; } - /* If queue still has events, put it back to active queue */ - if (!ucs_queue_is_empty(&q_desc->event_queue)) { - ucs_queue_push(&iface->active_queue, &q_desc->queue); + ucs_free(event_desc); + count++; + } + + return count; +} + +static unsigned uct_ze_ipc_iface_progress(uct_iface_h tl_iface) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + unsigned max_poll = iface->config.max_poll; + unsigned count = 0; + uct_ze_ipc_queue_desc_t *q_desc; + ucs_queue_iter_t iter; + + ucs_queue_for_each_safe(q_desc, iter, &iface->active_queue, queue) { + count += uct_ze_ipc_progress_event_queue(iface, &q_desc->event_queue, + max_poll - count); + if (ucs_queue_is_empty(&q_desc->event_queue)) { + ucs_queue_del_iter(&iface->active_queue, iter); } } @@ -329,12 +338,96 @@ static unsigned uct_ze_ipc_iface_progress(uct_iface_h tl_iface) } +static void uct_ze_ipc_flush_desc_complete(uct_completion_t *self) +{ + uct_ze_ipc_flush_desc_t *flush_desc = + ucs_container_of(self, uct_ze_ipc_flush_desc_t, super); + + uct_invoke_completion(flush_desc->comp, self->status); + ucs_free(flush_desc); +} + + +/* A marker queued behind the events of each active command list reaches the + * head of its queue only after those events complete, and the last marker to + * get there completes the flush */ +static ucs_status_t +uct_ze_ipc_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_ep->iface, + uct_ze_ipc_iface_t); + uct_ze_ipc_event_desc_t *markers[UCT_ZE_IPC_MAX_PEERS]; + uct_ze_ipc_flush_desc_t *flush_desc; + uct_ze_ipc_queue_desc_t *q_desc; + unsigned i, num_markers; + + if (ucs_queue_is_empty(&iface->active_queue)) { + UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t)); + return UCS_OK; + } + + if (comp == NULL) { + UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); + return UCS_INPROGRESS; + } + + flush_desc = ucs_malloc(sizeof(*flush_desc), "uct_ze_ipc_flush_desc_t"); + if (flush_desc == NULL) { + ucs_error("failed to allocate flush descriptor"); + return UCS_ERR_NO_MEMORY; + } + + num_markers = 0; + ucs_queue_for_each(q_desc, &iface->active_queue, queue) { + ucs_assert(num_markers < UCT_ZE_IPC_MAX_PEERS); + markers[num_markers] = ucs_malloc(sizeof(*markers[num_markers]), + "uct_ze_ipc_flush_marker"); + if (markers[num_markers] == NULL) { + ucs_error("failed to allocate flush marker"); + goto err_free_markers; + } + + ++num_markers; + } + + flush_desc->super.func = uct_ze_ipc_flush_desc_complete; + flush_desc->super.count = num_markers; + flush_desc->super.status = UCS_OK; + flush_desc->comp = comp; + + i = 0; + ucs_queue_for_each(q_desc, &iface->active_queue, queue) { + markers[i]->event = NULL; + markers[i]->event_pool = NULL; + markers[i]->mapped_addr = NULL; + markers[i]->comp = &flush_desc->super; + ucs_queue_push(&q_desc->event_queue, &markers[i]->queue); + ++i; + } + + UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); + return UCS_INPROGRESS; + +err_free_markers: + for (i = 0; i < num_markers; ++i) { + ucs_free(markers[i]); + } + + ucs_free(flush_desc); + return UCS_ERR_NO_MEMORY; +} + + static ucs_status_t uct_ze_ipc_iface_flush(uct_iface_h tl_iface, unsigned flags, uct_completion_t *comp) { uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); unsigned i; + if (comp != NULL) { + return UCS_ERR_UNSUPPORTED; + } + /* Check if all command list queues are empty */ for (i = 0; i < iface->num_cmd_lists; i++) { if (!ucs_queue_is_empty(&iface->queue_desc[i].event_queue)) { @@ -354,7 +447,7 @@ static uct_iface_ops_t uct_ze_ipc_iface_ops = { .ep_put_zcopy = uct_ze_ipc_ep_put_zcopy, .ep_pending_add = (uct_ep_pending_add_func_t)ucs_empty_function_return_busy, .ep_pending_purge = (uct_ep_pending_purge_func_t)ucs_empty_function, - .ep_flush = uct_base_ep_flush, + .ep_flush = uct_ze_ipc_ep_flush, .ep_fence = uct_base_ep_fence, .ep_check = (uct_ep_check_func_t)ucs_empty_function_return_unsupported, .ep_create = UCS_CLASS_NEW_FUNC_NAME(uct_ze_ipc_ep_t), @@ -692,6 +785,7 @@ static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) { uct_ze_ipc_event_desc_t *event_desc; + uct_ze_ipc_flush_desc_t *flush_desc; unsigned i; uct_base_iface_progress_disable(&self->super.super, @@ -711,6 +805,14 @@ static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) } if (event_desc->event != NULL) { zeEventDestroy(event_desc->event); + } else { + /* A flush still pending at close is dropped without + * completing it */ + flush_desc = ucs_container_of(event_desc->comp, + uct_ze_ipc_flush_desc_t, super); + if (--flush_desc->super.count == 0) { + ucs_free(flush_desc); + } } if (event_desc->event_pool != NULL) { zeEventPoolDestroy(event_desc->event_pool); diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.h b/src/uct/ze/ze_ipc/ze_ipc_iface.h index 190d61f090f..6013e15efc0 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_iface.h +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.h @@ -76,6 +76,12 @@ typedef struct uct_ze_ipc_event_desc { } uct_ze_ipc_event_desc_t; +typedef struct uct_ze_ipc_flush_desc { + uct_completion_t super; /* counts the flush markers still queued */ + uct_completion_t *comp; /* user completion of the flush */ +} uct_ze_ipc_flush_desc_t; + + /** * Allocate an event from the shared event pool, or from a private one-event * pool when the shared pool is exhausted diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index 60be56c3a2b..85fd2207747 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -242,6 +242,10 @@ uct_ze_ipc_md_open(uct_component_h component, const char *md_name, ucs_empty_function_return_unsupported, .detect_memory_type = (uct_md_detect_memory_type_func_t) ucs_empty_function_return_unsupported, + .mem_elem_pack = (uct_md_mem_elem_pack_func_t) + ucs_empty_function_return_unsupported, + .mem_elem_release = (uct_md_mem_elem_release_func_t) + ucs_empty_function, }; uct_ze_ipc_md_config_t *config = ucs_derived_of(uct_md_config, uct_ze_ipc_md_config_t); diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index 58b151112ce..6f9b5c82872 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -341,17 +341,7 @@ endif # HAVE_HIP if HAVE_ZE gtest_SOURCES += \ ucm/ze_hooks.cc \ - uct/ze/test_ze.cc -gtest_CPPFLAGS += \ - $(ZE_CPPFLAGS) -gtest_LDFLAGS += \ - $(ZE_LDFLAGS) -gtest_LDADD += \ - $(ZE_LIBS) -endif - -if HAVE_ZE -gtest_SOURCES += \ + uct/ze/test_ze.cc \ uct/ze/test_ze_base.cc \ uct/ze/test_ze_ipc_md.cc \ uct/ze/test_ze_ipc_cache.cc \ @@ -359,8 +349,9 @@ gtest_SOURCES += \ uct/ze/test_ze_copy_md.cc gtest_CPPFLAGS += \ $(ZE_CPPFLAGS) +gtest_LDFLAGS += \ + $(ZE_LDFLAGS) gtest_LDADD += \ - $(ZE_LDFLAGS) \ $(ZE_LIBS) \ $(top_builddir)/src/uct/ze/libuct_ze.la endif diff --git a/test/gtest/uct/ze/test_ze_ipc_rma.cc b/test/gtest/uct/ze/test_ze_ipc_rma.cc index 3503755df43..d5fe658d1ff 100644 --- a/test/gtest/uct/ze/test_ze_ipc_rma.cc +++ b/test/gtest/uct/ze/test_ze_ipc_rma.cc @@ -37,6 +37,23 @@ class test_ze_ipc_rma : public uct_test { m_sender->connect(0, *m_receiver, 0); } + struct flush_comp { + uct_completion_t super; + uct_completion_t *copy_comp; + int copies_left; + }; + + static void copy_done(uct_completion_t *self) + { + } + + static void flush_done(uct_completion_t *self) + { + flush_comp *comp = ucs_container_of(self, flush_comp, super); + + comp->copies_left = comp->copy_comp->count; + } + entity *m_sender; entity *m_receiver; @@ -77,6 +94,72 @@ UCS_TEST_P(test_ze_ipc_rma, get_zcopy) recvbuf.pattern_check(SEED1); } +UCS_TEST_P(test_ze_ipc_rma, ep_flush_waits_for_copies) +{ + static const unsigned num_copies = 8; + static const size_t length = UCS_MBYTE; + + mapped_buffer sendbuf(length * num_copies, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length * num_copies, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + uct_completion_t copy_comp = {copy_done, (int)num_copies, UCS_OK}; + flush_comp flush = {{flush_done, 1, UCS_OK}, ©_comp, -1}; + uct_iov_t iov = *sendbuf.iov(); + + iov.length = length; + for (unsigned i = 0; i < num_copies; ++i) { + iov.buffer = UCS_PTR_BYTE_OFFSET(sendbuf.ptr(), i * length); + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_put_zcopy(m_sender->ep(0), &iov, 1, + recvbuf.addr() + (i * length), + recvbuf.rkey(), ©_comp)); + } + + /* Nothing has progressed the copies yet */ + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_flush(m_sender->ep(0), 0, &flush.super)); + wait_for_value(&flush.super.count, 0, true); + ASSERT_EQ(0, flush.super.count); + + EXPECT_EQ(UCS_OK, flush.super.status); + EXPECT_EQ(0, flush.copies_left); + EXPECT_EQ(UCS_OK, copy_comp.status); + recvbuf.pattern_check(SEED1); +} + +UCS_TEST_P(test_ze_ipc_rma, ep_flush_pending_at_close, "ZE_IPC_MAX_POLL=1") +{ + static const size_t length = 4096; + uct_completion_t copy_comp = {copy_done, 1, UCS_OK}; + flush_comp flush = {{flush_done, 1, UCS_OK}, ©_comp, -1}; + + { + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_put_zcopy(m_sender->ep(0), sendbuf.iov(), 1, + recvbuf.addr(), recvbuf.rkey(), + ©_comp)); + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_flush(m_sender->ep(0), 0, &flush.super)); + + /* With MAX_POLL=1 a progress call completes at most one event, so the + * flush marker is still queued when the copy completes */ + test_base::wait_for_value(©_comp.count, 0, + [this]() { progress(); }); + ASSERT_EQ(0, copy_comp.count); + } + + EXPECT_EQ(1, flush.super.count); + m_sender->destroy_eps(); + m_entities.remove(m_sender); + EXPECT_EQ(1, flush.super.count); +} + UCS_TEST_P(test_ze_ipc_rma, put_zcopy_rejects_multi_iov) { size_t length = 64; From 92d2fa3a64f3b9d0aeccec4641ed09497f8c897c Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Fri, 2 Oct 2026 23:59:29 -0700 Subject: [PATCH 35/35] UCT/ZE: Avoid importer fd matching exporter IPC fd compute-runtime 26.27 keeps the fd that zeMemGetIpcHandle returned inside the opaque IPC handle. zeMemOpenIpcHandle treats a handle whose fd field equals that stored fd as unmodified, and fetches that fd number from the exporter with pidfd_getfd instead of using the fd it was given. The exporter packs a dup and puts the handle right away, which closes the original fd. So when the importer's pidfd_getfd copy lands on the same number, the driver fetches a closed fd, and the open fails with 0x78000004, or it fetches whatever file has since reused the number. The rkey now carries the exporter's original fd. When the importer's copy has the same number, it moves the copy to another fd. The new xproc test forces the collision by filling the importer's fd table up to the exporter's fd, and checks that the importer's copy was moved. Signed-off-by: Tushar Gohad --- src/uct/ze/ze_ipc/ze_ipc_cache.c | 20 +++++- src/uct/ze/ze_ipc/ze_ipc_md.c | 1 + src/uct/ze/ze_ipc/ze_ipc_md.h | 3 + test/gtest/uct/ze/test_ze_ipc_rma.cc | 99 ++++++++++++++++++++++++++-- 4 files changed, 117 insertions(+), 6 deletions(-) diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c index c726959d350..1379a7b57e0 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_cache.c +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -187,7 +188,7 @@ static ucs_status_t uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, void **mapped_addr, int *dup_fd) { ze_ipc_mem_handle_t local_handle; - int remote_fd; + int remote_fd, fd; ze_result_t ret; /* Extract fd from IPC handle */ @@ -208,6 +209,23 @@ static ucs_status_t uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, key->pid); return UCS_ERR_IO_ERROR; } + + /* zeMemOpenIpcHandle takes a handle whose fd equals the exporter's + * original fd as unmodified and fetches that fd number from the + * exporter, where it is already closed or reused */ + if (*dup_fd == key->export_fd) { + fd = fcntl(*dup_fd, F_DUPFD_CLOEXEC, 0); + if (fd < 0) { + ucs_error("failed to duplicate fd %d: %m", *dup_fd); + } + + close(*dup_fd); + *dup_fd = fd; + if (fd < 0) { + return UCS_ERR_IO_ERROR; + } + } + /* Update handle with local fd */ *(int*)local_handle.data = *dup_fd; } else { diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c index 85fd2207747..5d50d5bd912 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.c +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -121,6 +121,7 @@ static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, return UCS_ERR_IO_ERROR; } + key->export_fd = *(int*)key->ipc_handle.data; *(int*)key->ipc_handle.data = fd; key->pid = getpid(); diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.h b/src/uct/ze/ze_ipc/ze_ipc_md.h index 209308c3cc1..5db80c4e9e2 100644 --- a/src/uct/ze/ze_ipc/ze_ipc_md.h +++ b/src/uct/ze/ze_ipc/ze_ipc_md.h @@ -43,6 +43,9 @@ typedef struct uct_ze_ipc_key { distinguishes processes that reuse a PID */ pid_t pid; /**< Remote process ID (for cache) */ + int export_fd; /**< fd zeMemGetIpcHandle returned in the + remote process, before the handle + was repacked with a dup */ uintptr_t address; /**< Base address of the allocation */ size_t length; /**< Size of the allocation */ } uct_ze_ipc_key_t; diff --git a/test/gtest/uct/ze/test_ze_ipc_rma.cc b/test/gtest/uct/ze/test_ze_ipc_rma.cc index d5fe658d1ff..1ba6f23e142 100644 --- a/test/gtest/uct/ze/test_ze_ipc_rma.cc +++ b/test/gtest/uct/ze/test_ze_ipc_rma.cc @@ -10,6 +10,7 @@ extern "C" { #include } +#include #include #include #include @@ -250,11 +251,44 @@ class test_ze_ipc_rma_xproc : } }; + /* Holds every free fd below a limit, so the next fd opened lands above */ + struct fd_fillers { + std::vector fds; + + ~fd_fillers() + { + release(); + } + + void occupy_below(int limit) + { + int fd; + + while ((fd = open("/dev/null", O_RDONLY | O_CLOEXEC)) < limit) { + ASSERT_GE(fd, 0) << "open(/dev/null) failed: " + << strerror(errno); + fds.push_back(fd); + } + + close(fd); + } + + void release() + { + for (int fd : fds) { + close(fd); + } + fds.clear(); + } + }; + static const char *PEER_FD_ENV; static const int PEER_FD = 3; static const int TIMEOUT_MS = 60000; static const unsigned NUM_ROUNDS = 3; static const size_t LENGTH = UCS_MBYTE; + /* Above any fd the importer opens on its own */ + static const int MIN_EXPORT_FD = 256; static uint64_t put_seed(unsigned round) { @@ -427,16 +461,51 @@ class test_ze_ipc_rma_xproc : EXPECT_EQ(ZE_RESULT_SUCCESS, zeContextDestroy(context)); } - void run_exporter(bool other_device) + /* The transport opens a pidfd and then takes the exporter's fd, each into + * the lowest free fd. Leaving only export_fd - 1 and export_fd free below + * export_fd makes it receive the exporter's fd under the number that + * zeMemGetIpcHandle returned to the exporter */ + static void force_fd_collision(const uct_ze_ipc_key_t *key, + fd_fillers &fillers) + { + ASSERT_GE(key->export_fd, MIN_EXPORT_FD); + ASSERT_NO_FATAL_FAILURE(fillers.occupy_below(key->export_fd - 1)); + ASSERT_EQ(-1, fcntl(key->export_fd - 1, F_GETFD)); + ASSERT_EQ(-1, fcntl(key->export_fd, F_GETFD)); + } + + /* With the collision forced, the fix leaves the importer's copy of the + * exporter's fd at export_fd - 1, the lowest fd it can move it to */ + static void check_fd_moved(const uct_ze_ipc_key_t *key) + { + std::string path = "/proc/self/fd/" + + ucs::to_string(key->export_fd - 1); + char target[256]; + ssize_t len; + + len = readlink(path.c_str(), target, sizeof(target) - 1); + ASSERT_GE(len, 0) << path << ": " << strerror(errno); + target[len] = '\0'; + EXPECT_NE(nullptr, strstr(target, "dmabuf")) + << path << " -> " << target; + } + + void run_exporter(bool other_device, bool collide_fd) { entity *e = uct_test::create_entity(0); m_entities.push_back(e); + fd_fillers fillers; + if (collide_fd) { + ASSERT_NO_FATAL_FAILURE(fillers.occupy_below(MIN_EXPORT_FD)); + } + mapped_buffer buffer(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); std::vector info; peer_process peer; int status; + fillers.release(); ASSERT_NO_FATAL_FAILURE(pack_exporter_info(*e, buffer, info)); ASSERT_NO_FATAL_FAILURE(start_importer(peer)); if (other_device) { @@ -458,7 +527,7 @@ class test_ze_ipc_rma_xproc : << "importer exit status " << status; } - void run_importer(int fd) + void run_importer(int fd, bool collide_fd) { std::vector info(UCS_KBYTE * 4); uct_iface_is_reachable_params_t params = {}; @@ -514,12 +583,26 @@ class test_ze_ipc_rma_xproc : mapped_buffer sendbuf(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); mapped_buffer recvbuf(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); + fd_fillers fillers; for (unsigned round = 0; round < NUM_ROUNDS; ++round) { + bool collide = collide_fd && (round == 0); + sendbuf.pattern_fill(put_seed(round)); + if (collide) { + ASSERT_NO_FATAL_FAILURE(force_fd_collision( + (const uct_ze_ipc_key_t*)rkey.rkey, fillers)); + } + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_put_zcopy(ep, sendbuf.iov(), 1, hdr.address, rkey.rkey, NULL)); + if (collide) { + ASSERT_NO_FATAL_FAILURE(check_fd_moved( + (const uct_ze_ipc_key_t*)rkey.rkey)); + } + + fillers.release(); e->flush(); send_op(fd, MSG_PUT_DONE, round); @@ -535,19 +618,20 @@ class test_ze_ipc_rma_xproc : send_op(fd, MSG_DONE, NUM_ROUNDS); } - void test_put_get(bool other_device = false) + void test_put_get(bool other_device = false, bool collide_fd = false) { const char *peer_fd = getenv(PEER_FD_ENV); if (peer_fd == NULL) { - run_exporter(other_device); + run_exporter(other_device, collide_fd); } else { - run_importer(atoi(peer_fd)); + run_importer(atoi(peer_fd), collide_fd); } } }; const char *test_ze_ipc_rma_xproc::PEER_FD_ENV = "UCT_ZE_IPC_TEST_PEER_FD"; +const int test_ze_ipc_rma_xproc::MIN_EXPORT_FD; UCS_TEST_P(test_ze_ipc_rma_xproc, put_get) @@ -569,4 +653,9 @@ UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_after_other_device_access) test_put_get(true); } +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_importer_fd_equals_export_fd) +{ + test_put_get(false, true); +} + _UCT_INSTANTIATE_TEST_CASE(test_ze_ipc_rma_xproc, ze_ipc)