diff --git a/AUTHORS b/AUTHORS index 1f7b2218fb9..78f1ed59459 100644 --- a/AUTHORS +++ b/AUTHORS @@ -63,6 +63,7 @@ Jeff Daily Jeffrey Mahou Jianxin Xiong John Snyder +Jon Stevens <159235294+jon-hotaisle@users.noreply.github.com> Jonas Zhou Joseph Schuchart Keisuke Fukuda @@ -126,6 +127,7 @@ Swen Boehm Thomas Vegas Tomer Gilad Tony Curtis +Tushar Gohad Tzafrir Cohen Valentin Petrov Wenbin Lu @@ -144,7 +146,10 @@ Zhu Yanjun Zihao Zhao lzhang2 michaelzli +shasson5 <103439971+shasson5@users.noreply.github.com> +wehy <18174858434@163.com> ybenabou +yuanwu In addition we would like to acknowledge the following members of UCX community for their participation in annual face-to-face meeting, design discussions, and diff --git a/src/uct/ze/Makefile.am b/src/uct/ze/Makefile.am index 6b7ea473bab..988ab6a3a3b 100644 --- a/src/uct/ze/Makefile.am +++ b/src/uct/ze/Makefile.am @@ -27,12 +27,20 @@ libuct_ze_la_SOURCES = \ noinst_HEADERS += \ copy/ze_copy_md.h \ copy/ze_copy_iface.h \ - copy/ze_copy_ep.h + copy/ze_copy_ep.h \ + ze_ipc/ze_ipc_md.h \ + ze_ipc/ze_ipc_iface.h \ + ze_ipc/ze_ipc_ep.h \ + ze_ipc/ze_ipc_cache.h libuct_ze_la_SOURCES += \ copy/ze_copy_md.c \ copy/ze_copy_iface.c \ - copy/ze_copy_ep.c + copy/ze_copy_ep.c \ + ze_ipc/ze_ipc_md.c \ + ze_ipc/ze_ipc_iface.c \ + ze_ipc/ze_ipc_ep.c \ + ze_ipc/ze_ipc_cache.c PKG_CONFIG_NAME=ze diff --git a/src/uct/ze/base/ze_base.c b/src/uct/ze/base/ze_base.c index ab09ae89dd7..b975715aace 100644 --- a/src/uct/ze/base/ze_base.c +++ b/src/uct/ze/base/ze_base.c @@ -374,6 +374,29 @@ const uct_ze_subdevice_t *uct_ze_base_get_subdevice_by_global_id(int global_id) return &ze_subdevices[global_id]; } +ze_device_handle_t uct_ze_base_get_device(int ordinal) +{ + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + return NULL; + } + + if ((ordinal < 0) || (ordinal >= uct_ze_base.num_devices)) { + return NULL; + } + + return uct_ze_base.devices[ordinal].root_device; +} + + +int uct_ze_base_get_num_devices(void) +{ + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + return 0; + } + + return uct_ze_base.num_devices; +} + ze_device_handle_t uct_ze_base_get_device_handle_from_subdevice(const uct_ze_subdevice_t *subdevice) { @@ -512,6 +535,18 @@ static void UCS_F_DTOR uct_ze_base_cleanup(void) /* ZE driver cleanup happens automatically on process exit */ } + +ucs_status_t +uct_ze_base_query_devices_common(uct_md_h md, uct_device_type_t dev_type, + uct_tl_device_resource_t **tl_devices_p, + unsigned *num_tl_devices_p) +{ + return uct_single_device_resource(md, md->component->name, dev_type, + UCS_SYS_DEVICE_ID_UNKNOWN, tl_devices_p, + num_tl_devices_p); +} + + UCS_MODULE_INIT() { return UCS_OK; diff --git a/src/uct/ze/base/ze_base.h b/src/uct/ze/base/ze_base.h index 40ca69f3346..d72c9d8c3d4 100644 --- a/src/uct/ze/base/ze_base.h +++ b/src/uct/ze/base/ze_base.h @@ -65,6 +65,13 @@ ze_device_handle_t uct_ze_base_get_device_handle_from_subdevice( ucs_sys_device_t uct_ze_base_get_sys_dev_from_handle(ze_device_handle_t device); +/* Get root device handle by ordinal, returns NULL if out of range */ +ze_device_handle_t uct_ze_base_get_device(int ordinal); + + +int uct_ze_base_get_num_devices(void); + + ucs_status_t uct_ze_base_query_md_resources(uct_component_h component, uct_md_resource_desc_t **resources_p, @@ -74,4 +81,9 @@ ucs_status_t uct_ze_base_query_devices(uct_md_h md, uct_tl_device_resource_t **tl_devices_p, unsigned *num_tl_devices_p); +ucs_status_t +uct_ze_base_query_devices_common(uct_md_h md, uct_device_type_t dev_type, + uct_tl_device_resource_t **tl_devices_p, + unsigned *num_tl_devices_p); + #endif diff --git a/src/uct/ze/copy/ze_copy_ep.c b/src/uct/ze/copy/ze_copy_ep.c index 8cc89371c51..635a163bb12 100644 --- a/src/uct/ze/copy/ze_copy_ep.c +++ b/src/uct/ze/copy/ze_copy_ep.c @@ -50,8 +50,8 @@ ucs_status_t uct_ze_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr, ze_result_t ret; void *src, *dst; - ucs_trace("remote addr %p rkey %p size %zu", (void*)remote_addr, - (void*)rkey, size); + ucs_trace("ze_copy_ep: zcopy %s remote_addr %p local_buf %p size %zu", + is_put ? "put" : "get", (void*)remote_addr, iov->buffer, size); if (is_put) { src = iov->buffer; @@ -147,6 +147,9 @@ ucs_status_t uct_ze_copy_ep_put_short(uct_ep_h tl_ep, const void *buffer, }; ucs_status_t status; + ucs_trace("ze_copy_ep: put_short buffer %p length %u remote_addr %p", + buffer, length, (void*)remote_addr); + status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 1); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), PUT, SHORT, length); @@ -166,6 +169,9 @@ ucs_status_t uct_ze_copy_ep_get_short(uct_ep_h tl_ep, void *buffer, }; ucs_status_t status; + ucs_trace("ze_copy_ep: get_short buffer %p length %u remote_addr %p", + buffer, length, (void*)remote_addr); + status = uct_ze_copy_ep_zcopy(tl_ep, remote_addr, &iov, rkey, 0); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), GET, SHORT, length); diff --git a/src/uct/ze/copy/ze_copy_iface.c b/src/uct/ze/copy/ze_copy_iface.c index fbc6ed4c8a0..770162106fa 100644 --- a/src/uct/ze/copy/ze_copy_iface.c +++ b/src/uct/ze/copy/ze_copy_iface.c @@ -259,16 +259,19 @@ static UCS_CLASS_INIT_FUNC(uct_ze_copy_iface_t, uct_md_h md, /* Use the device configured in the MD */ device = ze_md->ze_device; if (device == NULL) { + ucs_error("ze_copy_iface: memory domain has no device"); return UCS_ERR_NO_DEVICE; } ret = zeCommandQueueCreate(ze_md->ze_context, device, &cq_desc, &cmdq); if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_copy_iface: zeCommandQueueCreate failed with 0x%x", ret); return UCS_ERR_NO_DEVICE; } ret = zeCommandListCreate(ze_md->ze_context, device, &cl_desc, &cmdl); if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_copy_iface: zeCommandListCreate failed with 0x%x", ret); zeCommandQueueDestroy(cmdq); return UCS_ERR_NO_DEVICE; } @@ -277,6 +280,9 @@ static UCS_CLASS_INIT_FUNC(uct_ze_copy_iface_t, uct_md_h md, self->ze_cmdl = cmdl; self->id = ucs_generate_uuid((uintptr_t)self); + ucs_debug("ze_copy_iface: initialized iface on device %p cmdq %p cmdl %p", + device, cmdq, cmdl); + return UCS_OK; } diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.c b/src/uct/ze/ze_ipc/ze_ipc_cache.c new file mode 100644 index 00000000000..1379a7b57e0 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.c @@ -0,0 +1,621 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ze_ipc_cache.h" +#include "ze_ipc_iface.h" +#include "ze_ipc_ep.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +typedef struct uct_ze_ipc_cache_hash_key { + pid_t pid; + ze_context_handle_t ze_context; +} uct_ze_ipc_cache_hash_key_t; + + +static UCS_F_ALWAYS_INLINE int +uct_ze_ipc_cache_hash_equal(uct_ze_ipc_cache_hash_key_t key1, + uct_ze_ipc_cache_hash_key_t key2) +{ + return (key1.pid == key2.pid) && (key1.ze_context == key2.ze_context); +} + + +static UCS_F_ALWAYS_INLINE khint32_t +uct_ze_ipc_cache_hash_func(uct_ze_ipc_cache_hash_key_t key) +{ + return kh_int_hash_func((uintptr_t)key.pid ^ (uintptr_t)key.ze_context); +} + + +KHASH_INIT(ze_ipc_rem_cache, uct_ze_ipc_cache_hash_key_t, uct_ze_ipc_cache_t*, + 1, uct_ze_ipc_cache_hash_func, uct_ze_ipc_cache_hash_equal); + + +typedef struct uct_ze_ipc_remote_cache { + khash_t(ze_ipc_rem_cache) hash; + ucs_recursive_spinlock_t lock; +} uct_ze_ipc_remote_cache_t; + + +static uct_ze_ipc_remote_cache_t uct_ze_ipc_remote_cache; + + +static ucs_status_t +uct_ze_ipc_create_cache_with_ops(uct_ze_ipc_cache_t **cache, const char *name, + const uct_ze_ipc_cache_ops_t *ops); + + +static ucs_pgt_dir_t * +uct_ze_ipc_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) +{ + void *ptr; + int ret; + + ret = ucs_posix_memalign(&ptr, + ucs_max(sizeof(void*), UCS_PGT_ENTRY_MIN_ALIGN), + sizeof(ucs_pgt_dir_t), "ze_ipc_cache_pgdir"); + return (ret == 0) ? ptr : NULL; +} + + +static void uct_ze_ipc_cache_pgt_dir_release(const ucs_pgtable_t *pgtable, + ucs_pgt_dir_t *dir) +{ + ucs_free(dir); +} + + +static void +uct_ze_ipc_cache_region_collect_callback(const ucs_pgtable_t *pgtable, + ucs_pgt_region_t *pgt_region, + void *arg) +{ + ucs_list_link_t *list = arg; + uct_ze_ipc_cache_region_t *region; + + region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + ucs_list_add_tail(list, ®ion->list); +} + + +static ucs_status_t uct_ze_ipc_close_mapping(ze_context_handle_t ze_context, + void *mapped_addr, int dup_fd) +{ + ze_result_t ret; + + ret = zeMemCloseIpcHandle(ze_context, mapped_addr); + if (dup_fd >= 0) { + close(dup_fd); + } + + if (ret != ZE_RESULT_SUCCESS) { + ucs_warn("zeMemCloseIpcHandle failed with error 0x%x", ret); + return UCS_ERR_IO_ERROR; + } + + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_close_memhandle(uct_ze_ipc_cache_t *cache, + uct_ze_ipc_cache_region_t *region) +{ + return cache->ops.close(region->ze_context, region->mapped_addr, + region->dup_fd); +} + + +/* + * Drop a region that has already been unlinked from the page table. If an + * operation is still in flight on the mapping, closing it here would pull the + * memory out from under the running copy, so keep it on the orphan list and let + * the matching uct_ze_ipc_unmap_memhandle() close it. + */ +static void uct_ze_ipc_cache_region_put(uct_ze_ipc_cache_t *cache, + uct_ze_ipc_cache_region_t *region) +{ + if (region->refcount > 0) { + ucs_list_add_tail(&cache->orphan_list, ®ion->list); + return; + } + + uct_ze_ipc_close_memhandle(cache, region); + ucs_free(region); +} + + +static uct_ze_ipc_cache_region_t * +uct_ze_ipc_cache_find_orphan(uct_ze_ipc_cache_t *cache, void *mapped_addr) +{ + uct_ze_ipc_cache_region_t *region; + + ucs_list_for_each(region, &cache->orphan_list, list) { + if (region->mapped_addr == mapped_addr) { + return region; + } + } + + return NULL; +} + + +static void uct_ze_ipc_cache_purge(uct_ze_ipc_cache_t *cache) +{ + uct_ze_ipc_cache_region_t *region, *tmp; + ucs_list_link_t region_list; + + ucs_list_head_init(®ion_list); + ucs_pgtable_purge(&cache->pgtable, uct_ze_ipc_cache_region_collect_callback, + ®ion_list); + ucs_list_for_each_safe(region, tmp, ®ion_list, list) { + uct_ze_ipc_close_memhandle(cache, region); + ucs_free(region); + } + + /* the context backing these mappings is going away, so orphans cannot be + * left for their in-flight operations to reclaim: closing them later would + * pass a destroyed context to zeMemCloseIpcHandle(). This is only safe + * because the caller must be quiesced - see uct_ze_ipc_purge_cache_by_context() */ + ucs_list_for_each_safe(region, tmp, &cache->orphan_list, list) { + ucs_list_del(®ion->list); + uct_ze_ipc_close_memhandle(cache, region); + ucs_free(region); + } + + ucs_trace("%s: ze ipc cache purged", cache->name); +} + + +static ucs_status_t uct_ze_ipc_open_memhandle(uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + void **mapped_addr, int *dup_fd) +{ + ze_ipc_mem_handle_t local_handle; + int remote_fd, fd; + ze_result_t ret; + + /* Extract fd from IPC handle */ + memcpy(&local_handle, &key->ipc_handle, sizeof(local_handle)); + remote_fd = *(int*)local_handle.data; + + /* Duplicate the file descriptor from remote process */ + if (key->pid != getpid()) { + if (remote_fd < 0) { + ucs_error("invalid fd %d in ipc handle from pid %d", remote_fd, + key->pid); + return UCS_ERR_INVALID_PARAM; + } + + *dup_fd = uct_ze_ipc_dup_fd_from_pid(key->pid, remote_fd); + if (*dup_fd < 0) { + ucs_error("failed to duplicate fd %d from pid %d", remote_fd, + key->pid); + return UCS_ERR_IO_ERROR; + } + + /* zeMemOpenIpcHandle takes a handle whose fd equals the exporter's + * original fd as unmodified and fetches that fd number from the + * exporter, where it is already closed or reused */ + if (*dup_fd == key->export_fd) { + fd = fcntl(*dup_fd, F_DUPFD_CLOEXEC, 0); + if (fd < 0) { + ucs_error("failed to duplicate fd %d: %m", *dup_fd); + } + + close(*dup_fd); + *dup_fd = fd; + if (fd < 0) { + return UCS_ERR_IO_ERROR; + } + } + + /* Update handle with local fd */ + *(int*)local_handle.data = *dup_fd; + } else { + *dup_fd = -1; + } + + ret = zeMemOpenIpcHandle(ze_context, ze_device, local_handle, 0, + mapped_addr); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeMemOpenIpcHandle failed with error 0x%x", ret); + if (*dup_fd >= 0) { + close(*dup_fd); + *dup_fd = -1; + } + return UCS_ERR_IO_ERROR; + } + + return UCS_OK; +} + + +static const uct_ze_ipc_cache_ops_t uct_ze_ipc_cache_default_ops = { + .open = uct_ze_ipc_open_memhandle, + .close = uct_ze_ipc_close_mapping +}; + + +static void uct_ze_ipc_cache_invalidate_regions(uct_ze_ipc_cache_t *cache, + void *from, void *to) +{ + ucs_list_link_t region_list; + ucs_status_t status; + uct_ze_ipc_cache_region_t *region, *tmp; + + ucs_list_head_init(®ion_list); + ucs_pgtable_search_range(&cache->pgtable, (ucs_pgt_addr_t)from, + (ucs_pgt_addr_t)to - 1, + uct_ze_ipc_cache_region_collect_callback, + ®ion_list); + ucs_list_for_each_safe(region, tmp, ®ion_list, list) { + status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_error("failed to remove address:%p from cache (%s)", + (void*)region->key.address, ucs_status_string(status)); + } + + ucs_list_del(®ion->list); + uct_ze_ipc_cache_region_put(cache, region); + } + ucs_trace("%s: closed memhandles in the range [%p..%p]", cache->name, from, + to); +} + + +static ucs_status_t +uct_ze_ipc_get_remote_cache(pid_t pid, ze_context_handle_t ze_context, + const uct_ze_ipc_cache_ops_t *ops, + uct_ze_ipc_cache_t **cache) +{ + ucs_status_t status = UCS_OK; + char target_name[64]; + uct_ze_ipc_cache_hash_key_t key; + khiter_t khiter; + int khret; + + ucs_recursive_spin_lock(&uct_ze_ipc_remote_cache.lock); + + key.ze_context = ze_context; + key.pid = pid; + + khiter = kh_put(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, key, + &khret); + if ((khret == UCS_KH_PUT_BUCKET_EMPTY) || + (khret == UCS_KH_PUT_BUCKET_CLEAR)) { + ucs_snprintf_safe(target_name, sizeof(target_name), "dest:%d:%p", + key.pid, key.ze_context); + status = uct_ze_ipc_create_cache_with_ops( + cache, target_name, + (ops == NULL) ? &uct_ze_ipc_cache_default_ops : ops); + if (status != UCS_OK) { + kh_del(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, khiter); + ucs_error("could not create ze ipc cache: %s", + ucs_status_string(status)); + goto err_unlock; + } + + kh_val(&uct_ze_ipc_remote_cache.hash, khiter) = *cache; + } else if (khret == UCS_KH_PUT_KEY_PRESENT) { + *cache = kh_val(&uct_ze_ipc_remote_cache.hash, khiter); + } else { + ucs_error("unable to use ze_ipc remote_cache hash"); + status = UCS_ERR_NO_RESOURCE; + } +err_unlock: + ucs_recursive_spin_unlock(&uct_ze_ipc_remote_cache.lock); + return status; +} + + +ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, + void *mapped_addr, + ze_context_handle_t ze_context, + int dup_fd, int cache_enabled) +{ + ucs_status_t status = UCS_OK; + uct_ze_ipc_cache_t *cache; + ucs_pgt_region_t *pgt_region; + uct_ze_ipc_cache_region_t *region; + int orphaned; + + status = uct_ze_ipc_get_remote_cache(pid, ze_context, NULL, &cache); + if (status != UCS_OK) { + return status; + } + + /* use write lock because cache maybe modified */ + pthread_rwlock_wrlock(&cache->lock); + pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, &cache->pgtable, address); + region = NULL; + if (pgt_region != NULL) { + region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + } + + /* the cached region may have been replaced by a remap of a reused remote + * address while this operation was in flight, in which case the mapping it + * used was moved to the orphan list */ + orphaned = (region == NULL) || (region->mapped_addr != mapped_addr); + if (orphaned) { + region = uct_ze_ipc_cache_find_orphan(cache, mapped_addr); + } + + if (region == NULL) { + pthread_rwlock_unlock(&cache->lock); + ucs_error("%s: no ze_ipc cache region for mapped address %p", + cache->name, mapped_addr); + return UCS_ERR_NO_ELEM; + } + + ucs_assert(region->refcount >= 1); + region->refcount--; + + /* + * check refcount to see if an in-flight transfer is using the same mapping + */ + if ((region->refcount == 0) && (orphaned || !cache_enabled)) { + if (orphaned) { + ucs_list_del(®ion->list); + } else { + status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_error("failed to remove address:%p from cache (%s)", + (void*)region->key.address, + ucs_status_string(status)); + } + } + + status = uct_ze_ipc_close_memhandle(cache, region); + ucs_free(region); + } + + pthread_rwlock_unlock(&cache->lock); + return status; +} + + +UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_map_memhandle, + (key, ze_context, ze_device, ops, mapped_addr, dup_fd), + uct_ze_ipc_key_t *key, ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + const uct_ze_ipc_cache_ops_t *ops, void **mapped_addr, + int *dup_fd) +{ + uct_ze_ipc_cache_t *cache; + ucs_status_t status; + ucs_pgt_region_t *pgt_region; + uct_ze_ipc_cache_region_t *region; + int ret; + + status = uct_ze_ipc_get_remote_cache(key->pid, ze_context, ops, &cache); + if (status != UCS_OK) { + return status; + } + + pthread_rwlock_wrlock(&cache->lock); + pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, &cache->pgtable, + key->address); + if (ucs_likely(pgt_region != NULL)) { + region = ucs_derived_of(pgt_region, uct_ze_ipc_cache_region_t); + + /* the IPC handle itself cannot discriminate here: on Linux it carries + * an fd number in the exporting process, and fd numbers are reused as + * eagerly as virtual addresses. The driver allocation id is a better + * discriminator, though ZE only defines it as an allocation identifier + * and does not specify its uniqueness scope; the Intel compute runtime + * increments it per process. Since that counter restarts in a fresh + * process, pid alone does not separate a dead peer from a new one that + * reused its PID - the creation time supplies that generation. */ + if (ucs_likely( + (region->key.alloc_id == key->alloc_id) && + (region->key.proc_create_time == key->proc_create_time) && + (region->key.length == key->length))) { + /* cache hit */ + ucs_trace("%s: ze_ipc cache hit addr:%p size:%lu " + "region:" UCS_PGT_REGION_FMT, + cache->name, (void*)key->address, key->length, + UCS_PGT_REGION_ARG(®ion->super)); + + *mapped_addr = region->mapped_addr; + *dup_fd = region->dup_fd; + ucs_assert(region->refcount < UINT64_MAX); + region->refcount++; + pthread_rwlock_unlock(&cache->lock); + return UCS_OK; + } + + /* remote allocation at this address was freed and a different one + * reused the same VA - drop the stale mapping and remap below */ + ucs_trace("%s: ze_ipc cache stale mapping addr:%p, invalidating", + cache->name, (void*)key->address); + status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_error("failed to remove address:%p from cache (%s)", + (void*)region->key.address, ucs_status_string(status)); + } + uct_ze_ipc_cache_region_put(cache, region); + } + + status = cache->ops.open(key, ze_context, ze_device, mapped_addr, dup_fd); + if (ucs_unlikely(status != UCS_OK)) { + goto err; + } + + /* create new cache entry */ + ret = ucs_posix_memalign((void**)®ion, + ucs_max(sizeof(void*), UCS_PGT_ENTRY_MIN_ALIGN), + sizeof(uct_ze_ipc_cache_region_t), + "uct_ze_ipc_cache_region"); + if (ret != 0) { + ucs_warn("failed to allocate uct_ze_ipc_cache region"); + status = UCS_ERR_NO_MEMORY; + goto err_close; + } + + region->super.start = ucs_align_down_pow2((uintptr_t)key->address, + UCS_PGT_ADDR_ALIGN); + region->super.end = ucs_align_up_pow2((uintptr_t)key->address + key->length, + UCS_PGT_ADDR_ALIGN); + region->key = *key; + region->mapped_addr = *mapped_addr; + region->refcount = 1; + region->ze_context = ze_context; + region->dup_fd = *dup_fd; + + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); + if (status == UCS_ERR_ALREADY_EXISTS) { + /* overlapped region means memory freed at source. remove and try insert */ + uct_ze_ipc_cache_invalidate_regions(cache, (void*)region->super.start, + (void*)region->super.end); + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); + } + if (status != UCS_OK) { + ucs_error("%s: failed to insert region:" UCS_PGT_REGION_FMT + " size:%lu :%s", + cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length, + ucs_status_string(status)); + ucs_free(region); + goto err_close; + } + + ucs_trace("%s: ze_ipc cache new region:" UCS_PGT_REGION_FMT " size:%lu", + cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length); + + pthread_rwlock_unlock(&cache->lock); + return UCS_OK; + +err_close: + cache->ops.close(ze_context, *mapped_addr, *dup_fd); + *mapped_addr = NULL; + *dup_fd = -1; +err: + pthread_rwlock_unlock(&cache->lock); + return status; +} + + +static ucs_status_t +uct_ze_ipc_create_cache_with_ops(uct_ze_ipc_cache_t **cache, const char *name, + const uct_ze_ipc_cache_ops_t *ops) +{ + ucs_status_t status; + uct_ze_ipc_cache_t *cache_desc; + int ret; + + ucs_assert((ops != NULL) && (ops->open != NULL) && (ops->close != NULL)); + + cache_desc = ucs_malloc(sizeof(uct_ze_ipc_cache_t), "uct_ze_ipc_cache_t"); + if (cache_desc == NULL) { + ucs_error("failed to allocate memory for ze_ipc cache"); + return UCS_ERR_NO_MEMORY; + } + + ret = pthread_rwlock_init(&cache_desc->lock, NULL); + if (ret) { + ucs_error("pthread_rwlock_init() failed: %m"); + status = UCS_ERR_INVALID_PARAM; + goto err; + } + + status = ucs_pgtable_init(&cache_desc->pgtable, + uct_ze_ipc_cache_pgt_dir_alloc, + uct_ze_ipc_cache_pgt_dir_release); + if (status != UCS_OK) { + goto err_destroy_rwlock; + } + + cache_desc->name = strdup(name); + if (cache_desc->name == NULL) { + status = UCS_ERR_NO_MEMORY; + goto err_destroy_rwlock; + } + + ucs_list_head_init(&cache_desc->orphan_list); + cache_desc->ops = *ops; + + *cache = cache_desc; + return UCS_OK; + +err_destroy_rwlock: + pthread_rwlock_destroy(&cache_desc->lock); +err: + free(cache_desc); + return status; +} + + +ucs_status_t +uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name) +{ + return uct_ze_ipc_create_cache_with_ops(cache, name, + &uct_ze_ipc_cache_default_ops); +} + + +void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache) +{ + uct_ze_ipc_cache_purge(cache); + ucs_pgtable_cleanup(&cache->pgtable); + pthread_rwlock_destroy(&cache->lock); + free(cache->name); + ucs_free(cache); +} + + +void uct_ze_ipc_purge_cache_by_context(ze_context_handle_t ze_context) +{ + uct_ze_ipc_cache_t *cache; + khint_t khiter; + + ucs_recursive_spin_lock(&uct_ze_ipc_remote_cache.lock); + for (khiter = kh_begin(&uct_ze_ipc_remote_cache.hash); + khiter != kh_end(&uct_ze_ipc_remote_cache.hash); ++khiter) { + if (!kh_exist(&uct_ze_ipc_remote_cache.hash, khiter) || + (kh_key(&uct_ze_ipc_remote_cache.hash, khiter).ze_context != + ze_context)) { + continue; + } + + cache = kh_val(&uct_ze_ipc_remote_cache.hash, khiter); + kh_del(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash, khiter); + uct_ze_ipc_destroy_cache(cache); + } + ucs_recursive_spin_unlock(&uct_ze_ipc_remote_cache.lock); +} + + +UCS_STATIC_INIT +{ + ucs_recursive_spinlock_init(&uct_ze_ipc_remote_cache.lock, 0); + kh_init_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); +} + + +UCS_STATIC_CLEANUP +{ + uct_ze_ipc_cache_t *rem_cache; + + kh_foreach_value(&uct_ze_ipc_remote_cache.hash, rem_cache, + { uct_ze_ipc_destroy_cache(rem_cache); }) + kh_destroy_inplace(ze_ipc_rem_cache, &uct_ze_ipc_remote_cache.hash); + ucs_recursive_spinlock_destroy(&uct_ze_ipc_remote_cache.lock); +} diff --git a/src/uct/ze/ze_ipc/ze_ipc_cache.h b/src/uct/ze/ze_ipc/ze_ipc_cache.h new file mode 100644 index 00000000000..ad8bd0df8c1 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_cache.h @@ -0,0 +1,136 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_CACHE_H_ +#define UCT_ZE_IPC_CACHE_H_ + +#include +#include +#include +#include +#include "ze_ipc_md.h" + + +typedef struct uct_ze_ipc_cache uct_ze_ipc_cache_t; +typedef struct uct_ze_ipc_cache_region uct_ze_ipc_cache_region_t; +typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; + + +typedef ucs_status_t +(*uct_ze_ipc_cache_open_func_t)(uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + void **mapped_addr, int *dup_fd); + +typedef ucs_status_t +(*uct_ze_ipc_cache_close_func_t)(ze_context_handle_t ze_context, + void *mapped_addr, int dup_fd); + +typedef struct uct_ze_ipc_cache_ops { + uct_ze_ipc_cache_open_func_t open; + uct_ze_ipc_cache_close_func_t close; +} uct_ze_ipc_cache_ops_t; + + +/** + * Cache region structure for storing mapped IPC handles + */ +struct uct_ze_ipc_cache_region { + ucs_pgt_region_t super; /**< Base class - page table region */ + ucs_list_link_t list; /**< List element */ + uct_ze_ipc_key_t key; /**< Remote memory key */ + void *mapped_addr; /**< Local mapped address */ + uint64_t refcount; /**< Track in-flight ops before unmapping*/ + ze_context_handle_t ze_context; /**< Level Zero context */ + int dup_fd; /**< Duplicated file descriptor */ +}; + + +/** + * Cache structure for managing IPC handle mappings + */ +struct uct_ze_ipc_cache { + pthread_rwlock_t lock; /**< Protects the page table and orphan list */ + ucs_pgtable_t pgtable; /**< Page table to hold the regions */ + ucs_list_link_t orphan_list; /**< Regions dropped from the page table + while still referenced by in-flight + operations */ + uct_ze_ipc_cache_ops_t ops; /**< Mapping operations */ + char *name; /**< Name for debugging */ +}; + + +/** + * Create a new IPC cache + * + * @param cache Pointer to store the created cache + * @param name Name for the cache (for debugging) + * @return UCS_OK on success, error code otherwise + */ +ucs_status_t +uct_ze_ipc_create_cache(uct_ze_ipc_cache_t **cache, const char *name); + + +/** + * Destroy an IPC cache + * + * @param cache Cache to destroy + */ +void uct_ze_ipc_destroy_cache(uct_ze_ipc_cache_t *cache); + + +/** + * Map an IPC memory handle to a local address (with caching) + * + * @param key Remote memory key containing IPC handle + * @param ze_context Level Zero context + * @param ze_device Level Zero device + * @param ops Mapping operations, or NULL to use Level Zero + * @param mapped_addr Pointer to store the mapped address + * @param dup_fd Pointer to store the duplicated file descriptor + * @return UCS_OK on success, error code otherwise + */ +ucs_status_t uct_ze_ipc_map_memhandle(uct_ze_ipc_key_t *key, + ze_context_handle_t ze_context, + ze_device_handle_t ze_device, + const uct_ze_ipc_cache_ops_t *ops, + void **mapped_addr, int *dup_fd); + + +/** + * Unmap an IPC memory handle (with caching) + * + * @param pid Process ID of the remote process + * @param address Base address of the remote memory + * @param mapped_addr Mapped local address + * @param ze_context Level Zero context + * @param dup_fd Duplicated file descriptor + * @param cache_enabled Whether caching is enabled + * @return UCS_OK on success, error code otherwise + */ +ucs_status_t uct_ze_ipc_unmap_memhandle(pid_t pid, uintptr_t address, + void *mapped_addr, + ze_context_handle_t ze_context, + int dup_fd, int cache_enabled); + + +/** + * Purge and free all remote cache entries keyed by the given local Level Zero + * context, so that a closed context does not leave behind cache entries that + * would later be used with a destroyed context. + * + * Mappings are closed regardless of refcount, and cache descriptors are freed. + * The caller must therefore have quiesced this context: all operations issued + * against it must have completed, and no thread may enter map/unmap for it + * concurrently. This matches the UCT requirement that a memory domain have no + * outstanding operations at uct_md_close(), which is the only caller - and it is + * not an extra constraint, since md_close() destroys the context immediately + * afterwards and would invalidate any in-flight copy anyway. + * + * @param ze_context Level Zero context being closed + */ +void uct_ze_ipc_purge_cache_by_context(ze_context_handle_t ze_context); + +#endif /* UCT_ZE_IPC_CACHE_H_ */ diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.c b/src/uct/ze/ze_ipc/ze_ipc_ep.c new file mode 100644 index 00000000000..f1e87a6114b --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.c @@ -0,0 +1,299 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ze_ipc_ep.h" +#include "ze_ipc_iface.h" +#include "ze_ipc_md.h" +#include "ze_ipc_cache.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define UCT_ZE_IPC_PUT 0 +#define UCT_ZE_IPC_GET 1 + +/* + * + * @param remote_pid PID of the process that owns the fd + * @param remote_fd File descriptor in the remote process + * @return Local fd on success, -1 on error + */ +int uct_ze_ipc_dup_fd_from_pid(pid_t remote_pid, int remote_fd) +{ +#if defined(SYS_pidfd_open) && defined(SYS_pidfd_getfd) + int pidfd, local_fd; + + /* Skip if same process */ + if (remote_pid == getpid()) { + return remote_fd; + } + + /* Open pidfd for the remote process */ + pidfd = syscall(SYS_pidfd_open, remote_pid, 0); + if (pidfd < 0) { + ucs_error("ze_ipc: pidfd_open(%d) failed: %m", remote_pid); + return -1; + } + + /* Duplicate the fd from remote process */ + local_fd = syscall(SYS_pidfd_getfd, pidfd, remote_fd, 0); + if (local_fd < 0) { + ucs_error("ze_ipc: pidfd_getfd(pidfd=%d, remote_fd=%d) failed: %m", + pidfd, remote_fd); + close(pidfd); + return -1; + } + + close(pidfd); + ucs_debug("ze_ipc: duplicated fd %d from pid %d -> local fd %d (uncached)", + remote_fd, remote_pid, local_fd); + return local_fd; +#else + ucs_error("ze_ipc: pidfd_getfd not supported on this system"); + return -1; +#endif +} + + +static UCS_CLASS_INIT_FUNC(uct_ze_ipc_ep_t, const uct_ep_params_t *params) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(params->iface, + uct_ze_ipc_iface_t); + + UCT_EP_PARAMS_CHECK_DEV_IFACE_ADDRS(params); + UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super); + + self->remote_pid = *(const pid_t*)params->iface_addr; + + ucs_debug("ze_ipc_ep: created endpoint to remote pid %d (local pid %d)", + self->remote_pid, getpid()); + return UCS_OK; +} + + +static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_ep_t) +{ +} + + +UCS_CLASS_DEFINE(uct_ze_ipc_ep_t, uct_base_ep_t) +UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t*); +UCS_CLASS_DEFINE_DELETE_FUNC(uct_ze_ipc_ep_t, uct_ep_t); + + +#define uct_ze_ipc_trace_data(_addr, _rkey, _fmt, ...) \ + ucs_trace_data(_fmt " to %" PRIx64 "(%+ld)", ##__VA_ARGS__, (_addr), \ + (_rkey)) + + +int uct_ze_ipc_ep_is_connected(const uct_ep_h tl_ep, + const uct_ep_is_connected_params_t *params) +{ + const uct_ze_ipc_ep_t *ep = ucs_derived_of(tl_ep, uct_ze_ipc_ep_t); + + if (!uct_base_ep_is_connected(tl_ep, params)) { + return 0; + } + + return ep->remote_pid == *(pid_t*)params->iface_addr; +} + + +static UCS_F_ALWAYS_INLINE ucs_status_t +uct_ze_ipc_post_copy(uct_ep_h tl_ep, uint64_t remote_addr, const uct_iov_t *iov, + uct_rkey_t rkey, uct_completion_t *comp, int direction) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_ep->iface, + uct_ze_ipc_iface_t); + uct_ze_ipc_key_t *key = (uct_ze_ipc_key_t*)rkey; + uct_ze_ipc_event_desc_t *event_desc; + uct_ze_ipc_queue_desc_t *q_desc; + void *mapped_addr = NULL; + void *mapped_rem_addr; + void *dst, *src; + size_t offset; + ze_result_t ret; + ucs_status_t status; + int local_fd; + unsigned cmd_list_idx; + + if (ucs_unlikely(iov[0].length == 0)) { + ucs_trace_data("Zero length request: skip it"); + return UCS_OK; + } + + if (ucs_unlikely((remote_addr < key->address) || + ((offset = remote_addr - key->address) > key->length) || + (iov[0].length > (key->length - offset)))) { + ucs_error("ze_ipc_ep: remote_addr 0x%" PRIx64 " length %zu is out of " + "rkey range [0x%lx..0x%lx)", + remote_addr, iov[0].length, key->address, + key->address + key->length); + return UCS_ERR_INVALID_PARAM; + } + + /* Open the handle on the device whose command lists perform the copy; + * the exporter's device ordinal need not name the same device here */ + status = uct_ze_ipc_map_memhandle(key, iface->ze_context, iface->ze_device, + NULL, &mapped_addr, &local_fd); + if (status != UCS_OK) { + ucs_error("ze_ipc_ep: uct_ze_ipc_map_memhandle failed"); + return status; + } + + ucs_debug("ze_ipc_ep: IPC handle mapped (cached), mapped_addr=%p", + mapped_addr); + + mapped_rem_addr = (void*)((uintptr_t)mapped_addr + offset); + + /* Allocate event descriptor */ + event_desc = ucs_malloc(sizeof(*event_desc), "uct_ze_ipc_event_desc_t"); + if (event_desc == NULL) { + ucs_error("failed to allocate event descriptor"); + uct_ze_ipc_unmap_memhandle(key->pid, key->address, mapped_addr, + iface->ze_context, local_fd, + iface->config.enable_cache); + return UCS_ERR_NO_MEMORY; + } + + status = uct_ze_ipc_alloc_event(iface, event_desc); + if (status != UCS_OK) { + ucs_free(event_desc); + uct_ze_ipc_unmap_memhandle(key->pid, key->address, mapped_addr, + iface->ze_context, local_fd, + iface->config.enable_cache); + return status; + } + + /* Store information for cache-based cleanup and event tracking */ + event_desc->dup_fd = local_fd; + event_desc->pid = key->pid; + event_desc->address = key->address; + + /* Set up source and destination based on direction */ + if (direction == UCT_ZE_IPC_PUT) { + dst = mapped_rem_addr; + src = iov[0].buffer; + } else { + dst = iov[0].buffer; + src = mapped_rem_addr; + } + + /* + * Select command list using round-robin scheduling + * Similar to CUDA IPC's stream selection: key->stream_id % iface->config.max_streams + * This distributes copy operations across multiple command lists for parallel execution + */ + cmd_list_idx = iface->next_cmd_list % iface->num_cmd_lists; + iface->next_cmd_list++; + + q_desc = &iface->queue_desc[cmd_list_idx]; + + /* + * Append memory copy to selected immediate command list with event signaling. + * Immediate command lists execute asynchronously without needing + * explicit close/execute/reset calls. + * + * Using event for completion tracking improves performance by: + * - Allowing async progress checking + * - Enabling proper completion callbacks + * - Supporting multiple outstanding operations + */ + ret = zeCommandListAppendMemoryCopy(q_desc->cmd_list, dst, src, + iov[0].length, event_desc->event, 0, + NULL); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeCommandListAppendMemoryCopy failed with error 0x%x", ret); + goto err_cleanup; + } + + /* Store event info for progress tracking */ + event_desc->mapped_addr = mapped_addr; + event_desc->comp = comp; + + /* Add to active queue if this is the first event for this command list */ + if (ucs_queue_is_empty(&q_desc->event_queue)) { + ucs_queue_push(&iface->active_queue, &q_desc->queue); + } + + /* Push event to this command list's event queue */ + ucs_queue_push(&q_desc->event_queue, &event_desc->queue); + + ucs_trace("zeCommandListAppendMemoryCopy issued (async): " + "cmd_list[%u/%u]=%p dst=%p src=%p len=%zu", + cmd_list_idx, iface->num_cmd_lists, q_desc->cmd_list, dst, src, + iov[0].length); + + return UCS_INPROGRESS; + +err_cleanup: + uct_ze_ipc_free_event(iface, event_desc); + ucs_free(event_desc); + uct_ze_ipc_unmap_memhandle(key->pid, key->address, mapped_addr, + iface->ze_context, local_fd, + iface->config.enable_cache); + return UCS_ERR_IO_ERROR; +} + +UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_get_zcopy, + (tl_ep, iov, iovcnt, remote_addr, rkey, comp), uct_ep_h tl_ep, + const uct_iov_t *iov, size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp) +{ + ucs_status_t status; + + UCT_CHECK_IOV_SIZE(iovcnt, 1ul, "uct_ze_ipc_ep_get_zcopy"); + + status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, + UCT_ZE_IPC_GET); + if (UCS_STATUS_IS_ERR(status)) { + ucs_error("ze_ipc_ep: GET_ZCOPY failed with status %s", + ucs_status_string(status)); + return status; + } + + UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), GET, ZCOPY, + uct_iov_total_length(iov, iovcnt)); + uct_ze_ipc_trace_data(remote_addr, rkey, "GET_ZCOPY [length %zu]", + uct_iov_total_length(iov, iovcnt)); + return status; +} + + +UCS_PROFILE_FUNC(ucs_status_t, uct_ze_ipc_ep_put_zcopy, + (tl_ep, iov, iovcnt, remote_addr, rkey, comp), uct_ep_h tl_ep, + const uct_iov_t *iov, size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp) +{ + ucs_status_t status; + + UCT_CHECK_IOV_SIZE(iovcnt, 1ul, "uct_ze_ipc_ep_put_zcopy"); + + status = uct_ze_ipc_post_copy(tl_ep, remote_addr, iov, rkey, comp, + UCT_ZE_IPC_PUT); + if (UCS_STATUS_IS_ERR(status)) { + ucs_error("ze_ipc_ep: PUT_ZCOPY failed with status %s", + ucs_status_string(status)); + return status; + } + + UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), PUT, ZCOPY, + uct_iov_total_length(iov, iovcnt)); + uct_ze_ipc_trace_data(remote_addr, rkey, "PUT_ZCOPY [length %zu]", + uct_iov_total_length(iov, iovcnt)); + return status; +} diff --git a/src/uct/ze/ze_ipc/ze_ipc_ep.h b/src/uct/ze/ze_ipc/ze_ipc_ep.h new file mode 100644 index 00000000000..a737dd63e09 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_ep.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_EP_H +#define UCT_ZE_IPC_EP_H + +#include +#include +#include + +#include + +/* Forward declaration */ +typedef struct uct_ze_ipc_iface uct_ze_ipc_iface_t; + + +typedef struct uct_ze_ipc_ep { + uct_base_ep_t super; + pid_t remote_pid; +} uct_ze_ipc_ep_t; + + +UCS_CLASS_DECLARE_NEW_FUNC(uct_ze_ipc_ep_t, uct_ep_t, const uct_ep_params_t*); +UCS_CLASS_DECLARE_DELETE_FUNC(uct_ze_ipc_ep_t, uct_ep_t); + +ucs_status_t uct_ze_ipc_ep_get_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, + size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp); + +ucs_status_t uct_ze_ipc_ep_put_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, + size_t iovcnt, uint64_t remote_addr, + uct_rkey_t rkey, uct_completion_t *comp); + +int uct_ze_ipc_ep_is_connected(const uct_ep_h tl_ep, + const uct_ep_is_connected_params_t *params); + +int uct_ze_ipc_dup_fd_from_pid(pid_t remote_pid, int remote_fd); + +#endif diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.c b/src/uct/ze/ze_ipc/ze_ipc_iface.c new file mode 100644 index 00000000000..c25eb04f789 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.c @@ -0,0 +1,863 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ze_ipc_iface.h" +#include "ze_ipc_ep.h" +#include "ze_ipc_cache.h" + +#include +#include +#include +#include + +#include +#include + + +static ucs_config_field_t uct_ze_ipc_iface_config_table[] = { + {"", "", NULL, ucs_offsetof(uct_ze_ipc_iface_config_t, super), + UCS_CONFIG_TYPE_TABLE(uct_iface_config_table)}, + + {"MAX_POLL", "16", + "Max number of event completions to pick during ze events polling", + ucs_offsetof(uct_ze_ipc_iface_config_t, max_poll), UCS_CONFIG_TYPE_UINT}, + + {"MAX_CMD_LISTS", UCS_PP_MAKE_STRING(UCT_ZE_IPC_MAX_PEERS), + "Max number of command lists (upper limit, actual count matches copy " + "engine queues)", + ucs_offsetof(uct_ze_ipc_iface_config_t, max_cmd_lists), + UCS_CONFIG_TYPE_UINT}, + + {"ENABLE_CACHE", "yes", "Enable IPC handle caching to improve performance", + ucs_offsetof(uct_ze_ipc_iface_config_t, enable_cache), + UCS_CONFIG_TYPE_BOOL}, + + {"EVENT_POOL_SIZE", "1024", + "Number of events in the shared event pool. Operations in flight beyond " + "this number use a slower per-operation event pool", + ucs_offsetof(uct_ze_ipc_iface_config_t, event_pool_size), + UCS_CONFIG_TYPE_UINT}, + + {"BW", "50000MBs", "Effective p2p memory bandwidth", + ucs_offsetof(uct_ze_ipc_iface_config_t, bandwidth), UCS_CONFIG_TYPE_BW}, + + {"LAT", "1.8us", "Estimated latency", + ucs_offsetof(uct_ze_ipc_iface_config_t, latency), UCS_CONFIG_TYPE_TIME}, + + {"OVERHEAD", "4.0us", "Estimated CPU overhead for transferring GPU memory", + ucs_offsetof(uct_ze_ipc_iface_config_t, overhead), UCS_CONFIG_TYPE_TIME}, + + {NULL} +}; + + +/* Forward declaration for the delete function */ +static void UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t)(uct_iface_t*); + + +static ucs_status_t uct_ze_ipc_iface_get_device_address(uct_iface_t *tl_iface, + uct_device_addr_t *addr) +{ + *(uint64_t*)addr = ucs_get_system_id(); + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_iface_get_address(uct_iface_h tl_iface, uct_iface_addr_t *iface_addr) +{ + *(pid_t*)iface_addr = getpid(); + return UCS_OK; +} + + +static int +uct_ze_ipc_iface_is_reachable_v2(const uct_iface_h tl_iface, + const uct_iface_is_reachable_params_t *params) +{ + uint64_t *dev_addr; + pid_t remote_pid; + + if (!uct_iface_is_reachable_params_addrs_valid(params)) { + return 0; + } + + dev_addr = (uint64_t*)params->device_addr; + remote_pid = *(pid_t*)params->iface_addr; + + /* Reject same process */ + if (remote_pid == getpid()) { + uct_iface_fill_info_str_buf(params, "same process"); + return 0; + } + + /* Check same system */ + if (ucs_get_system_id() != *dev_addr) { + uct_iface_fill_info_str_buf(params, "different system"); + return 0; + } + + return uct_iface_scope_is_reachable(tl_iface, params); +} + +static ucs_status_t +uct_ze_ipc_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + + uct_base_iface_query(&iface->super, iface_attr); + + iface_attr->iface_addr_len = sizeof(pid_t); + iface_attr->device_addr_len = sizeof(uint64_t); + iface_attr->ep_addr_len = 0; + iface_attr->max_conn_priv = 0; + iface_attr->cap.flags = UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE | + UCT_IFACE_FLAG_CONNECT_TO_IFACE | + UCT_IFACE_FLAG_PENDING | UCT_IFACE_FLAG_GET_ZCOPY | + UCT_IFACE_FLAG_PUT_ZCOPY; + + iface_attr->cap.put.max_short = 0; + iface_attr->cap.put.max_bcopy = 0; + iface_attr->cap.put.min_zcopy = 0; + iface_attr->cap.put.max_zcopy = ULONG_MAX; + iface_attr->cap.put.opt_zcopy_align = 1; + iface_attr->cap.put.align_mtu = iface_attr->cap.put.opt_zcopy_align; + iface_attr->cap.put.max_iov = 1; + + iface_attr->cap.get.max_bcopy = 0; + iface_attr->cap.get.min_zcopy = 0; + iface_attr->cap.get.max_zcopy = ULONG_MAX; + iface_attr->cap.get.opt_zcopy_align = 1; + iface_attr->cap.get.align_mtu = iface_attr->cap.get.opt_zcopy_align; + iface_attr->cap.get.max_iov = 1; + + iface_attr->latency = ucs_linear_func_make(1e-6, 0); + iface_attr->bandwidth.dedicated = 0; + iface_attr->bandwidth.shared = iface->config.bandwidth; + iface_attr->overhead = 7.0e-6; + iface_attr->priority = 0; + + return UCS_OK; +} + + +static ucs_status_t uct_ze_ipc_create_event(ze_event_pool_handle_t event_pool, + unsigned index, + ze_event_handle_t *event_p) +{ + ze_event_desc_t event_desc = { + .stype = ZE_STRUCTURE_TYPE_EVENT_DESC, + .index = index, + .signal = ZE_EVENT_SCOPE_FLAG_HOST, + .wait = ZE_EVENT_SCOPE_FLAG_HOST + }; + + return UCT_ZE_FUNC_LOG_ERR(zeEventCreate(event_pool, &event_desc, event_p)); +} + + +ucs_status_t uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc) +{ + ze_event_pool_desc_t pool_desc = { + .stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, + .flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE, + .count = 1 + }; + unsigned i, word_idx, bit_idx; + uint64_t mask; + ucs_status_t status; + + ucs_spin_lock(&iface->event_lock); + + /* Find first free event in bitmap */ + for (i = 0; i < iface->event_pool_size; i++) { + word_idx = i / 64; + bit_idx = i % 64; + mask = 1ULL << bit_idx; + + if (!(iface->event_bitmap[word_idx] & mask)) { + /* Found free event, mark as used */ + iface->event_bitmap[word_idx] |= mask; + ucs_spin_unlock(&iface->event_lock); + + status = uct_ze_ipc_create_event(iface->ze_event_pool, i, + &event_desc->event); + if (status != UCS_OK) { + /* Mark as free again */ + ucs_spin_lock(&iface->event_lock); + iface->event_bitmap[word_idx] &= ~mask; + ucs_spin_unlock(&iface->event_lock); + return status; + } + + event_desc->event_pool = NULL; + event_desc->event_index = i; + return UCS_OK; + } + } + + ucs_spin_unlock(&iface->event_lock); + + /* ep_pending_add cannot queue the operation, so UCS_ERR_NO_RESOURCE here + * would make UCP retry the send in a tight loop */ + ucs_trace("ze_ipc: event pool of %u events exhausted, using a private " + "event pool", + iface->event_pool_size); + + status = UCT_ZE_FUNC_LOG_ERR( + zeEventPoolCreate(iface->ze_context, &pool_desc, 1, + &iface->ze_device, &event_desc->event_pool)); + if (status != UCS_OK) { + return status; + } + + status = uct_ze_ipc_create_event(event_desc->event_pool, 0, + &event_desc->event); + if (status != UCS_OK) { + zeEventPoolDestroy(event_desc->event_pool); + return status; + } + + event_desc->event_index = (unsigned)-1; + return UCS_OK; +} + + +void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc) +{ + unsigned word_idx, bit_idx; + uint64_t mask; + + zeEventDestroy(event_desc->event); + + if (event_desc->event_pool != NULL) { + zeEventPoolDestroy(event_desc->event_pool); + return; + } + + word_idx = event_desc->event_index / 64; + bit_idx = event_desc->event_index % 64; + mask = 1ULL << bit_idx; + + ucs_spin_lock(&iface->event_lock); + iface->event_bitmap[word_idx] &= ~mask; + ucs_spin_unlock(&iface->event_lock); +} + +/* Completes events in posting order, so that a flush marker completes only + * after the completions of the copies posted before it. An event leaves the + * queue only after its completion runs, so a completion that posts on the same + * command list does not find the queue empty and add it to the active queue + * a second time */ +static unsigned +uct_ze_ipc_progress_event_queue(uct_ze_ipc_iface_t *iface, + ucs_queue_head_t *event_queue, + unsigned max_events) +{ + unsigned count = 0; + uct_ze_ipc_event_desc_t *event_desc; + ucs_status_t status; + ze_result_t ret; + + while ((count < max_events) && !ucs_queue_is_empty(event_queue)) { + event_desc = ucs_queue_head_elem_non_empty(event_queue, + uct_ze_ipc_event_desc_t, + queue); + if (event_desc->event == NULL) { + /* A flush marker is done once it reaches the head */ + ret = ZE_RESULT_SUCCESS; + } else { + ret = zeEventQueryStatus(event_desc->event); + if (ret == ZE_RESULT_NOT_READY) { + break; + } + + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeEventQueryStatus failed with error 0x%x", ret); + } + } + + /* Unmap IPC handle using cache */ + if (event_desc->mapped_addr != NULL) { + status = uct_ze_ipc_unmap_memhandle(event_desc->pid, + event_desc->address, + event_desc->mapped_addr, + iface->ze_context, + event_desc->dup_fd, + iface->config.enable_cache); + if (status != UCS_OK) { + ucs_warn("failed to unmap IPC handle addr:%p", + event_desc->mapped_addr); + } + } + + if (event_desc->comp != NULL) { + uct_invoke_completion(event_desc->comp, + (ret == ZE_RESULT_SUCCESS) ? + UCS_OK : + UCS_ERR_IO_ERROR); + } + + ucs_queue_pull_non_empty(event_queue); + if (event_desc->event != NULL) { + uct_ze_ipc_free_event(iface, event_desc); + } + + ucs_free(event_desc); + count++; + } + + return count; +} + +static unsigned uct_ze_ipc_iface_progress(uct_iface_h tl_iface) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + unsigned max_poll = iface->config.max_poll; + unsigned count = 0; + uct_ze_ipc_queue_desc_t *q_desc; + ucs_queue_iter_t iter; + + ucs_queue_for_each_safe(q_desc, iter, &iface->active_queue, queue) { + count += uct_ze_ipc_progress_event_queue(iface, &q_desc->event_queue, + max_poll - count); + if (ucs_queue_is_empty(&q_desc->event_queue)) { + ucs_queue_del_iter(&iface->active_queue, iter); + } + } + + return count; +} + + +static void uct_ze_ipc_flush_desc_complete(uct_completion_t *self) +{ + uct_ze_ipc_flush_desc_t *flush_desc = + ucs_container_of(self, uct_ze_ipc_flush_desc_t, super); + + uct_invoke_completion(flush_desc->comp, self->status); + ucs_free(flush_desc); +} + + +/* A marker queued behind the events of each active command list reaches the + * head of its queue only after those events complete, and the last marker to + * get there completes the flush */ +static ucs_status_t +uct_ze_ipc_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_ep->iface, + uct_ze_ipc_iface_t); + uct_ze_ipc_event_desc_t *markers[UCT_ZE_IPC_MAX_PEERS]; + uct_ze_ipc_flush_desc_t *flush_desc; + uct_ze_ipc_queue_desc_t *q_desc; + unsigned i, num_markers; + + if (ucs_queue_is_empty(&iface->active_queue)) { + UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t)); + return UCS_OK; + } + + if (comp == NULL) { + UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); + return UCS_INPROGRESS; + } + + flush_desc = ucs_malloc(sizeof(*flush_desc), "uct_ze_ipc_flush_desc_t"); + if (flush_desc == NULL) { + ucs_error("failed to allocate flush descriptor"); + return UCS_ERR_NO_MEMORY; + } + + num_markers = 0; + ucs_queue_for_each(q_desc, &iface->active_queue, queue) { + ucs_assert(num_markers < UCT_ZE_IPC_MAX_PEERS); + markers[num_markers] = ucs_malloc(sizeof(*markers[num_markers]), + "uct_ze_ipc_flush_marker"); + if (markers[num_markers] == NULL) { + ucs_error("failed to allocate flush marker"); + goto err_free_markers; + } + + ++num_markers; + } + + flush_desc->super.func = uct_ze_ipc_flush_desc_complete; + flush_desc->super.count = num_markers; + flush_desc->super.status = UCS_OK; + flush_desc->comp = comp; + + i = 0; + ucs_queue_for_each(q_desc, &iface->active_queue, queue) { + markers[i]->event = NULL; + markers[i]->event_pool = NULL; + markers[i]->mapped_addr = NULL; + markers[i]->comp = &flush_desc->super; + ucs_queue_push(&q_desc->event_queue, &markers[i]->queue); + ++i; + } + + UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); + return UCS_INPROGRESS; + +err_free_markers: + for (i = 0; i < num_markers; ++i) { + ucs_free(markers[i]); + } + + ucs_free(flush_desc); + return UCS_ERR_NO_MEMORY; +} + + +static ucs_status_t uct_ze_ipc_iface_flush(uct_iface_h tl_iface, unsigned flags, + uct_completion_t *comp) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + unsigned i; + + if (comp != NULL) { + return UCS_ERR_UNSUPPORTED; + } + + /* Check if all command list queues are empty */ + for (i = 0; i < iface->num_cmd_lists; i++) { + if (!ucs_queue_is_empty(&iface->queue_desc[i].event_queue)) { + UCT_TL_IFACE_STAT_FLUSH_WAIT( + ucs_derived_of(tl_iface, uct_base_iface_t)); + return UCS_INPROGRESS; + } + } + + UCT_TL_IFACE_STAT_FLUSH(ucs_derived_of(tl_iface, uct_base_iface_t)); + return UCS_OK; +} + + +static uct_iface_ops_t uct_ze_ipc_iface_ops = { + .ep_get_zcopy = uct_ze_ipc_ep_get_zcopy, + .ep_put_zcopy = uct_ze_ipc_ep_put_zcopy, + .ep_pending_add = (uct_ep_pending_add_func_t)ucs_empty_function_return_busy, + .ep_pending_purge = (uct_ep_pending_purge_func_t)ucs_empty_function, + .ep_flush = uct_ze_ipc_ep_flush, + .ep_fence = uct_base_ep_fence, + .ep_check = (uct_ep_check_func_t)ucs_empty_function_return_unsupported, + .ep_create = UCS_CLASS_NEW_FUNC_NAME(uct_ze_ipc_ep_t), + .ep_destroy = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_ep_t), + .iface_flush = uct_ze_ipc_iface_flush, + .iface_fence = uct_base_iface_fence, + .iface_progress_enable = uct_base_iface_progress_enable, + .iface_progress_disable = uct_base_iface_progress_disable, + .iface_progress = uct_ze_ipc_iface_progress, + .iface_event_fd_get = (uct_iface_event_fd_get_func_t) + ucs_empty_function_return_unsupported, + .iface_event_arm = (uct_iface_event_arm_func_t) + ucs_empty_function_return_unsupported, + .iface_close = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_ipc_iface_t), + .iface_query = uct_ze_ipc_iface_query, + .iface_get_device_address = uct_ze_ipc_iface_get_device_address, + .iface_get_address = uct_ze_ipc_iface_get_address, + .iface_is_reachable = uct_base_iface_is_reachable, +}; + + +static ucs_status_t +uct_ze_ipc_estimate_perf(uct_iface_h tl_iface, uct_perf_attr_t *perf_attr) +{ + uct_ze_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_ze_ipc_iface_t); + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_BANDWIDTH) { + perf_attr->bandwidth.dedicated = 0; + perf_attr->bandwidth.shared = iface->config.bandwidth; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_PATH_BANDWIDTH) { + perf_attr->path_bandwidth.dedicated = 0; + perf_attr->path_bandwidth.shared = iface->config.bandwidth; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_SEND_PRE_OVERHEAD) { + perf_attr->send_pre_overhead = iface->config.overhead; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_SEND_POST_OVERHEAD) { + perf_attr->send_post_overhead = 0; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_RECV_OVERHEAD) { + perf_attr->recv_overhead = 0; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_LATENCY) { + perf_attr->latency = ucs_linear_func_make(iface->config.latency, 0.0); + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_MAX_INFLIGHT_EPS) { + perf_attr->max_inflight_eps = SIZE_MAX; + } + + if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_FLAGS) { + perf_attr->flags = 0; + } + + return UCS_OK; +} + +static uct_iface_internal_ops_t uct_ze_ipc_iface_internal_ops = { + .iface_query_v2 = uct_iface_base_query_v2, + .iface_estimate_perf = uct_ze_ipc_estimate_perf, + .iface_vfs_refresh = (uct_iface_vfs_refresh_func_t)ucs_empty_function, + .ep_query = (uct_ep_query_func_t)ucs_empty_function_return_unsupported, + .ep_invalidate = (uct_ep_invalidate_func_t) + ucs_empty_function_return_unsupported, + .ep_connect_to_ep_v2 = (uct_ep_connect_to_ep_v2_func_t) + ucs_empty_function_return_unsupported, + .iface_is_reachable_v2 = uct_ze_ipc_iface_is_reachable_v2, + .ep_is_connected = uct_ze_ipc_ep_is_connected, + .ep_outstanding_purge = (uct_ep_outstanding_purge_func_t) + ucs_empty_function_return_unsupported +}; + + +/* Find the copy engine queue group ordinal for a device */ +static ucs_status_t uct_ze_ipc_find_copy_ordinal(ze_device_handle_t device, + uint32_t *ordinal_p, + uint32_t *num_queues_p) +{ + ze_command_queue_group_properties_t queue_props[16]; + uint32_t num_queue_groups = 16; + uint32_t i; + ze_result_t ret; + + /* Initialize structure types */ + for (i = 0; i < 16; i++) { + queue_props[i].stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES; + queue_props[i].pNext = NULL; + } + + /* Get queue group properties */ + ret = zeDeviceGetCommandQueueGroupProperties(device, &num_queue_groups, + queue_props); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_ipc_iface: zeDeviceGetCommandQueueGroupProperties " + "failed: 0x%x", + ret); + return UCS_ERR_IO_ERROR; + } + + /* First pass: find dedicated copy engine (COPY flag but NOT COMPUTE) */ + for (i = 0; i < num_queue_groups; i++) { + if ((queue_props[i].flags & + ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) && + !(queue_props[i].flags & + ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE)) { + *ordinal_p = i; + *num_queues_p = queue_props[i].numQueues; + ucs_debug("ze_ipc_iface: using dedicated copy engine queue group " + "%u (numQueues=%u)", + i, queue_props[i].numQueues); + return UCS_OK; + } + } + + /* Second pass: find any queue group that supports COPY */ + for (i = 0; i < num_queue_groups; i++) { + if (queue_props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) { + *ordinal_p = i; + *num_queues_p = queue_props[i].numQueues; + ucs_debug("ze_ipc_iface: using copy-capable queue group %u " + "(numQueues=%u)", + i, queue_props[i].numQueues); + return UCS_OK; + } + } + + ucs_error("ze_ipc_iface: no copy-capable queue group found"); + return UCS_ERR_NO_DEVICE; +} + +static UCS_CLASS_INIT_FUNC(uct_ze_ipc_iface_t, uct_md_h md, uct_worker_h worker, + const uct_iface_params_t *params, + const uct_iface_config_t *tl_config) +{ + uct_ze_ipc_iface_config_t *config; + uct_ze_ipc_md_t *ze_md; + ze_command_queue_desc_t queue_desc = {}; + ze_event_pool_desc_t event_pool_desc; + uint32_t copy_ordinal = 0; + uint32_t num_queues = 0; + ucs_status_t status; + ze_result_t ret; + unsigned i; + size_t bitmap_size; + + config = ucs_derived_of(tl_config, uct_ze_ipc_iface_config_t); + ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + + UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_ze_ipc_iface_ops, + &uct_ze_ipc_iface_internal_ops, md, worker, + params, + tl_config UCS_STATS_ARG(params->stats_root) + UCS_STATS_ARG(UCT_ZE_IPC_TL_NAME)); + + self->ze_context = ze_md->ze_context; + self->ze_device = ze_md->ze_device; + self->config = *config; + self->next_cmd_list = 0; + + /* Find copy engine queue group ordinal and available queue count */ + status = uct_ze_ipc_find_copy_ordinal(self->ze_device, ©_ordinal, + &num_queues); + if (status != UCS_OK) { + return status; + } + + /* + * Determine number of command lists based on hardware Copy Engine availability. + * + * Strategy: + * 1. If dedicated Copy Engines exist: create one command list per Copy Engine queue + * 2. If no Copy Engines: create 1 command list (fallback to compute engine) + * 3. Respect user configuration limit (MAX_CMD_LISTS) as upper bound + * + * Rationale: + * - Each Copy Engine queue can execute independently in parallel + * - Creating more command lists than hardware queues provides no benefit + * - One command list per Copy Engine queue maximizes hardware utilization + */ + if (num_queues == 0) { + /* No copy engines available, use single command list on compute engine */ + self->num_cmd_lists = 1; + ucs_debug("ze_ipc_iface: no dedicated copy engines found, using 1 " + "command list on queue group %u (compute engine fallback)", + copy_ordinal); + } else { + /* Match command list count to available Copy Engine queues */ + self->num_cmd_lists = ucs_min(num_queues, config->max_cmd_lists); + + if (self->num_cmd_lists < num_queues) { + ucs_debug("ze_ipc_iface: limiting command lists to %u (hardware " + "has %u copy queues, config limit is %u)", + self->num_cmd_lists, num_queues, config->max_cmd_lists); + } else { + ucs_debug("ze_ipc_iface: creating %u command list(s) to match %u " + "copy engine queue(s) on queue group %u", + self->num_cmd_lists, num_queues, copy_ordinal); + } + } + + /* Validate final count */ + if (self->num_cmd_lists == 0) { + ucs_error("ze_ipc_iface: MAX_CMD_LISTS=0 is invalid, at least 1 " + "command list is required"); + return UCS_ERR_INVALID_PARAM; + } + if (self->num_cmd_lists > UCT_ZE_IPC_MAX_PEERS) { + ucs_error("ze_ipc_iface: computed num_cmd_lists (%u) exceeds maximum " + "(%u)", + self->num_cmd_lists, UCT_ZE_IPC_MAX_PEERS); + return UCS_ERR_INVALID_PARAM; + } + if (config->event_pool_size == 0) { + ucs_error("ze_ipc_iface: EVENT_POOL_SIZE must be at least 1"); + return UCS_ERR_INVALID_PARAM; + } + + /* Initialize active queue for command lists with pending operations */ + ucs_queue_head_init(&self->active_queue); + + /* + * Create multiple immediate command lists for parallel progress. + * + * According to Level Zero spec, for immediate command lists: + * - queue_desc.index MUST be 0 (not 1, 2, 3...) + * - Multiple immediate command lists with index=0 are allowed + * - Driver manages parallel execution automatically + * + * Benefits of multiple immediate command lists: + * - Round-robin distribution of IPC copy operations + * - Reduced contention on single command list + * - Better hardware utilization for concurrent operations + * - Independent event queues for each command list + */ + queue_desc.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC; + queue_desc.ordinal = copy_ordinal; + queue_desc.mode = ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS; + queue_desc.index = 0; /* MUST be 0 for immediate command lists */ + queue_desc.flags = 0; + queue_desc.priority = ZE_COMMAND_QUEUE_PRIORITY_NORMAL; + + for (i = 0; i < self->num_cmd_lists; i++) { + ret = zeCommandListCreateImmediate(self->ze_context, self->ze_device, + &queue_desc, + &self->queue_desc[i].cmd_list); + if (ret != ZE_RESULT_SUCCESS) { + unsigned j; + ucs_error("ze_ipc_iface: zeCommandListCreateImmediate[%u] failed " + "with error 0x%x", + i, ret); + /* Clean up previously created command lists */ + for (j = 0; j < i; j++) { + zeCommandListDestroy(self->queue_desc[j].cmd_list); + } + return UCS_ERR_IO_ERROR; + } + + /* Initialize event queue for this command list */ + ucs_queue_head_init(&self->queue_desc[i].event_queue); + + ucs_debug("ze_ipc_iface: created immediate command list %u/%u: %p", + i + 1, self->num_cmd_lists, self->queue_desc[i].cmd_list); + } + + /* + * Create a shared event pool for all copy operations to avoid + * per-operation event pool creation/destruction overhead. + * + * Performance optimization: + * - Single event pool shared across all operations + * - Pre-allocated events reduce allocation overhead + * - Event reuse via bitmap tracking + * - Typical pool size: 1024 events (configurable) + */ + self->event_pool_size = config->event_pool_size; + + event_pool_desc.stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC; + event_pool_desc.pNext = NULL; + event_pool_desc.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE; + event_pool_desc.count = self->event_pool_size; + + ret = zeEventPoolCreate(self->ze_context, &event_pool_desc, 1, + &self->ze_device, &self->ze_event_pool); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("ze_ipc_iface: zeEventPoolCreate failed with error 0x%x", + ret); + /* Clean up command lists */ + for (i = 0; i < self->num_cmd_lists; i++) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + return UCS_ERR_IO_ERROR; + } + + /* Initialize event bitmap for tracking free events */ + bitmap_size = (self->event_pool_size + 63) / + 64; /* Round up to 64-bit words */ + self->event_bitmap = ucs_calloc(bitmap_size, sizeof(uint64_t), + "ze_ipc_event_bitmap"); + if (self->event_bitmap == NULL) { + ucs_error("ze_ipc_iface: failed to allocate event bitmap"); + zeEventPoolDestroy(self->ze_event_pool); + for (i = 0; i < self->num_cmd_lists; i++) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + return UCS_ERR_NO_MEMORY; + } + + /* Initialize spinlock for event allocation */ + status = ucs_spinlock_init(&self->event_lock, 0); + if (status != UCS_OK) { + ucs_error("ze_ipc_iface: failed to initialize event_lock spinlock"); + ucs_free(self->event_bitmap); + zeEventPoolDestroy(self->ze_event_pool); + for (i = 0; i < self->num_cmd_lists; i++) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + return status; + } + + ucs_debug("ze_ipc_iface: initialized iface for device %p context %p pid %d " + "copy ordinal %u, %u command lists, %u events", + self->ze_device, self->ze_context, getpid(), copy_ordinal, + self->num_cmd_lists, self->event_pool_size); + + return UCS_OK; +} + + +static UCS_CLASS_CLEANUP_FUNC(uct_ze_ipc_iface_t) +{ + uct_ze_ipc_event_desc_t *event_desc; + uct_ze_ipc_flush_desc_t *flush_desc; + unsigned i; + + uct_base_iface_progress_disable(&self->super.super, + UCT_PROGRESS_SEND | UCT_PROGRESS_RECV); + + /* Clean up outstanding events from all command list queues */ + for (i = 0; i < self->num_cmd_lists; i++) { + while (!ucs_queue_is_empty(&self->queue_desc[i].event_queue)) { + event_desc = ucs_queue_pull_elem_non_empty( + &self->queue_desc[i].event_queue, uct_ze_ipc_event_desc_t, + queue); + if (event_desc->mapped_addr != NULL) { + uct_ze_ipc_unmap_memhandle(event_desc->pid, event_desc->address, + event_desc->mapped_addr, + self->ze_context, event_desc->dup_fd, + self->config.enable_cache); + } + if (event_desc->event != NULL) { + zeEventDestroy(event_desc->event); + } else { + /* A flush still pending at close is dropped without + * completing it */ + flush_desc = ucs_container_of(event_desc->comp, + uct_ze_ipc_flush_desc_t, super); + if (--flush_desc->super.count == 0) { + ucs_free(flush_desc); + } + } + if (event_desc->event_pool != NULL) { + zeEventPoolDestroy(event_desc->event_pool); + } + ucs_free(event_desc); + } + } + + /* Destroy all command lists */ + for (i = 0; i < self->num_cmd_lists; i++) { + if (self->queue_desc[i].cmd_list != NULL) { + zeCommandListDestroy(self->queue_desc[i].cmd_list); + } + } + /* cmd_queue is not used with immediate command lists */ + + /* Destroy shared event pool */ + if (self->ze_event_pool != NULL) { + zeEventPoolDestroy(self->ze_event_pool); + } + + /* Free event bitmap */ + ucs_free(self->event_bitmap); + + /* Destroy spinlock */ + ucs_spinlock_destroy(&self->event_lock); +} + + +static ucs_status_t +uct_ze_ipc_query_devices(uct_md_h uct_md, + uct_tl_device_resource_t **tl_devices_p, + unsigned *num_tl_devices_p) +{ + return uct_ze_base_query_devices_common(uct_md, UCT_DEVICE_TYPE_ACC, + tl_devices_p, num_tl_devices_p); +} + + +UCS_CLASS_DEFINE(uct_ze_ipc_iface_t, uct_base_iface_t); +UCS_CLASS_DEFINE_NEW_FUNC(uct_ze_ipc_iface_t, uct_iface_t, uct_md_h, + uct_worker_h, const uct_iface_params_t*, + const uct_iface_config_t*); +static UCS_CLASS_DEFINE_DELETE_FUNC(uct_ze_ipc_iface_t, uct_iface_t); + +UCT_TL_DEFINE(&uct_ze_ipc_component, ze_ipc, uct_ze_ipc_query_devices, + uct_ze_ipc_iface_t, "ZE_IPC_", uct_ze_ipc_iface_config_table, + uct_ze_ipc_iface_config_t); diff --git a/src/uct/ze/ze_ipc/ze_ipc_iface.h b/src/uct/ze/ze_ipc/ze_ipc_iface.h new file mode 100644 index 00000000000..6013e15efc0 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_iface.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_IFACE_H +#define UCT_ZE_IPC_IFACE_H + +#include +#include +#include +#include + +#include "ze_ipc_md.h" + + +#define UCT_ZE_IPC_TL_NAME "ze_ipc" +#define UCT_ZE_IPC_MAX_PEERS 16 + + +typedef struct uct_ze_ipc_iface_config { + uct_iface_config_t super; + unsigned max_poll; /* query attempts w.o success */ + unsigned max_cmd_lists; /* max number of command lists for parallel progress */ + int enable_cache; /* enable/disable ipc handle cache */ + unsigned event_pool_size; /* number of events in the shared event pool */ + double bandwidth; /* estimated bandwidth */ + double latency; /* estimated latency */ + double overhead; /* estimated CPU overhead */ +} uct_ze_ipc_iface_config_t; + + +/** + * Queue descriptor for each command list + * Similar to CUDA IPC's uct_cuda_queue_desc_t design + */ +typedef struct uct_ze_ipc_queue_desc { + ucs_queue_elem_t queue; /* element in active queue list */ + ze_command_list_handle_t cmd_list; /* immediate command list */ + ucs_queue_head_t event_queue; /* queue of outstanding events */ +} uct_ze_ipc_queue_desc_t; + + +typedef struct uct_ze_ipc_iface { + uct_base_iface_t super; + ze_context_handle_t ze_context; + ze_device_handle_t ze_device; + uct_ze_ipc_iface_config_t config; + + /* Multi-command-list support for parallel progress */ + uct_ze_ipc_queue_desc_t + queue_desc[UCT_ZE_IPC_MAX_PEERS]; /* array of queue descriptors */ + ucs_queue_head_t active_queue; /* queue of active queue descriptors */ + unsigned num_cmd_lists; /* actual number of command lists */ + unsigned next_cmd_list; /* round-robin index for load balancing */ + + /* Pre-allocated event pool for performance optimization */ + ze_event_pool_handle_t + ze_event_pool; /* shared event pool for all operations */ + unsigned event_pool_size; /* number of events in the pool */ + ucs_spinlock_t event_lock; /* lock for event allocation */ + uint64_t *event_bitmap; /* bitmap to track free events */ +} uct_ze_ipc_iface_t; + + +typedef struct uct_ze_ipc_event_desc { + ze_event_handle_t event; + ze_event_pool_handle_t event_pool; /* NULL when using shared pool */ + void *mapped_addr; + uct_completion_t *comp; + ucs_queue_elem_t queue; + int dup_fd; /* duplicated fd to close, or -1 if none */ + pid_t pid; /* remote process id for cache lookup */ + uintptr_t address; /* remote base address for cache lookup */ + unsigned event_index; /* index in shared event pool, or -1 if using private pool */ +} uct_ze_ipc_event_desc_t; + + +typedef struct uct_ze_ipc_flush_desc { + uct_completion_t super; /* counts the flush markers still queued */ + uct_completion_t *comp; /* user completion of the flush */ +} uct_ze_ipc_flush_desc_t; + + +/** + * Allocate an event from the shared event pool, or from a private one-event + * pool when the shared pool is exhausted + * + * @param iface Pointer to ze_ipc interface + * @param event_desc Descriptor to fill with the event, its pool and index + * @return UCS_OK on success, error status on Level Zero failure + */ +ucs_status_t uct_ze_ipc_alloc_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc); + +/** + * Release an event allocated by uct_ze_ipc_alloc_event + * + * @param iface Pointer to ze_ipc interface + * @param event_desc Descriptor filled by uct_ze_ipc_alloc_event + */ +void uct_ze_ipc_free_event(uct_ze_ipc_iface_t *iface, + uct_ze_ipc_event_desc_t *event_desc); + + +#endif diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.c b/src/uct/ze/ze_ipc/ze_ipc_md.c new file mode 100644 index 00000000000..5d50d5bd912 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_md.c @@ -0,0 +1,314 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ze_ipc_md.h" +#include "ze_ipc_cache.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +static ucs_config_field_t uct_ze_ipc_md_config_table[] = { + {"", "", NULL, ucs_offsetof(uct_ze_ipc_md_config_t, super), + UCS_CONFIG_TYPE_TABLE(uct_md_config_table)}, + + {"DEVICE_ORDINAL", "0", "Ordinal of the GPU device to use for IPC.", + ucs_offsetof(uct_ze_ipc_md_config_t, device_ordinal), UCS_CONFIG_TYPE_INT}, + + {NULL} +}; + +static ucs_status_t uct_ze_ipc_md_query(uct_md_h md, uct_md_attr_v2_t *md_attr) +{ + uct_md_base_md_query(md_attr); + md_attr->rkey_packed_size = sizeof(uct_ze_ipc_key_t); + md_attr->flags = UCT_MD_FLAG_REG | UCT_MD_FLAG_NEED_RKEY; + md_attr->reg_mem_types = UCS_BIT(UCS_MEMORY_TYPE_ZE_DEVICE); + md_attr->cache_mem_types = UCS_BIT(UCS_MEMORY_TYPE_ZE_DEVICE); + md_attr->access_mem_types = UCS_BIT(UCS_MEMORY_TYPE_ZE_DEVICE); + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_mkey_pack(uct_md_h uct_md, uct_mem_h memh, void *address, + size_t length, const uct_md_mkey_pack_params_t *params, + void *mkey_buffer) +{ + uct_ze_ipc_key_t *packed = mkey_buffer; + uct_ze_ipc_key_t *key = memh; + + *packed = *key; + return UCS_OK; +} + + +static ucs_status_t uct_ze_ipc_pack_key(uct_ze_ipc_md_t *md, void *address, + size_t length, uct_ze_ipc_key_t *key) +{ + ze_memory_allocation_properties_t props = { + .stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES + }; + void *base_address; + size_t alloc_size; + unsigned long proc_create_time; + ze_result_t ret; + ucs_status_t status; + int fd; + + /* Get memory allocation properties to verify this is ZE device memory */ + ret = zeMemGetAllocProperties(md->ze_context, address, &props, NULL); + if ((ret != ZE_RESULT_SUCCESS) || (props.type == ZE_MEMORY_TYPE_UNKNOWN)) { + ucs_error("failed to get allocation properties for %p", address); + return UCS_ERR_INVALID_ADDR; + } + + /* Get the base address and allocation size */ + ret = zeMemGetAddressRange(md->ze_context, address, &base_address, + &alloc_size); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("failed to get address range for %p", address); + return UCS_ERR_INVALID_ADDR; + } + + /* Resolve this before exporting the handle: without it the importer cannot + * tell our allocations from those of a dead process whose PID we reused, + * and exporting a key it must then distrust is worse than failing here */ + proc_create_time = ucs_sys_get_proc_create_time(getpid()); + if (proc_create_time == 0) { + ucs_error("failed to get process creation time for pid %d", getpid()); + return UCS_ERR_IO_ERROR; + } + + /* Get IPC handle for the memory */ + status = UCT_ZE_FUNC_LOG_ERR( + zeMemGetIpcHandle(md->ze_context, base_address, &key->ipc_handle)); + if (status != UCS_OK) { + ucs_error("failed to get IPC handle for %p", address); + return status; + } + + /* Peers get our own duplicate of the fd and the handle goes back at once: + * while a handle is held, the driver may close its fd when another device + * accesses the allocation, and then close that fd number again on put */ + fd = fcntl(*(int*)key->ipc_handle.data, F_DUPFD_CLOEXEC, 0); + if (fd < 0) { + ucs_error("failed to duplicate IPC handle fd %d for %p: %m", + *(int*)key->ipc_handle.data, address); + } + + ret = zeMemPutIpcHandle(md->ze_context, key->ipc_handle); + if (ret != ZE_RESULT_SUCCESS) { + ucs_warn("zeMemPutIpcHandle failed with error 0x%x", ret); + } + + if (fd < 0) { + return UCS_ERR_IO_ERROR; + } + + key->export_fd = *(int*)key->ipc_handle.data; + *(int*)key->ipc_handle.data = fd; + + key->pid = getpid(); + key->address = (uintptr_t)base_address; + key->length = alloc_size; + key->alloc_id = props.id; + key->proc_create_time = proc_create_time; + + ucs_trace("packed ipc handle for %p base %p length %zu pid %d", address, + base_address, alloc_size, key->pid); + + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_mem_reg(uct_md_h uct_md, void *address, size_t length, + const uct_md_mem_reg_params_t *params, uct_mem_h *memh_p) +{ + uct_ze_ipc_md_t *md = ucs_derived_of(uct_md, uct_ze_ipc_md_t); + uct_ze_ipc_key_t *key; + ucs_status_t status; + + key = ucs_malloc(sizeof(*key), "uct_ze_ipc_key_t"); + if (key == NULL) { + ucs_error("ze_ipc_md: failed to allocate memory for uct_ze_ipc_key_t"); + return UCS_ERR_NO_MEMORY; + } + + status = uct_ze_ipc_pack_key(md, address, length, key); + if (status != UCS_OK) { + ucs_error("ze_ipc_md: pack_key failed for address=%p status=%s", + address, ucs_status_string(status)); + ucs_free(key); + return status; + } + + *memh_p = key; + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_mem_dereg(uct_md_h uct_md, const uct_md_mem_dereg_params_t *params) +{ + uct_ze_ipc_key_t *key; + + UCT_MD_MEM_DEREG_CHECK_PARAMS(params, 0); + + key = params->memh; + close(*(int*)key->ipc_handle.data); + ucs_free(key); + return UCS_OK; +} + + +static ucs_status_t +uct_ze_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, + const uct_rkey_unpack_params_t *params, + uct_rkey_t *rkey_p, void **handle_p) +{ + uct_ze_ipc_key_t *packed = (uct_ze_ipc_key_t*)rkey_buffer; + uct_ze_ipc_key_t *key; + + key = ucs_malloc(sizeof(uct_ze_ipc_key_t), "uct_ze_ipc_key_t"); + if (key == NULL) { + ucs_error("ze_ipc_md: failed to allocate memory for uct_ze_ipc_key_t"); + return UCS_ERR_NO_MEMORY; + } + + *key = *packed; + *handle_p = NULL; + *rkey_p = (uintptr_t)key; + + return UCS_OK; +} + + +static ucs_status_t uct_ze_ipc_rkey_release(uct_component_t *component, + uct_rkey_t rkey, void *handle) +{ + ucs_assert(handle == NULL); + ucs_free((void*)rkey); + return UCS_OK; +} + + +static void uct_ze_ipc_md_close(uct_md_h uct_md) +{ + uct_ze_ipc_md_t *md = ucs_derived_of(uct_md, uct_ze_ipc_md_t); + + if (md->ze_context != NULL) { + uct_ze_ipc_purge_cache_by_context(md->ze_context); + zeContextDestroy(md->ze_context); + } + ucs_free(md); +} + + +static ucs_status_t +uct_ze_ipc_md_open(uct_component_h component, const char *md_name, + const uct_md_config_t *uct_md_config, uct_md_h *md_p) +{ + static uct_md_ops_t md_ops = { + .close = uct_ze_ipc_md_close, + .query = uct_ze_ipc_md_query, + .mem_alloc = (uct_md_mem_alloc_func_t) + ucs_empty_function_return_unsupported, + .mem_free = (uct_md_mem_free_func_t) + ucs_empty_function_return_unsupported, + .mem_advise = (uct_md_mem_advise_func_t) + ucs_empty_function_return_unsupported, + .mem_reg = uct_ze_ipc_mem_reg, + .mem_dereg = uct_ze_ipc_mem_dereg, + .mem_query = (uct_md_mem_query_func_t) + ucs_empty_function_return_unsupported, + .mkey_pack = uct_ze_ipc_mkey_pack, + .mem_attach = (uct_md_mem_attach_func_t) + ucs_empty_function_return_unsupported, + .detect_memory_type = (uct_md_detect_memory_type_func_t) + ucs_empty_function_return_unsupported, + .mem_elem_pack = (uct_md_mem_elem_pack_func_t) + ucs_empty_function_return_unsupported, + .mem_elem_release = (uct_md_mem_elem_release_func_t) + ucs_empty_function, + }; + uct_ze_ipc_md_config_t *config = ucs_derived_of(uct_md_config, + uct_ze_ipc_md_config_t); + uct_ze_ipc_md_t *md; + ze_driver_handle_t ze_driver; + ze_context_desc_t context_desc = {}; + ze_result_t ret; + + ze_driver = uct_ze_base_get_driver(); + if (ze_driver == NULL) { + return UCS_ERR_NO_DEVICE; + } + + md = ucs_malloc(sizeof(uct_ze_ipc_md_t), "uct_ze_ipc_md_t"); + if (md == NULL) { + ucs_error("failed to allocate memory for uct_ze_ipc_md_t"); + return UCS_ERR_NO_MEMORY; + } + + md->ze_device = uct_ze_base_get_device(config->device_ordinal); + if (md->ze_device == NULL) { + ucs_error("failed to get device at ordinal %d", config->device_ordinal); + ucs_free(md); + return UCS_ERR_NO_DEVICE; + } + + ret = zeContextCreate(ze_driver, &context_desc, &md->ze_context); + if (ret != ZE_RESULT_SUCCESS) { + ucs_error("zeContextCreate failed with error 0x%x", ret); + ucs_free(md); + return UCS_ERR_NO_DEVICE; + } + + md->super.ops = &md_ops; + md->super.component = &uct_ze_ipc_component; + + *md_p = (uct_md_h)md; + return UCS_OK; +} + + +uct_component_t uct_ze_ipc_component = { + .query_md_resources = uct_ze_base_query_md_resources, + .md_open = uct_ze_ipc_md_open, + .cm_open = (uct_component_cm_open_func_t) + ucs_empty_function_return_unsupported, + .rkey_unpack = uct_ze_ipc_rkey_unpack, + .rkey_ptr = (uct_component_rkey_ptr_func_t) + ucs_empty_function_return_unsupported, + .rkey_release = uct_ze_ipc_rkey_release, + .rkey_compare = uct_base_rkey_compare, + .name = "ze_ipc", + .md_config = + { + .name = "ZE-IPC memory domain", + .prefix = "ZE_IPC_", + .table = uct_ze_ipc_md_config_table, + .size = sizeof(uct_ze_ipc_md_config_t), + }, + .cm_config = UCS_CONFIG_EMPTY_GLOBAL_LIST_ENTRY, + .tl_list = UCT_COMPONENT_TL_LIST_INITIALIZER(&uct_ze_ipc_component), + .flags = 0, + .md_vfs_init = (uct_component_md_vfs_init_func_t)ucs_empty_function +}; +UCT_COMPONENT_REGISTER(&uct_ze_ipc_component); diff --git a/src/uct/ze/ze_ipc/ze_ipc_md.h b/src/uct/ze/ze_ipc/ze_ipc_md.h new file mode 100644 index 00000000000..5db80c4e9e2 --- /dev/null +++ b/src/uct/ze/ze_ipc/ze_ipc_md.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) Intel Corporation, 2023-2024. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#ifndef UCT_ZE_IPC_MD_H +#define UCT_ZE_IPC_MD_H + +#include +#include + + +extern uct_component_t uct_ze_ipc_component; + + +/** + * @brief ze ipc MD descriptor + */ +typedef struct uct_ze_ipc_md { + uct_md_t super; /**< Domain info */ + ze_context_handle_t ze_context; /**< Level Zero context */ + ze_device_handle_t ze_device; /**< Level Zero device */ +} uct_ze_ipc_md_t; + + +/** + * @brief ze ipc domain configuration. + */ +typedef struct uct_ze_ipc_md_config { + uct_md_config_t super; + int device_ordinal; +} uct_ze_ipc_md_config_t; + + +/** + * @brief ze ipc remote key for put/get + */ +typedef struct uct_ze_ipc_key { + ze_ipc_mem_handle_t ipc_handle; /**< IPC memory handle */ + uint64_t alloc_id; /**< Driver allocation id, distinguishes + allocations that reuse a VA */ + uint64_t proc_create_time; /**< Remote process creation time, + distinguishes processes that + reuse a PID */ + pid_t pid; /**< Remote process ID (for cache) */ + int export_fd; /**< fd zeMemGetIpcHandle returned in the + remote process, before the handle + was repacked with a dup */ + uintptr_t address; /**< Base address of the allocation */ + size_t length; /**< Size of the allocation */ +} uct_ze_ipc_key_t; + + +#endif diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index d6f676ebd4d..6f9b5c82872 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -341,13 +341,19 @@ endif # HAVE_HIP if HAVE_ZE gtest_SOURCES += \ ucm/ze_hooks.cc \ - uct/ze/test_ze.cc + uct/ze/test_ze.cc \ + uct/ze/test_ze_base.cc \ + uct/ze/test_ze_ipc_md.cc \ + uct/ze/test_ze_ipc_cache.cc \ + uct/ze/test_ze_ipc_rma.cc \ + uct/ze/test_ze_copy_md.cc gtest_CPPFLAGS += \ $(ZE_CPPFLAGS) gtest_LDFLAGS += \ $(ZE_LDFLAGS) gtest_LDADD += \ - $(ZE_LIBS) + $(ZE_LIBS) \ + $(top_builddir)/src/uct/ze/libuct_ze.la endif noinst_HEADERS = \ diff --git a/test/gtest/uct/ze/test_ze_base.cc b/test/gtest/uct/ze/test_ze_base.cc new file mode 100644 index 00000000000..0901d16cba8 --- /dev/null +++ b/test/gtest/uct/ze/test_ze_base.cc @@ -0,0 +1,67 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +} + + +/* + * Unit tests for the helpers added to make ZE_IPC compatible with the + * post-merge upstream uct_ze_base data layout (root device per index). + */ +class test_ze_base : public ucs::test { +}; + + +UCS_TEST_F(test_ze_base, init_succeeds_or_skip) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } +} + + +UCS_TEST_F(test_ze_base, get_num_devices_consistent) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + + int n = uct_ze_base_get_num_devices(); + EXPECT_GE(n, 0); + + /* second call must be stable */ + EXPECT_EQ(n, uct_ze_base_get_num_devices()); +} + + +UCS_TEST_F(test_ze_base, get_device_by_ordinal) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + + int n = uct_ze_base_get_num_devices(); + if (n == 0) { + UCS_TEST_SKIP_R("No Level Zero devices available"); + } + + for (int i = 0; i < n; ++i) { + EXPECT_TRUE(uct_ze_base_get_device(i) != NULL) << "ordinal " << i; + } +} + + +UCS_TEST_F(test_ze_base, get_device_out_of_range) { + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + + int n = uct_ze_base_get_num_devices(); + + EXPECT_TRUE(uct_ze_base_get_device(-1) == NULL); + EXPECT_TRUE(uct_ze_base_get_device(n) == NULL); + EXPECT_TRUE(uct_ze_base_get_device(n + 100) == NULL); +} diff --git a/test/gtest/uct/ze/test_ze_copy_md.cc b/test/gtest/uct/ze/test_ze_copy_md.cc new file mode 100644 index 00000000000..57c8de29b60 --- /dev/null +++ b/test/gtest/uct/ze/test_ze_copy_md.cc @@ -0,0 +1,203 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +#include +#include +} + + +/* + * Smoke tests for the ze_copy MD: lifecycle, capability query, allocation + * and memory-type detection. The ze_copy MD is the path used by UCP for + * VRAM<->host staging on Intel XPU. + */ +class test_ze_copy_md : public ucs::test { +protected: + void SetUp() override + { + ucs::test::SetUp(); + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + if (uct_ze_base_get_num_devices() == 0) { + UCS_TEST_SKIP_R("No Level Zero devices available"); + } + } + + static uct_md_h open_first_md() + { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + ucs_status_t status; + + status = uct_ze_copy_component.query_md_resources( + &uct_ze_copy_component, &resources, &num); + EXPECT_UCS_OK(status); + if (num == 0) { + ucs_free(resources); + return NULL; + } + + uct_md_config_t *md_config = NULL; + status = uct_md_config_read(&uct_ze_copy_component, NULL, NULL, + &md_config); + EXPECT_UCS_OK(status); + + uct_md_h md = NULL; + status = uct_md_open(&uct_ze_copy_component, resources[0].md_name, + md_config, &md); + uct_config_release(md_config); + ucs_free(resources); + + return (status == UCS_OK) ? md : NULL; + } +}; + + +UCS_TEST_F(test_ze_copy_md, component_is_registered) { + EXPECT_STREQ("ze_cpy", uct_ze_copy_component.name); +} + + +UCS_TEST_F(test_ze_copy_md, query_md_resources_returns_at_least_one) { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + + EXPECT_UCS_OK( + uct_ze_copy_component.query_md_resources(&uct_ze_copy_component, + &resources, &num)); + EXPECT_GE(num, 1u); + ucs_free(resources); +} + + +UCS_TEST_F(test_ze_copy_md, open_close_md) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + auto *ze_md = ucs_derived_of(md, uct_ze_copy_md_t); + EXPECT_TRUE(ze_md->ze_device != NULL); + EXPECT_TRUE(ze_md->ze_context != NULL); + + uct_md_close(md); +} + + +UCS_TEST_F(test_ze_copy_md, md_attr_advertises_alloc_caps) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + uct_md_attr_v2_t attr; + attr.field_mask = UINT64_MAX; + EXPECT_UCS_OK(uct_md_query_v2(md, &attr)); + + /* ze_copy MD must be able to allocate ZE memory. */ + EXPECT_NE(0u, attr.alloc_mem_types); + EXPECT_GT(attr.max_alloc, 0u); + + uct_md_close(md); +} + + +/* + * Allocate ZE_DEVICE memory through the MD, then free it. + */ +UCS_TEST_F(test_ze_copy_md, mem_alloc_free_device) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + size_t length = 4096; + void *addr = NULL; + uct_mem_h memh = NULL; + + ucs_status_t status = md->ops->mem_alloc(md, &length, &addr, + UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); + if (status == UCS_ERR_UNSUPPORTED) { + UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); + } + ASSERT_UCS_OK(status); + ASSERT_TRUE(addr != NULL); + ASSERT_GE(length, 4096u); + + EXPECT_UCS_OK(md->ops->mem_free(md, memh)); + uct_md_close(md); +} + + +/* + * detect_memory_type must classify ZE device pointers as ZE_DEVICE. + */ +UCS_TEST_F(test_ze_copy_md, detect_memory_type_device) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + size_t length = 4096; + void *addr = NULL; + uct_mem_h memh = NULL; + ucs_status_t status = md->ops->mem_alloc(md, &length, &addr, + UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); + if (status == UCS_ERR_UNSUPPORTED) { + UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); + } + ASSERT_UCS_OK(status); + + ucs_memory_type_t mem_type = UCS_MEMORY_TYPE_HOST; + EXPECT_UCS_OK(md->ops->detect_memory_type(md, addr, length, &mem_type)); + EXPECT_EQ(UCS_MEMORY_TYPE_ZE_DEVICE, mem_type); + + EXPECT_UCS_OK(md->ops->mem_free(md, memh)); + uct_md_close(md); +} + + +/* + * mem_query reports base address and allocation length back. + */ +UCS_TEST_F(test_ze_copy_md, mem_query_base_and_length) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ze_cpy MD on this system"); + } + + size_t length = 8192; + void *addr = NULL; + uct_mem_h memh = NULL; + ucs_status_t status = md->ops->mem_alloc(md, &length, &addr, + UCS_MEMORY_TYPE_ZE_DEVICE, + UCS_SYS_DEVICE_ID_UNKNOWN, 0, + "test_ze_copy", &memh); + if (status == UCS_ERR_UNSUPPORTED) { + UCS_TEST_SKIP_R("ZE_DEVICE alloc unsupported on this system"); + } + ASSERT_UCS_OK(status); + + uct_md_mem_attr_v2_t mem_attr = {}; + mem_attr.field_mask = UCT_MD_MEM_ATTR_V2_FIELD_MEM_TYPE | + UCT_MD_MEM_ATTR_V2_FIELD_BASE_ADDRESS | + UCT_MD_MEM_ATTR_V2_FIELD_ALLOC_LENGTH; + EXPECT_UCS_OK(md->ops->mem_query(md, addr, length, &mem_attr)); + EXPECT_EQ(UCS_MEMORY_TYPE_ZE_DEVICE, mem_attr.mem_type); + EXPECT_TRUE(mem_attr.base_address != NULL); + EXPECT_GE(mem_attr.alloc_length, length); + + EXPECT_UCS_OK(md->ops->mem_free(md, memh)); + uct_md_close(md); +} diff --git a/test/gtest/uct/ze/test_ze_ipc_cache.cc b/test/gtest/uct/ze/test_ze_ipc_cache.cc new file mode 100644 index 00000000000..bb07430580e --- /dev/null +++ b/test/gtest/uct/ze/test_ze_ipc_cache.cc @@ -0,0 +1,193 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +} + +#include + + +/* + * The cache tests use synthetic mapping operations to cover cache identity and + * lifecycle behavior without requiring a Level Zero device. + */ +class test_ze_ipc_cache : public ucs::test { +protected: + struct cache_ops_state { + unsigned open_count; + unsigned close_count; + uintptr_t next_mapped_addr; + }; + + void SetUp() override + { + ucs::test::SetUp(); + m_state.open_count = 0; + m_state.close_count = 0; + m_state.next_mapped_addr = 0x10000000ul; + } + + void TearDown() override + { + uct_ze_ipc_purge_cache_by_context(context()); + ucs::test::TearDown(); + } + + static ucs_status_t open_memhandle(uct_ze_ipc_key_t *, + ze_context_handle_t ze_context, + ze_device_handle_t, + void **mapped_addr, int *dup_fd) + { + cache_ops_state *state = reinterpret_cast(ze_context); + + *mapped_addr = reinterpret_cast(state->next_mapped_addr); + state->next_mapped_addr += 0x10000; + state->open_count++; + *dup_fd = -1; + return UCS_OK; + } + + static ucs_status_t close_memhandle(ze_context_handle_t ze_context, + void *, int) + { + cache_ops_state *state = reinterpret_cast(ze_context); + + state->close_count++; + return UCS_OK; + } + + ze_context_handle_t context() + { + return reinterpret_cast(&m_state); + } + + uct_ze_ipc_key_t make_key(uint64_t alloc_id = 1, + uint64_t proc_create_time = 1) + { + uct_ze_ipc_key_t key = {}; + + key.pid = getpid(); + key.address = 0x20000000ul; + key.length = 4096; + key.alloc_id = alloc_id; + key.proc_create_time = proc_create_time; + return key; + } + + ucs_status_t map(uct_ze_ipc_key_t *key, void **mapped_addr, int *dup_fd) + { + static const uct_ze_ipc_cache_ops_t ops = { + open_memhandle, + close_memhandle + }; + + return uct_ze_ipc_map_memhandle(key, context(), NULL, &ops, mapped_addr, + dup_fd); + } + + cache_ops_state m_state; +}; + + +UCS_TEST_F(test_ze_ipc_cache, create_destroy) { + uct_ze_ipc_cache_t *cache = NULL; + ASSERT_UCS_OK(uct_ze_ipc_create_cache(&cache, "test")); + ASSERT_TRUE(cache != NULL); + uct_ze_ipc_destroy_cache(cache); +} + + +UCS_TEST_F(test_ze_ipc_cache, create_destroy_repeated) { + for (int i = 0; i < 16; ++i) { + uct_ze_ipc_cache_t *cache = NULL; + ASSERT_UCS_OK(uct_ze_ipc_create_cache(&cache, "test_repeat")); + ASSERT_TRUE(cache != NULL); + uct_ze_ipc_destroy_cache(cache); + } +} + + +UCS_TEST_F(test_ze_ipc_cache, multiple_instances) { + const int N = 8; + uct_ze_ipc_cache_t *caches[N] = {}; + + for (int i = 0; i < N; ++i) { + ASSERT_UCS_OK(uct_ze_ipc_create_cache(&caches[i], "test_multi")); + ASSERT_TRUE(caches[i] != NULL); + } + for (int i = 0; i < N; ++i) { + uct_ze_ipc_destroy_cache(caches[i]); + } +} + + +UCS_TEST_F(test_ze_ipc_cache, stale_allocation_is_remapped) { + uct_ze_ipc_key_t key_a = make_key(1); + uct_ze_ipc_key_t key_b = make_key(2); + void *mapped_a = NULL; + void *mapped_b = NULL; + int dup_fd_a = -1; + int dup_fd_b = -1; + + ASSERT_UCS_OK(map(&key_a, &mapped_a, &dup_fd_a)); + ASSERT_UCS_OK(map(&key_b, &mapped_b, &dup_fd_b)); + + EXPECT_EQ(2u, m_state.open_count); + + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_a.pid, key_a.address, mapped_a, + context(), dup_fd_a, 0)); + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, + context(), dup_fd_b, 0)); + EXPECT_EQ(2u, m_state.close_count); +} + + +UCS_TEST_F(test_ze_ipc_cache, pid_reuse_is_remapped) { + uct_ze_ipc_key_t key_a = make_key(1, 1); + uct_ze_ipc_key_t key_b = make_key(1, 2); + void *mapped_a = NULL; + void *mapped_b = NULL; + int dup_fd_a = -1; + int dup_fd_b = -1; + + ASSERT_UCS_OK(map(&key_a, &mapped_a, &dup_fd_a)); + ASSERT_UCS_OK(map(&key_b, &mapped_b, &dup_fd_b)); + + EXPECT_EQ(2u, m_state.open_count); + + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_a.pid, key_a.address, mapped_a, + context(), dup_fd_a, 0)); + EXPECT_UCS_OK(uct_ze_ipc_unmap_memhandle(key_b.pid, key_b.address, mapped_b, + context(), dup_fd_b, 0)); + EXPECT_EQ(2u, m_state.close_count); +} + + +UCS_TEST_F(test_ze_ipc_cache, purge_by_context_then_remap) { + uct_ze_ipc_key_t key = make_key(); + void *mapped_a = NULL; + void *mapped_b = NULL; + int dup_fd_a = -1; + int dup_fd_b = -1; + + ASSERT_UCS_OK(map(&key, &mapped_a, &dup_fd_a)); + ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped_a, + context(), dup_fd_a, 1)); + EXPECT_EQ(1u, m_state.open_count); + EXPECT_EQ(0u, m_state.close_count); + + uct_ze_ipc_purge_cache_by_context(context()); + EXPECT_EQ(1u, m_state.close_count); + + ASSERT_UCS_OK(map(&key, &mapped_b, &dup_fd_b)); + EXPECT_EQ(2u, m_state.open_count); + + ASSERT_UCS_OK(uct_ze_ipc_unmap_memhandle(key.pid, key.address, mapped_b, + context(), dup_fd_b, 0)); + EXPECT_EQ(2u, m_state.close_count); +} diff --git a/test/gtest/uct/ze/test_ze_ipc_md.cc b/test/gtest/uct/ze/test_ze_ipc_md.cc new file mode 100644 index 00000000000..f562da81844 --- /dev/null +++ b/test/gtest/uct/ze/test_ze_ipc_md.cc @@ -0,0 +1,210 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +#include +#include +} + + +/* + * Lightweight smoke tests for the ZE_IPC component. They do not require a + * peer process; they validate that the component is registered, that it + * advertises at least one MD resource, and that the MD opens/closes cleanly + * with the post-merge per-sub-device layout. + */ +class test_ze_ipc_md : public ucs::test { +protected: + void SetUp() override + { + ucs::test::SetUp(); + if (uct_ze_base_init() != ZE_RESULT_SUCCESS) { + UCS_TEST_SKIP_R("Level Zero runtime not available"); + } + if (uct_ze_base_get_num_devices() == 0) { + UCS_TEST_SKIP_R("No Level Zero devices available"); + } + } + + /* Query MD resources via the component API used by the rest of UCT. */ + static ucs_status_t + query_md_resources(uct_md_resource_desc_t **resources_p, unsigned *num_p) + { + uct_component_attr_t attr; + ucs_status_t status; + + attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT; + status = uct_component_query(&uct_ze_ipc_component, &attr); + if (status != UCS_OK) { + return status; + } + + unsigned count = attr.md_resource_count; + uct_md_resource_desc_t *res = (uct_md_resource_desc_t*)ucs_calloc( + count, sizeof(uct_md_resource_desc_t), "ze_ipc_md_resources"); + if ((res == NULL) && (count > 0)) { + return UCS_ERR_NO_MEMORY; + } + + attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES; + attr.md_resources = res; + status = uct_component_query(&uct_ze_ipc_component, &attr); + if (status != UCS_OK) { + ucs_free(res); + return status; + } + + *resources_p = res; + *num_p = count; + return UCS_OK; + } + + static uct_md_h open_first_md() + { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + + if (query_md_resources(&resources, &num) != UCS_OK) { + return NULL; + } + if (num == 0) { + ucs_free(resources); + return NULL; + } + + uct_md_config_t *md_config = NULL; + ucs_status_t status = uct_md_config_read(&uct_ze_ipc_component, NULL, + NULL, &md_config); + if (status != UCS_OK) { + ucs_free(resources); + return NULL; + } + + uct_md_h md = NULL; + status = uct_md_open(&uct_ze_ipc_component, resources[0].md_name, + md_config, &md); + uct_config_release(md_config); + ucs_free(resources); + + return (status == UCS_OK) ? md : NULL; + } +}; + + +UCS_TEST_F(test_ze_ipc_md, component_is_registered) { + EXPECT_STREQ("ze_ipc", uct_ze_ipc_component.name); +} + + +UCS_TEST_F(test_ze_ipc_md, query_md_resources_returns_at_least_one) { + uct_md_resource_desc_t *resources = NULL; + unsigned num = 0; + + EXPECT_UCS_OK(query_md_resources(&resources, &num)); + EXPECT_GE(num, 1u); + ucs_free(resources); +} + + +UCS_TEST_F(test_ze_ipc_md, open_close_md) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + + /* The post-merge ze_ipc_md must populate device + context. */ + auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + EXPECT_TRUE(ze_md->ze_device != NULL); + EXPECT_TRUE(ze_md->ze_context != NULL); + + uct_md_close(md); +} + + +UCS_TEST_F(test_ze_ipc_md, md_attr_advertises_registerable_mem) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + + uct_md_attr_v2_t attr; + attr.field_mask = UINT64_MAX; + EXPECT_UCS_OK(uct_md_query_v2(md, &attr)); + EXPECT_NE(0u, attr.reg_mem_types) << "no registerable mem types"; + /* IPC MD must advertise the rkey size and require an rkey. */ + EXPECT_GT(attr.rkey_packed_size, 0u); + EXPECT_TRUE(attr.flags & UCT_MD_FLAG_NEED_RKEY); + EXPECT_TRUE(attr.flags & UCT_MD_FLAG_REG); + + uct_md_close(md); +} + + +/* + * End-to-end: allocate ZE device memory, register with the IPC MD, pack + * the IPC key, and deregister. This is the path NIXL relies on for KV + * cache exchange (mem_reg -> mkey_pack -> mem_dereg). + */ +UCS_TEST_F(test_ze_ipc_md, mem_reg_pack_dereg_device) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + auto *ze_md = ucs_derived_of(md, uct_ze_ipc_md_t); + + const size_t size = 4096; + void *dev_ptr = NULL; + ze_device_mem_alloc_desc_t dev_desc = {}; + dev_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeMemAllocDevice(ze_md->ze_context, &dev_desc, size, 64, + ze_md->ze_device, &dev_ptr)); + ASSERT_TRUE(dev_ptr != NULL); + + uct_mem_h memh = NULL; + EXPECT_UCS_OK(md->ops->mem_reg(md, dev_ptr, size, NULL, &memh)); + ASSERT_TRUE(memh != NULL); + + uct_ze_ipc_key_t packed = {}; + EXPECT_UCS_OK(md->ops->mkey_pack(md, memh, dev_ptr, size, NULL, &packed)); + EXPECT_EQ((pid_t)getpid(), packed.pid); + EXPECT_GE(packed.length, size); + EXPECT_NE(0u, packed.address); + + uct_md_mem_dereg_params_t dereg_params = {}; + dereg_params.field_mask = UCT_MD_MEM_DEREG_FIELD_MEMH; + dereg_params.memh = memh; + EXPECT_UCS_OK(md->ops->mem_dereg(md, &dereg_params)); + + EXPECT_EQ(ZE_RESULT_SUCCESS, zeMemFree(ze_md->ze_context, dev_ptr)); + uct_md_close(md); +} + + +/* + * mem_reg must reject non-ZE pointers (the IPC MD only supports VRAM). + * The MD logs an error in this path, so we suppress error log capture. + */ +UCS_TEST_F(test_ze_ipc_md, mem_reg_rejects_host_pointer) { + uct_md_h md = open_first_md(); + if (md == NULL) { + UCS_TEST_SKIP_R("Could not open ZE_IPC MD on this system"); + } + + char buf[64]; + uct_mem_h memh = NULL; + ucs_status_t status; + { + scoped_log_handler slh(hide_errors_logger); + status = md->ops->mem_reg(md, buf, sizeof(buf), NULL, &memh); + } + EXPECT_EQ(UCS_ERR_INVALID_ADDR, status); + + uct_md_close(md); +} diff --git a/test/gtest/uct/ze/test_ze_ipc_rma.cc b/test/gtest/uct/ze/test_ze_ipc_rma.cc new file mode 100644 index 00000000000..1ba6f23e142 --- /dev/null +++ b/test/gtest/uct/ze/test_ze_ipc_rma.cc @@ -0,0 +1,661 @@ +/** + * Copyright (C) Intel Corporation, 2026. ALL RIGHTS RESERVED. + * See file LICENSE for terms. + */ + +#include + +extern "C" { +#include +#include +} + +#include +#include +#include +#include +#include +#include + + +/* + * Endpoint-level RMA tests for ze_ipc with both entities in this process. The + * transport takes its same-pid path, so these tests skip the pidfd_getfd fd + * duplication and the reachability check; test_ze_ipc_rma_xproc covers them. + */ +class test_ze_ipc_rma : public uct_test { +protected: + void init() + { + uct_test::init(); + + m_receiver = uct_test::create_entity(0); + m_entities.push_back(m_receiver); + + m_sender = uct_test::create_entity(0); + m_entities.push_back(m_sender); + + m_sender->connect(0, *m_receiver, 0); + } + + struct flush_comp { + uct_completion_t super; + uct_completion_t *copy_comp; + int copies_left; + }; + + static void copy_done(uct_completion_t *self) + { + } + + static void flush_done(uct_completion_t *self) + { + flush_comp *comp = ucs_container_of(self, flush_comp, super); + + comp->copies_left = comp->copy_comp->count; + } + + entity *m_sender; + entity *m_receiver; + + static const uint64_t SEED1 = 0xABClu; + static const uint64_t SEED2 = 0xDEFlu; +}; + + +UCS_TEST_P(test_ze_ipc_rma, put_zcopy) +{ + size_t length = 4096; + + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_put_zcopy(m_sender->ep(0), sendbuf.iov(), + 1, recvbuf.addr(), + recvbuf.rkey(), NULL)); + m_sender->flush(); + recvbuf.pattern_check(SEED1); +} + +UCS_TEST_P(test_ze_ipc_rma, get_zcopy) +{ + size_t length = 4096; + + mapped_buffer sendbuf(length, SEED1, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_get_zcopy(m_sender->ep(0), recvbuf.iov(), + 1, sendbuf.addr(), + sendbuf.rkey(), NULL)); + m_sender->flush(); + recvbuf.pattern_check(SEED1); +} + +UCS_TEST_P(test_ze_ipc_rma, ep_flush_waits_for_copies) +{ + static const unsigned num_copies = 8; + static const size_t length = UCS_MBYTE; + + mapped_buffer sendbuf(length * num_copies, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length * num_copies, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + uct_completion_t copy_comp = {copy_done, (int)num_copies, UCS_OK}; + flush_comp flush = {{flush_done, 1, UCS_OK}, ©_comp, -1}; + uct_iov_t iov = *sendbuf.iov(); + + iov.length = length; + for (unsigned i = 0; i < num_copies; ++i) { + iov.buffer = UCS_PTR_BYTE_OFFSET(sendbuf.ptr(), i * length); + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_put_zcopy(m_sender->ep(0), &iov, 1, + recvbuf.addr() + (i * length), + recvbuf.rkey(), ©_comp)); + } + + /* Nothing has progressed the copies yet */ + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_flush(m_sender->ep(0), 0, &flush.super)); + wait_for_value(&flush.super.count, 0, true); + ASSERT_EQ(0, flush.super.count); + + EXPECT_EQ(UCS_OK, flush.super.status); + EXPECT_EQ(0, flush.copies_left); + EXPECT_EQ(UCS_OK, copy_comp.status); + recvbuf.pattern_check(SEED1); +} + +UCS_TEST_P(test_ze_ipc_rma, ep_flush_pending_at_close, "ZE_IPC_MAX_POLL=1") +{ + static const size_t length = 4096; + uct_completion_t copy_comp = {copy_done, 1, UCS_OK}; + flush_comp flush = {{flush_done, 1, UCS_OK}, ©_comp, -1}; + + { + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_put_zcopy(m_sender->ep(0), sendbuf.iov(), 1, + recvbuf.addr(), recvbuf.rkey(), + ©_comp)); + ASSERT_EQ(UCS_INPROGRESS, + uct_ep_flush(m_sender->ep(0), 0, &flush.super)); + + /* With MAX_POLL=1 a progress call completes at most one event, so the + * flush marker is still queued when the copy completes */ + test_base::wait_for_value(©_comp.count, 0, + [this]() { progress(); }); + ASSERT_EQ(0, copy_comp.count); + } + + EXPECT_EQ(1, flush.super.count); + m_sender->destroy_eps(); + m_entities.remove(m_sender); + EXPECT_EQ(1, flush.super.count); +} + +UCS_TEST_P(test_ze_ipc_rma, put_zcopy_rejects_multi_iov) +{ + size_t length = 64; + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + uct_iov_t iov[2]; + + iov[0] = *sendbuf.iov(); + iov[1] = *sendbuf.iov(); + + ucs_status_t status; + { + scoped_log_handler slh(hide_errors_logger); + status = uct_ep_put_zcopy(m_sender->ep(0), iov, 2, recvbuf.addr(), + recvbuf.rkey(), NULL); + } + EXPECT_EQ(UCS_ERR_INVALID_PARAM, status); +} + +UCS_TEST_P(test_ze_ipc_rma, put_zcopy_rejects_out_of_range_rkey) +{ + size_t length = 64; + mapped_buffer sendbuf(length, SEED1, *m_sender, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(length, SEED2, *m_receiver, 0, + UCS_MEMORY_TYPE_ZE_DEVICE); + + ucs_status_t status; + { + /* the rkey covers the whole driver allocation, which is page-rounded + * and therefore larger than 'length' - go far enough past the base + * address to land outside it */ + scoped_log_handler slh(hide_errors_logger); + status = uct_ep_put_zcopy(m_sender->ep(0), sendbuf.iov(), 1, + recvbuf.addr() + UCS_MBYTE, recvbuf.rkey(), + NULL); + } + EXPECT_EQ(UCS_ERR_INVALID_PARAM, status); +} + +_UCT_INSTANTIATE_TEST_CASE(test_ze_ipc_rma, ze_ipc) + + +/* + * Cross-process RMA. The test process exports a device buffer, and an exec'd + * copy of this gtest binary running the same test imports it and issues PUT + * and GET. Only the control socket survives the exec, so the importer can + * reach the exporter's dmabuf fd only through pidfd_getfd. + */ +class test_ze_ipc_rma_xproc : + private ucs::clear_dontcopy_regions, + public uct_test { +protected: + enum { + MSG_PUT_DONE, + MSG_GET_READY, + MSG_DONE + }; + + struct op_msg { + uint32_t op; + uint32_t round; + }; + + struct exporter_info { + pid_t pid; + uint64_t address; + uint32_t iface_addr_len; + uint32_t dev_addr_len; + uint32_t rkey_len; + }; + + struct peer_process { + pid_t pid = -1; + int fd = -1; + + ~peer_process() + { + if (fd >= 0) { + close(fd); + } + if (pid > 0) { + kill(pid, SIGKILL); + waitpid(pid, NULL, 0); + } + } + }; + + /* Holds every free fd below a limit, so the next fd opened lands above */ + struct fd_fillers { + std::vector fds; + + ~fd_fillers() + { + release(); + } + + void occupy_below(int limit) + { + int fd; + + while ((fd = open("/dev/null", O_RDONLY | O_CLOEXEC)) < limit) { + ASSERT_GE(fd, 0) << "open(/dev/null) failed: " + << strerror(errno); + fds.push_back(fd); + } + + close(fd); + } + + void release() + { + for (int fd : fds) { + close(fd); + } + fds.clear(); + } + }; + + static const char *PEER_FD_ENV; + static const int PEER_FD = 3; + static const int TIMEOUT_MS = 60000; + static const unsigned NUM_ROUNDS = 3; + static const size_t LENGTH = UCS_MBYTE; + /* Above any fd the importer opens on its own */ + static const int MIN_EXPORT_FD = 256; + + static uint64_t put_seed(unsigned round) + { + return 0x1000 + round; + } + + static uint64_t get_seed(unsigned round) + { + return 0x2000 + round; + } + + static void send_msg(int fd, const void *buffer, size_t length) + { + ssize_t ret = send(fd, buffer, length, MSG_NOSIGNAL); + + if (ret != (ssize_t)length) { + UCS_TEST_ABORT("send() returned " << ret << ": " + << strerror(errno)); + } + } + + static size_t recv_msg(int fd, void *buffer, size_t max_length) + { + struct pollfd pfd = {fd, POLLIN, 0}; + ssize_t ret; + + do { + ret = poll(&pfd, 1, TIMEOUT_MS * ucs::test_time_multiplier()); + } while ((ret < 0) && (errno == EINTR)); + + if (ret != 1) { + UCS_TEST_ABORT("no message from peer process: poll() returned " + << ret); + } + + ret = recv(fd, buffer, max_length, 0); + if (ret <= 0) { + UCS_TEST_ABORT("peer process closed the socket: recv() returned " + << ret); + } + + return ret; + } + + static void send_op(int fd, uint32_t op, uint32_t round) + { + op_msg msg = {op, round}; + + send_msg(fd, &msg, sizeof(msg)); + } + + static void recv_op(int fd, uint32_t op, uint32_t round) + { + op_msg msg = {}; + size_t length; + + length = recv_msg(fd, &msg, sizeof(msg)); + if ((length != sizeof(msg)) || (msg.op != op) || (msg.round != round)) { + UCS_TEST_ABORT("got message op " << msg.op << " round " << msg.round + << ", expected op " << op + << " round " << round); + } + } + + void pack_exporter_info(const entity &e, const mapped_buffer &buffer, + std::vector &info) + { + exporter_info hdr; + uct_md_mkey_pack_params_t pack_params = {}; + + hdr.pid = getpid(); + hdr.address = buffer.addr(); + hdr.iface_addr_len = e.iface_attr().iface_addr_len; + hdr.dev_addr_len = e.iface_attr().device_addr_len; + hdr.rkey_len = e.md_attr().rkey_packed_size; + + info.resize(sizeof(hdr) + hdr.iface_addr_len + hdr.dev_addr_len + + hdr.rkey_len); + memcpy(&info[0], &hdr, sizeof(hdr)); + + char *iface_addr = &info[sizeof(hdr)]; + char *dev_addr = iface_addr + hdr.iface_addr_len; + char *rkey_buf = dev_addr + hdr.dev_addr_len; + + ASSERT_UCS_OK(uct_iface_get_address(e.iface(), + (uct_iface_addr_t*)iface_addr)); + ASSERT_UCS_OK( + uct_iface_get_device_address(e.iface(), + (uct_device_addr_t*)dev_addr)); + ASSERT_UCS_OK(uct_md_mkey_pack_v2(e.md(), buffer.memh(), buffer.ptr(), + buffer.length(), &pack_params, + rkey_buf)); + } + + void start_importer(peer_process &peer) + { + const ::testing::TestInfo *test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + std::string filter = std::string("--gtest_filter=") + + test_info->test_suite_name() + "." + + test_info->name(); + std::string repeat = "--gtest_repeat=1"; + std::string exe = "/proc/self/exe"; + std::string fd_env = std::string(PEER_FD_ENV) + "=" + + std::to_string(PEER_FD); + std::vector argv = {&exe[0], &filter[0], &repeat[0], NULL}; + std::vector envp; + std::vector close_fds; + int fds[2]; + + /* Sharding would make the child skip its only test and exit 0 */ + for (char **var = environ; *var != NULL; ++var) { + if (strncmp(*var, "GTEST_TOTAL_SHARDS=", 19) && + strncmp(*var, "GTEST_SHARD_INDEX=", 18)) { + envp.push_back(*var); + } + } + envp.push_back(&fd_env[0]); + envp.push_back(NULL); + + ASSERT_EQ(0, socketpair(AF_UNIX, SOCK_SEQPACKET, 0, fds)); + for (int fd : ucs::get_open_fds()) { + if (fd > PEER_FD) { + close_fds.push_back(fd); + } + } + + peer.pid = fork(); + if (peer.pid == 0) { + /* Async-signal-safe calls only until execve() */ + if (dup2(fds[1], PEER_FD) == PEER_FD) { + for (int fd : close_fds) { + close(fd); + } + execve(exe.c_str(), argv.data(), envp.data()); + } + _exit(127); + } + + close(fds[1]); + peer.fd = fds[0]; + ASSERT_GT(peer.pid, 0) << "fork() failed: " << strerror(errno); + } + + /* The driver closes the fd of an outstanding IPC handle when a device + * other than the allocation's accesses the buffer */ + static void fill_from_other_device(const mapped_buffer &buffer) + { + ze_context_desc_t context_desc = {}; + ze_command_queue_desc_t queue_desc = {}; + ze_command_list_handle_t cmdlist; + ze_context_handle_t context; + uint8_t value = 0; + + context_desc.stype = ZE_STRUCTURE_TYPE_CONTEXT_DESC; + queue_desc.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC; + queue_desc.mode = ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS; + + ASSERT_EQ(ZE_RESULT_SUCCESS, zeContextCreate(uct_ze_base_get_driver(), + &context_desc, &context)); + ASSERT_EQ(ZE_RESULT_SUCCESS, + zeCommandListCreateImmediate(context, + uct_ze_base_get_device(1), + &queue_desc, &cmdlist)); + EXPECT_EQ(ZE_RESULT_SUCCESS, + zeCommandListAppendMemoryFill(cmdlist, buffer.ptr(), &value, + sizeof(value), buffer.length(), + NULL, 0, NULL)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeCommandListDestroy(cmdlist)); + EXPECT_EQ(ZE_RESULT_SUCCESS, zeContextDestroy(context)); + } + + /* The transport opens a pidfd and then takes the exporter's fd, each into + * the lowest free fd. Leaving only export_fd - 1 and export_fd free below + * export_fd makes it receive the exporter's fd under the number that + * zeMemGetIpcHandle returned to the exporter */ + static void force_fd_collision(const uct_ze_ipc_key_t *key, + fd_fillers &fillers) + { + ASSERT_GE(key->export_fd, MIN_EXPORT_FD); + ASSERT_NO_FATAL_FAILURE(fillers.occupy_below(key->export_fd - 1)); + ASSERT_EQ(-1, fcntl(key->export_fd - 1, F_GETFD)); + ASSERT_EQ(-1, fcntl(key->export_fd, F_GETFD)); + } + + /* With the collision forced, the fix leaves the importer's copy of the + * exporter's fd at export_fd - 1, the lowest fd it can move it to */ + static void check_fd_moved(const uct_ze_ipc_key_t *key) + { + std::string path = "/proc/self/fd/" + + ucs::to_string(key->export_fd - 1); + char target[256]; + ssize_t len; + + len = readlink(path.c_str(), target, sizeof(target) - 1); + ASSERT_GE(len, 0) << path << ": " << strerror(errno); + target[len] = '\0'; + EXPECT_NE(nullptr, strstr(target, "dmabuf")) + << path << " -> " << target; + } + + void run_exporter(bool other_device, bool collide_fd) + { + entity *e = uct_test::create_entity(0); + m_entities.push_back(e); + + fd_fillers fillers; + if (collide_fd) { + ASSERT_NO_FATAL_FAILURE(fillers.occupy_below(MIN_EXPORT_FD)); + } + + mapped_buffer buffer(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); + std::vector info; + peer_process peer; + int status; + + fillers.release(); + ASSERT_NO_FATAL_FAILURE(pack_exporter_info(*e, buffer, info)); + ASSERT_NO_FATAL_FAILURE(start_importer(peer)); + if (other_device) { + ASSERT_NO_FATAL_FAILURE(fill_from_other_device(buffer)); + } + send_msg(peer.fd, info.data(), info.size()); + + for (unsigned round = 0; round < NUM_ROUNDS; ++round) { + recv_op(peer.fd, MSG_PUT_DONE, round); + buffer.pattern_check(put_seed(round)); + buffer.pattern_fill(get_seed(round)); + send_op(peer.fd, MSG_GET_READY, round); + } + recv_op(peer.fd, MSG_DONE, NUM_ROUNDS); + + ASSERT_EQ(peer.pid, waitpid(peer.pid, &status, 0)); + peer.pid = -1; + EXPECT_TRUE(WIFEXITED(status) && (WEXITSTATUS(status) == 0)) + << "importer exit status " << status; + } + + void run_importer(int fd, bool collide_fd) + { + std::vector info(UCS_KBYTE * 4); + uct_iface_is_reachable_params_t params = {}; + uct_ep_params_t ep_params = {}; + ucs::handle ep; + uct_rkey_bundle_t rkey; + exporter_info hdr; + char info_str[256]; + size_t length; + + length = recv_msg(fd, info.data(), info.size()); + ASSERT_GE(length, sizeof(hdr)); + memcpy(&hdr, info.data(), sizeof(hdr)); + ASSERT_EQ(sizeof(hdr) + hdr.iface_addr_len + hdr.dev_addr_len + + hdr.rkey_len, + length); + + const void *iface_addr = &info[sizeof(hdr)]; + const void *dev_addr = &info[sizeof(hdr) + hdr.iface_addr_len]; + const void *rkey_buf = + &info[sizeof(hdr) + hdr.iface_addr_len + hdr.dev_addr_len]; + + entity *e = uct_test::create_entity(0); + m_entities.push_back(e); + + ASSERT_NE(getpid(), hdr.pid); + ASSERT_EQ(e->iface_attr().iface_addr_len, hdr.iface_addr_len); + ASSERT_EQ(e->iface_attr().device_addr_len, hdr.dev_addr_len); + ASSERT_EQ(hdr.pid, *(const pid_t*)iface_addr); + + params.field_mask = UCT_IFACE_IS_REACHABLE_FIELD_DEVICE_ADDR | + UCT_IFACE_IS_REACHABLE_FIELD_IFACE_ADDR | + UCT_IFACE_IS_REACHABLE_FIELD_INFO_STRING | + UCT_IFACE_IS_REACHABLE_FIELD_INFO_STRING_LENGTH; + params.device_addr = (const uct_device_addr_t*)dev_addr; + params.iface_addr = (const uct_iface_addr_t*)iface_addr; + params.info_string = info_str; + params.info_string_length = sizeof(info_str); + info_str[0] = '\0'; + ASSERT_TRUE(uct_iface_is_reachable_v2(e->iface(), ¶ms)) << info_str; + + ep_params.field_mask = UCT_EP_PARAM_FIELD_IFACE | + UCT_EP_PARAM_FIELD_DEV_ADDR | + UCT_EP_PARAM_FIELD_IFACE_ADDR; + ep_params.iface = e->iface(); + ep_params.dev_addr = (const uct_device_addr_t*)dev_addr; + ep_params.iface_addr = (const uct_iface_addr_t*)iface_addr; + UCS_TEST_CREATE_HANDLE(uct_ep_h, ep, uct_ep_destroy, uct_ep_create, + &ep_params); + + ASSERT_UCS_OK(uct_rkey_unpack(GetParam()->component, rkey_buf, &rkey)); + ASSERT_EQ(hdr.pid, ((const uct_ze_ipc_key_t*)rkey.rkey)->pid); + + mapped_buffer sendbuf(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); + mapped_buffer recvbuf(LENGTH, 0, *e, 0, UCS_MEMORY_TYPE_ZE_DEVICE); + fd_fillers fillers; + + for (unsigned round = 0; round < NUM_ROUNDS; ++round) { + bool collide = collide_fd && (round == 0); + + sendbuf.pattern_fill(put_seed(round)); + if (collide) { + ASSERT_NO_FATAL_FAILURE(force_fd_collision( + (const uct_ze_ipc_key_t*)rkey.rkey, fillers)); + } + + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_put_zcopy(ep, sendbuf.iov(), 1, + hdr.address, rkey.rkey, + NULL)); + if (collide) { + ASSERT_NO_FATAL_FAILURE(check_fd_moved( + (const uct_ze_ipc_key_t*)rkey.rkey)); + } + + fillers.release(); + e->flush(); + send_op(fd, MSG_PUT_DONE, round); + + recv_op(fd, MSG_GET_READY, round); + ASSERT_UCS_OK_OR_INPROGRESS(uct_ep_get_zcopy(ep, recvbuf.iov(), 1, + hdr.address, rkey.rkey, + NULL)); + e->flush(); + recvbuf.pattern_check(get_seed(round)); + } + + ASSERT_UCS_OK(uct_rkey_release(GetParam()->component, &rkey)); + send_op(fd, MSG_DONE, NUM_ROUNDS); + } + + void test_put_get(bool other_device = false, bool collide_fd = false) + { + const char *peer_fd = getenv(PEER_FD_ENV); + + if (peer_fd == NULL) { + run_exporter(other_device, collide_fd); + } else { + run_importer(atoi(peer_fd), collide_fd); + } + } +}; + +const char *test_ze_ipc_rma_xproc::PEER_FD_ENV = "UCT_ZE_IPC_TEST_PEER_FD"; +const int test_ze_ipc_rma_xproc::MIN_EXPORT_FD; + + +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get) +{ + test_put_get(); +} + +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_no_cache, "ZE_IPC_ENABLE_CACHE=n") +{ + test_put_get(); +} + +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_after_other_device_access) +{ + if (uct_ze_base_get_num_devices() < 2) { + UCS_TEST_SKIP_R("needs two Level Zero devices"); + } + + test_put_get(true); +} + +UCS_TEST_P(test_ze_ipc_rma_xproc, put_get_importer_fd_equals_export_fd) +{ + test_put_get(false, true); +} + +_UCT_INSTANTIATE_TEST_CASE(test_ze_ipc_rma_xproc, ze_ipc)