From e5d8739e5328294d26d63cf89f3ee6b6e1cc3c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 25 Aug 2025 13:35:07 +0200 Subject: [PATCH 01/49] Add resident device change call --- include/umf/memory_pool_ops.h | 12 + include/umf/memory_provider.h | 11 + include/umf/memory_provider_ops.h | 12 + include/umf/providers/provider_level_zero.h | 3 +- src/critnib/critnib.c | 15 +- src/critnib/critnib.h | 4 + src/libumf.def | 1 + src/libumf.map | 1 + src/memory_provider.c | 10 + src/provider/provider_cuda.c | 10 + src/provider/provider_level_zero.c | 265 +++++++++++++++++--- src/provider/provider_tracking.c | 33 +++ src/provider/provider_tracking.h | 4 + test/common/provider_null.c | 10 + 14 files changed, 360 insertions(+), 31 deletions(-) diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index c9628c77e0..dc75b8a7d9 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -191,6 +191,18 @@ typedef struct umf_memory_pool_ops_t { /// failure. /// umf_result_t (*ext_trim_memory)(void *pool, size_t minBytesToKeep); + + /// + /// @brief Adds or removes devices on which allocations should be made + /// resident. + /// @param pool pointer to the memory pool + /// @param peerIdx identifier of device + /// @param isAdding boolean indicating if peer is to be removed or added + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on + /// failure. + /// + umf_result_t (*ext_resident_device_change)(void *pool, uint32_t peerIdx, + _Bool isAdding); } umf_memory_pool_ops_t; #ifdef __cplusplus diff --git a/include/umf/memory_provider.h b/include/umf/memory_provider.h index b98b893468..3d31ce6b73 100644 --- a/include/umf/memory_provider.h +++ b/include/umf/memory_provider.h @@ -265,6 +265,17 @@ umf_result_t umfMemoryProviderAllocationMerge(umf_memory_provider_handle_t hProvider, void *lowPtr, void *highPtr, size_t totalSize); +/// @brief Adds or removes devices on which allocations should be made +/// resident. +/// @param hProvider handle to the memory provider +/// @param deviceIndex identifier of device +/// @param isAdding boolean indicating if peer is to be removed or added +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on +/// failure. +umf_result_t +umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, + uint32_t deviceIndex, bool isAdding); + #ifdef __cplusplus } #endif diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index a520ed8891..9efc2d3101 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -11,6 +11,7 @@ #define UMF_MEMORY_PROVIDER_OPS_H 1 #include +#include #include @@ -322,6 +323,17 @@ typedef struct umf_memory_provider_ops_t { void *provider, umf_memory_property_id_t memory_property_id, size_t *size); + /// @brief Adds or removes devices on which allocations should be made + /// resident. + /// @param provider handle to the memory provider + /// @param device_index identifier of device + /// @param is_adding boolean indicating if peer is to be removed or added + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on + /// failure. + umf_result_t (*ext_resident_device_change)(void *provider, + uint32_t device_index, + bool is_adding); + } umf_memory_provider_ops_t; #ifdef __cplusplus diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index 65d7e3e781..422958fbfa 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -66,7 +66,8 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetMemoryType( /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, - ze_device_handle_t *hDevices, uint32_t deviceCount); + ze_device_handle_t *hDevices, uint32_t deviceCount, + uint32_t *residentDevicesIndices, uint32_t residentDevicesCount); typedef enum umf_level_zero_memory_provider_free_policy_t { UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT = diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index 17a7d80be1..1e2e7c14b6 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -1094,7 +1094,8 @@ int critnib_find(struct critnib *c, uintptr_t key, enum find_dir_t dir, * * If func() returns non-zero, the search is aborted. */ -static int iter(struct critnib_node *__restrict n, word min, word max, +static int iter(struct critnib_node *__restrict n, const word min, + const word max, int (*func)(word key, void *value, void *privdata), void *privdata) { if (is_leaf(n)) { @@ -1129,9 +1130,21 @@ static int iter(struct critnib_node *__restrict n, word min, word max, void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, int (*func)(uintptr_t key, void *value, void *privdata), void *privdata) { + bool wasIterating = false; utils_mutex_lock(&c->mutex); if (c->root) { iter(c->root, min, max, func, privdata); + wasIterating = true; } utils_mutex_unlock(&c->mutex); + if (!wasIterating) { + LOG_DEBUG("there was no root, iterating critnib:%p was skipped", + (void *)c); + } +} + +void critnib_iter_all(critnib *c, + int (*func)(uintptr_t key, void *value, void *privdata), + void *privdata) { + critnib_iter(c, 0, (uintptr_t)-1, func, privdata); } diff --git a/src/critnib/critnib.h b/src/critnib/critnib.h index 690d75faef..c9f215d079 100644 --- a/src/critnib/critnib.h +++ b/src/critnib/critnib.h @@ -35,6 +35,10 @@ int critnib_insert(critnib *c, uintptr_t key, void *value, int update); void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, int (*func)(uintptr_t key, void *value, void *privdata), void *privdata); +void critnib_iter_all(critnib *c, + int (*func)(uintptr_t key, void *value, void *privdata), + void *privdata); + int critnib_remove_release(critnib *c, uintptr_t key); /* diff --git a/src/libumf.def b/src/libumf.def index 68163c6b5f..114682d3db 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -60,6 +60,7 @@ EXPORTS umfMemoryProviderPurgeForce umfMemoryProviderPurgeLazy umfMemoryProviderPutIPCHandle + umfMemoryProviderResidentDeviceChange umfMempolicyCreate umfMempolicyDestroy umfMempolicySetCustomSplitPartitions diff --git a/src/libumf.map b/src/libumf.map index 98b9134995..9dfbc30d90 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -54,6 +54,7 @@ UMF_1.0 { umfMemoryProviderPurgeForce; umfMemoryProviderPurgeLazy; umfMemoryProviderPutIPCHandle; + umfMemoryProviderResidentDeviceChange; umfMempolicyCreate; umfMempolicyDestroy; umfMempolicySetCustomSplitPartitions; diff --git a/src/memory_provider.c b/src/memory_provider.c index 324fa751bd..ad162095dd 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -606,3 +606,13 @@ umf_result_t umfMemoryProviderGetAllocationPropertiesSize( checkErrorAndSetLastProvider(res, hProvider); return res; } + +umf_result_t +umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, + uint32_t deviceIndex, bool isAdding) { + UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + umf_result_t res = hProvider->ops.ext_resident_device_change( + hProvider->provider_priv, deviceIndex, isAdding); + checkErrorAndSetLastProvider(res, hProvider); + return res; +} \ No newline at end of file diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index b6c15d2d1b..c867f76566 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -802,6 +802,15 @@ static umf_result_t cu_memory_provider_get_allocation_properties_size( return UMF_RESULT_ERROR_INVALID_ARGUMENT; } +static umf_result_t +cu_memory_provider_resident_device_change(void *provider, uint32_t device_index, + bool is_adding) { + (void)provider; + (void)device_index; + (void)is_adding; + return UMF_RESULT_SUCCESS; +} + static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = cu_memory_provider_initialize, @@ -829,6 +838,7 @@ static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { cu_memory_provider_get_allocation_properties, .ext_get_allocation_properties_size = cu_memory_provider_get_allocation_properties_size, + .ext_resident_device_change = cu_memory_provider_resident_device_change, }; const umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void) { diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index d5ab3e8e4f..b856cf46ba 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -20,6 +20,8 @@ #include "utils_load_library.h" #include "utils_log.h" +#include + static void *ze_lib_handle = NULL; void fini_ze_global_state(void) { @@ -51,7 +53,11 @@ typedef struct umf_level_zero_memory_provider_params_t { umf_usm_memory_type_t memory_type; ///< Allocation memory type ze_device_handle_t * - resident_device_handles; ///< Array of devices for which the memory should be made resident + device_handles; ///< Array of all devices, null if resident devices unsupported + uint32_t + device_count; ///< Number of items on device_handles, 0 if resident devices unsupported + uint32_t * + resident_device_indices; ///< Array of indices of devices for which the memory should be made resident uint32_t resident_device_count; ///< Number of devices for which the memory should be made resident @@ -67,7 +73,9 @@ typedef struct ze_memory_provider_t { ze_device_handle_t device; ze_memory_type_t memory_type; - ze_device_handle_t *resident_device_handles; + ze_device_handle_t *device_handles; + uint32_t device_count; // just for checks + uint32_t *resident_device_indices; // not sorted uint32_t resident_device_count; ze_device_properties_t device_properties; @@ -250,7 +258,9 @@ umf_result_t umfLevelZeroMemoryProviderParamsCreate( params->level_zero_context_handle = NULL; params->level_zero_device_handle = NULL; params->memory_type = UMF_MEMORY_TYPE_UNKNOWN; - params->resident_device_handles = NULL; + params->device_handles = NULL; + params->device_count = 0; + params->resident_device_indices = NULL; params->resident_device_count = 0; params->freePolicy = UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT; params->device_ordinal = 0; @@ -265,7 +275,6 @@ umf_result_t umfLevelZeroMemoryProviderParamsCreate( umf_result_t umfLevelZeroMemoryProviderParamsDestroy( umf_level_zero_memory_provider_params_handle_t hParams) { umf_ba_global_free(hParams); - return UMF_RESULT_SUCCESS; } @@ -345,19 +354,24 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetName( umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, - ze_device_handle_t *hDevices, uint32_t deviceCount) { + ze_device_handle_t *hDevices, uint32_t deviceCount, + uint32_t *residentDevicesIndices, uint32_t residentDevicesCount) { + if (!hParams) { LOG_ERR("Level Zero memory provider params handle is NULL"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (deviceCount && !hDevices) { - LOG_ERR("Resident devices array is NULL, but deviceCount is not zero"); + if (residentDevicesCount && !residentDevicesIndices) { + LOG_ERR("Resident devices indices array is NULL, but " + "residentDevicesCount is not zero"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - hParams->resident_device_handles = hDevices; - hParams->resident_device_count = deviceCount; + hParams->device_handles = hDevices; + hParams->device_count = deviceCount; + hParams->resident_device_indices = residentDevicesIndices; + hParams->resident_device_count = residentDevicesCount; return UMF_RESULT_SUCCESS; } @@ -460,12 +474,20 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, } for (uint32_t i = 0; i < ze_provider->resident_device_count; i++) { + const uint32_t resident_device_idx = + ze_provider->resident_device_indices[i]; + ze_result = g_ze_ops.zeContextMakeMemoryResident( - ze_provider->context, ze_provider->resident_device_handles[i], - *resultPtr, size); + ze_provider->context, + ze_provider->device_handles[resident_device_idx], *resultPtr, size); if (ze_result != ZE_RESULT_SUCCESS) { + LOG_DEBUG("making resident allocation %p of size:%lu on device %u " + "failed with %d", + *resultPtr, size, resident_device_idx, ze_result); return ze2umf_result(ze_result); } + LOG_DEBUG("allocation %p of size:%lu made resident on device %u", + *resultPtr, size, resident_device_idx); } if (update_stats) { @@ -544,9 +566,13 @@ static umf_result_t query_min_page_size(ze_memory_provider_t *ze_provider, } static umf_result_t ze_memory_provider_finalize(void *provider) { - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; - umf_ba_global_free(ze_provider->resident_device_handles); - + ze_memory_provider_t *ze_provider = provider; + if (ze_provider->device_handles != NULL) { + umf_ba_global_free(ze_provider->device_handles); + } + if (ze_provider->resident_device_indices != NULL) { + umf_ba_global_free(ze_provider->resident_device_indices); + } umf_ba_global_free(provider); return UMF_RESULT_SUCCESS; } @@ -571,9 +597,23 @@ static umf_result_t ze_memory_provider_initialize(const void *params, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if ((bool)ze_params->resident_device_count && - (ze_params->resident_device_handles == NULL)) { - LOG_ERR("Resident devices handles array is NULL, but device_count is " + if (ze_params->device_count < ze_params->resident_device_count) { + LOG_ERR( + "Device count should be not less than than resident devices count"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (ze_params->device_count > 0 && ze_params->device_handles == NULL) { + LOG_ERR("Device handler should be non-NULL if device_count:%d is " + "greater than 0", + ze_params->device_count); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (ze_params->resident_device_count > 0 && + (ze_params->resident_device_indices == NULL)) { + LOG_ERR("Resident devices indices array is NULL, but " + "resident_device_count is " "not zero"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -617,24 +657,41 @@ static umf_result_t ze_memory_provider_initialize(const void *params, } } - if (ze_params->resident_device_count) { - ze_provider->resident_device_handles = umf_ba_global_alloc( - sizeof(ze_device_handle_t) * ze_params->resident_device_count); - if (!ze_provider->resident_device_handles) { - LOG_ERR("Cannot allocate memory for resident devices"); + ze_provider->device_handles = ze_params->device_handles; + + if (ze_params->device_count > 0) { + // we allocate space for maximum possible set of resident indices + ze_provider->resident_device_indices = + umf_ba_global_alloc(sizeof(uint32_t) * ze_params->device_count); + if (ze_provider->resident_device_indices == NULL) { + LOG_ERR("Cannot allocate memory for resident device indices"); umf_ba_global_free(ze_provider); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - ze_provider->resident_device_count = ze_params->resident_device_count; - - for (uint32_t i = 0; i < ze_provider->resident_device_count; i++) { - ze_provider->resident_device_handles[i] = - ze_params->resident_device_handles[i]; + ze_provider->device_handles = umf_ba_global_alloc( + sizeof(ze_device_handle_t) * ze_params->device_count); + if (ze_provider->device_handles == NULL) { + LOG_ERR("Cannot allocate memory for device handles"); + umf_ba_global_free(ze_provider->resident_device_indices); + umf_ba_global_free(ze_provider); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } + + ze_provider->resident_device_count = ze_params->resident_device_count; + memcpy(ze_provider->resident_device_indices, + ze_params->resident_device_indices, + sizeof(uint32_t) * ze_params->resident_device_count); + ze_provider->device_count = ze_params->device_count; + memcpy(ze_provider->device_handles, ze_params->device_handles, + sizeof(ze_device_handle_t) * ze_params->device_count); + + LOG_INFO("L0 memory provider:%p have %d device(s) including %d " + "resident device(s)", + (void *)ze_provider, ze_params->device_count, + ze_params->resident_device_count); } else { - ze_provider->resident_device_handles = NULL; - ze_provider->resident_device_count = 0; + LOG_INFO("L0 memory provider does not use resident devices"); } umf_result_t result = @@ -930,6 +987,155 @@ static umf_result_t ze_memory_provider_get_allocation_properties_size( return UMF_RESULT_ERROR_INVALID_ARGUMENT; } +struct ze_memory_provider_resident_device_change_data { + bool isAdding; + uint32_t peer_device_index; + ze_memory_provider_t *source_memory_provider; + uint32_t success_changes; + uint32_t failed_changes; +}; + +static int ze_memory_provider_resident_device_change_helper(uintptr_t key, + void *value, + void *privdata) { + struct ze_memory_provider_resident_device_change_data *change_data = + privdata; + tracker_alloc_info_t *info = value; + if (info->props.provider->provider_priv != + (void *)change_data->source_memory_provider) { + LOG_DEBUG("ze_memory_provider_resident_device_change found not our " + "pointer %p", + (void *)key); + return 0; + } + + ze_device_handle_t peer_device = + change_data->source_memory_provider + ->device_handles[change_data->peer_device_index]; + + // TODO: add assertions to UMF and change it to be an assertion + if (info->props.base != (void *)key) { + LOG_ERR("key:%p is different than base:%p", (void *)key, + info->props.base); + abort(); + } + + ze_result_t result; + if (change_data->isAdding) { + result = g_ze_ops.zeContextMakeMemoryResident( + change_data->source_memory_provider->context, peer_device, + info->props.base, info->props.base_size); + } else { + result = ZE_RESULT_SUCCESS; + // TODO: currently not implemented call evict here + } + + if (result != ZE_RESULT_SUCCESS) { + LOG_ERR("ze_memory_provider_resident_device_change found our pointer " + "%p but failed to make it resident on device:%p due to err:%d", + (void *)key, (void *)peer_device, result); + ++change_data->failed_changes; + return 1; + } + + LOG_DEBUG("ze_memory_provider_resident_device_change found our pointer %p " + "and made it resident on device:%p", + (void *)key, (void *)peer_device); + ++change_data->success_changes; + return 0; +} + +static umf_result_t +ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, + bool isAdding) { + ze_memory_provider_t *ze_provider = provider; + + LOG_INFO("%s resident device with id:%d, src_provider:%p, existing peers " + "count:%d", + (isAdding ? "adding" : "removing"), device_index, provider, + ze_provider->resident_device_count); + + uint32_t existing_peer_index = 0; + while (existing_peer_index < ze_provider->resident_device_count && + ze_provider->resident_device_indices[existing_peer_index] != + device_index) { + ++existing_peer_index; + } + + if (ze_provider->resident_device_count == 0 || + existing_peer_index == ze_provider->resident_device_count) { + // not found + if (!isAdding) { // impossible for UR, should be an assertion + LOG_ERR("trying to remove resident device of idx:%d but the device " + "is not a peer of provider:%p currently", + device_index, provider); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + // adding case + if (ze_provider->device_count <= + ze_provider + ->resident_device_count) { // impossible for UR, should be an assertion + LOG_ERR("trying to add resident device of idx:%d while all devices " + "were already added as resident ones", + device_index); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (ze_provider->device_count <= + device_index) { // impossible for UR, should be an assertion + LOG_ERR("using too large peer device idx:%d, devices count is %d", + device_index, ze_provider->device_count); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + ze_provider->resident_device_indices[existing_peer_index] = + device_index; + ++ze_provider->resident_device_count; + + } else { + // found + if (isAdding) { // impossible for UR, should be an assertion + LOG_ERR("trying to add resident device of idx:%d but the device is " + "already a peer of provider:%p", + device_index, provider); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + // removing case + --ze_provider->resident_device_count; + ze_provider->resident_device_indices[existing_peer_index] = + ze_provider + ->resident_device_indices[ze_provider->resident_device_count]; + } + + struct ze_memory_provider_resident_device_change_data privData = { + .isAdding = isAdding, + .peer_device_index = device_index, + .source_memory_provider = ze_provider, + .success_changes = 0, + .failed_changes = 0, + }; + + umf_result_t result = umfMemoryTrackerIterateAll( + &ze_memory_provider_resident_device_change_helper, &privData); + if (result != UMF_RESULT_SUCCESS) { + LOG_ERR("umfMemoryTrackerIterateAll failed during resident device " + "change with result:%d numFailed:%d, numSuccess:%d", + result, privData.success_changes, privData.failed_changes); + return result; + } + + if (privData.failed_changes > 0) { + LOG_ERR("umfMemoryTrackerIterateAll did not manage to do some change " + "numFailed:%d, numSuccess:%d", + privData.success_changes, privData.failed_changes); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; // probably some other result is better, best just change into assertion + } + + LOG_INFO("ze_memory_provider_resident_device_change done, numSuccess:%d", + privData.success_changes); + return UMF_RESULT_SUCCESS; +} + static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = ze_memory_provider_initialize, @@ -954,6 +1160,7 @@ static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { ze_memory_provider_get_allocation_properties, .ext_get_allocation_properties_size = ze_memory_provider_get_allocation_properties_size, + .ext_resident_device_change = ze_memory_provider_resident_device_change, }; const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index cf76d2be74..706fc013df 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -1336,6 +1336,15 @@ static umf_result_t trackingGetAllocationPropertiesSize( p->hUpstream, memory_property_id, size); } +static umf_result_t trackingResidentDeviceChange(void *provider, + uint32_t device_index, + bool is_adding) { + (void)provider; + (void)device_index; + (void)is_adding; + return UMF_RESULT_SUCCESS; +} + umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = trackingInitialize, @@ -1358,6 +1367,7 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .ext_ctl = NULL, .ext_get_allocation_properties = trackingGetAllocationProperties, .ext_get_allocation_properties_size = trackingGetAllocationPropertiesSize, + .ext_resident_device_change = trackingResidentDeviceChange, }; static void free_ipc_cache_value(void *unused, void *ipc_cache_value) { @@ -1527,3 +1537,26 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { handle->ipc_info_allocator = NULL; umf_ba_global_free(handle); } + +umf_result_t umfMemoryTrackerIterateAll(int (*func)(uintptr_t key, void *value, + void *privdata), + void *privdata) { + if (UNLIKELY(TRACKER == NULL)) { + LOG_ERR("tracker does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + if (UNLIKELY(TRACKER->alloc_segments_map[0] == NULL)) { + LOG_ERR("tracker's alloc_segments_map does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; level++) { + critnib *alloc_segment = TRACKER->alloc_segments_map[level]; + LOG_DEBUG("iterating tracker's %d segment:%p", level, + (void *)alloc_segment); + critnib_iter_all(alloc_segment, func, privdata); + } + + return UMF_RESULT_SUCCESS; +} diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index cdbee3973f..254bbf6786 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -70,6 +70,10 @@ void umfTrackingMemoryProviderGetUpstreamProvider( umf_memory_provider_handle_t hTrackingProvider, umf_memory_provider_handle_t *hUpstream); +umf_result_t umfMemoryTrackerIterateAll(int (*func)(uintptr_t key, void *value, + void *privdata), + void *privdata); + #ifdef __cplusplus } #endif diff --git a/test/common/provider_null.c b/test/common/provider_null.c index 2ce8c78ddb..ae6f8e595e 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -162,6 +162,15 @@ static umf_result_t nullGetAllocationPropertiesSize( return UMF_RESULT_SUCCESS; } +static umf_result_t nullResidentDeviceChange(void *provider, + uint32_t device_index, + bool is_adding) { + (void)provider; + (void)device_index; + (void)is_adding; + return UMF_RESULT_SUCCESS; +} + umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = nullInitialize, @@ -183,4 +192,5 @@ umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .ext_close_ipc_handle = nullCloseIpcHandle, .ext_get_allocation_properties = nullGetAllocationProperties, .ext_get_allocation_properties_size = nullGetAllocationPropertiesSize, + .ext_resident_device_change = nullResidentDeviceChange, }; From 8c5469577ef92fb9cac58ccb3442e3d6a2bd1fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Thu, 11 Sep 2025 14:45:06 +0200 Subject: [PATCH 02/49] added missing doxygen documentation --- include/umf/providers/provider_level_zero.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index 422958fbfa..6c2ad16f49 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -61,8 +61,10 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetMemoryType( /// @brief Set the resident devices in the parameters struct. /// @param hParams handle to the parameters of the Level Zero Memory Provider. -/// @param hDevices array of devices for which the memory should be made resident. -/// @param deviceCount number of devices for which the memory should be made resident. +/// @param hDevices array of all devices for which the memory can be made resident. +/// @param deviceCount number of devices for which the memory can be made resident. +/// @param residentDevicesIndices array of indices in 'hDevices' array to devices for which the memory should be made resident. +/// @param residentDevicesCount number of indices in 'residentDevicesIndices' array. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, From 064d1ac68f2b1a49be82a1ab661baf31c99e4f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 12 Sep 2025 10:46:44 +0200 Subject: [PATCH 03/49] fix documentation spelling --- include/umf/memory_pool_ops.h | 2 +- include/umf/memory_provider.h | 2 +- include/umf/memory_provider_ops.h | 2 +- include/umf/providers/provider_level_zero.h | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index dc75b8a7d9..fe6f63dff4 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -197,7 +197,7 @@ typedef struct umf_memory_pool_ops_t { /// resident. /// @param pool pointer to the memory pool /// @param peerIdx identifier of device - /// @param isAdding boolean indicating if peer is to be removed or added + /// @param isAdding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. /// diff --git a/include/umf/memory_provider.h b/include/umf/memory_provider.h index 3d31ce6b73..bff17d18fc 100644 --- a/include/umf/memory_provider.h +++ b/include/umf/memory_provider.h @@ -269,7 +269,7 @@ umfMemoryProviderAllocationMerge(umf_memory_provider_handle_t hProvider, /// resident. /// @param hProvider handle to the memory provider /// @param deviceIndex identifier of device -/// @param isAdding boolean indicating if peer is to be removed or added +/// @param isAdding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. umf_result_t diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 9efc2d3101..26b5d616fd 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -327,7 +327,7 @@ typedef struct umf_memory_provider_ops_t { /// resident. /// @param provider handle to the memory provider /// @param device_index identifier of device - /// @param is_adding boolean indicating if peer is to be removed or added + /// @param is_adding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. umf_result_t (*ext_resident_device_change)(void *provider, diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index 6c2ad16f49..6ea4c6c5d3 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -63,8 +63,8 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetMemoryType( /// @param hParams handle to the parameters of the Level Zero Memory Provider. /// @param hDevices array of all devices for which the memory can be made resident. /// @param deviceCount number of devices for which the memory can be made resident. -/// @param residentDevicesIndices array of indices in 'hDevices' array to devices for which the memory should be made resident. -/// @param residentDevicesCount number of indices in 'residentDevicesIndices' array. +/// @param residentDevicesIndices array of indices in all devices array to devices for which the memory should be made resident. +/// @param residentDevicesCount number of items in indices array. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, From c4a7499e92144644d08ad0065623623bace5f3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 12 Sep 2025 09:02:29 +0200 Subject: [PATCH 04/49] applied Rafal's comments --- include/umf/memory_pool_ops.h | 6 +++--- src/libumf.map | 2 +- src/memory_provider.c | 3 +++ src/provider/provider_level_zero.c | 3 +-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index fe6f63dff4..af01e7c0c9 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -196,13 +196,13 @@ typedef struct umf_memory_pool_ops_t { /// @brief Adds or removes devices on which allocations should be made /// resident. /// @param pool pointer to the memory pool - /// @param peerIdx identifier of device + /// @param deviceIndex identifier of device /// @param isAdding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. /// - umf_result_t (*ext_resident_device_change)(void *pool, uint32_t peerIdx, - _Bool isAdding); + umf_result_t (*ext_resident_device_change)(void *pool, uint32_t deviceIndex, + bool isAdding); } umf_memory_pool_ops_t; #ifdef __cplusplus diff --git a/src/libumf.map b/src/libumf.map index 9dfbc30d90..0dd3557c5e 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -54,7 +54,7 @@ UMF_1.0 { umfMemoryProviderPurgeForce; umfMemoryProviderPurgeLazy; umfMemoryProviderPutIPCHandle; - umfMemoryProviderResidentDeviceChange; + umfMemoryProviderResidentDeviceChange; umfMempolicyCreate; umfMempolicyDestroy; umfMempolicySetCustomSplitPartitions; diff --git a/src/memory_provider.c b/src/memory_provider.c index ad162095dd..9c5109cd8f 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -610,9 +610,12 @@ umf_result_t umfMemoryProviderGetAllocationPropertiesSize( umf_result_t umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, uint32_t deviceIndex, bool isAdding) { + UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + umf_result_t res = hProvider->ops.ext_resident_device_change( hProvider->provider_priv, deviceIndex, isAdding); + checkErrorAndSetLastProvider(res, hProvider); return res; } \ No newline at end of file diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index b856cf46ba..2cc696ecd6 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -17,11 +17,10 @@ #include "memory_provider_internal.h" #include "provider_ctl_stats_type.h" #include "provider_level_zero_internal.h" +#include "provider_tracking.h" #include "utils_load_library.h" #include "utils_log.h" -#include - static void *ze_lib_handle = NULL; void fini_ze_global_state(void) { From 588d1afe4eecf3840efa28e819c46e684d630827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 12 Sep 2025 09:49:36 +0200 Subject: [PATCH 05/49] fixed compilation error --- src/provider/provider_level_zero.c | 13 +++++++++---- test/providers/provider_level_zero.cpp | 4 ++-- test/providers/provider_level_zero_not_impl.cpp | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 2cc696ecd6..895a1c7ad5 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -678,9 +678,11 @@ static umf_result_t ze_memory_provider_initialize(const void *params, } ze_provider->resident_device_count = ze_params->resident_device_count; - memcpy(ze_provider->resident_device_indices, - ze_params->resident_device_indices, - sizeof(uint32_t) * ze_params->resident_device_count); + if (ze_params->resident_device_count > 0) { + memcpy(ze_provider->resident_device_indices, + ze_params->resident_device_indices, + sizeof(uint32_t) * ze_params->resident_device_count); + } ze_provider->device_count = ze_params->device_count; memcpy(ze_provider->device_handles, ze_params->device_handles, sizeof(ze_device_handle_t) * ze_params->device_count); @@ -1216,10 +1218,13 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetMemoryType( umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, - ze_device_handle_t *hDevices, uint32_t deviceCount) { + ze_device_handle_t *hDevices, uint32_t deviceCount, + uint32_t *residentDevicesIndices, uint32_t residentDevicesCount) { (void)hParams; (void)hDevices; (void)deviceCount; + (void)residentDevicesIndices; + (void)residentDevicesCount; LOG_ERR("L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " "OFF)"); return UMF_RESULT_ERROR_NOT_SUPPORTED; diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index d91660399b..c7c5cca820 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -212,8 +212,8 @@ TEST_F(test, FailMismatchedResidentHandlesCount) { umf_result_t result = umfLevelZeroMemoryProviderParamsCreate(&hParams); ASSERT_EQ(result, UMF_RESULT_SUCCESS); - result = umfLevelZeroMemoryProviderParamsSetResidentDevices(hParams, - nullptr, 99); + result = umfLevelZeroMemoryProviderParamsSetResidentDevices( + hParams, nullptr, 99, nullptr, 0); ASSERT_EQ(result, UMF_RESULT_ERROR_INVALID_ARGUMENT); umfLevelZeroMemoryProviderParamsDestroy(hParams); diff --git a/test/providers/provider_level_zero_not_impl.cpp b/test/providers/provider_level_zero_not_impl.cpp index dfbd4a2b21..b255ec5780 100644 --- a/test/providers/provider_level_zero_not_impl.cpp +++ b/test/providers/provider_level_zero_not_impl.cpp @@ -27,8 +27,8 @@ TEST_F(test, level_zero_provider_not_implemented) { ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); ze_device_handle_t hDevices[1]; - result = umfLevelZeroMemoryProviderParamsSetResidentDevices(hParams, - hDevices, 1); + result = umfLevelZeroMemoryProviderParamsSetResidentDevices( + hParams, hDevices, 1, nullptr, 0); ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); result = umfLevelZeroMemoryProviderParamsSetFreePolicy( From b24574cacc7a72e3711653f8b751d6feb432adf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 12 Sep 2025 12:12:12 +0200 Subject: [PATCH 06/49] applied second round of Rafal's comments --- src/memory_provider.c | 13 ++++++++ src/provider/provider_cuda.c | 11 +------ src/provider/provider_level_zero.c | 50 +++++++++++++++++++++++------- src/provider/provider_tracking.c | 11 +------ 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/memory_provider.c b/src/memory_provider.c index 9c5109cd8f..6c6042174f 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -154,6 +154,15 @@ static umf_result_t umfDefaultGetAllocationPropertiesSize( return UMF_RESULT_ERROR_NOT_SUPPORTED; } +static umf_result_t umfDefaultResidentDeviceChange(void *provider, + uint32_t device_index, + bool is_adding) { + (void)provider; + (void)device_index; + (void)is_adding; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { if (!ops->ext_purge_lazy) { ops->ext_purge_lazy = umfDefaultPurgeLazy; @@ -183,6 +192,10 @@ void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { ops->ext_get_allocation_properties_size = umfDefaultGetAllocationPropertiesSize; } + + if (!ops->ext_resident_device_change) { + ops->ext_resident_device_change = umfDefaultResidentDeviceChange; + } } void assignOpsIpcDefaults(umf_memory_provider_ops_t *ops) { diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index c867f76566..4f4a00a3d5 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -802,15 +802,6 @@ static umf_result_t cu_memory_provider_get_allocation_properties_size( return UMF_RESULT_ERROR_INVALID_ARGUMENT; } -static umf_result_t -cu_memory_provider_resident_device_change(void *provider, uint32_t device_index, - bool is_adding) { - (void)provider; - (void)device_index; - (void)is_adding; - return UMF_RESULT_SUCCESS; -} - static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = cu_memory_provider_initialize, @@ -838,7 +829,7 @@ static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { cu_memory_provider_get_allocation_properties, .ext_get_allocation_properties_size = cu_memory_provider_get_allocation_properties_size, - .ext_resident_device_change = cu_memory_provider_resident_device_change, + .ext_resident_device_change = NULL, }; const umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void) { diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 895a1c7ad5..f534509899 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -361,12 +361,37 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (residentDevicesCount && !residentDevicesIndices) { + if (residentDevicesCount > 0 && residentDevicesIndices == NULL) { LOG_ERR("Resident devices indices array is NULL, but " "residentDevicesCount is not zero"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } + if (deviceCount > 0 && hDevices == NULL) { + LOG_ERR("All devices array is NULL, but deviceCount is not zero"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + for (uint32_t first_idx = 0; first_idx < residentDevicesCount; + first_idx++) { + if (residentDevicesIndices[first_idx] >= deviceCount) { + LOG_ERR("Resident device index:%u is out of range, should be less " + "than deviceCount:%u", + residentDevicesIndices[first_idx], deviceCount); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + for (uint32_t second_idx = 0; second_idx < first_idx; second_idx++) { + if (residentDevicesIndices[first_idx] == + residentDevicesIndices[second_idx]) { + LOG_ERR("resident device indices are not unique, idx:%u and " + "idx:%u both point to indice:%u", + first_idx, second_idx, + residentDevicesIndices[first_idx]); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + } + } + hParams->device_handles = hDevices; hParams->device_count = deviceCount; hParams->resident_device_indices = residentDevicesIndices; @@ -989,7 +1014,7 @@ static umf_result_t ze_memory_provider_get_allocation_properties_size( } struct ze_memory_provider_resident_device_change_data { - bool isAdding; + bool is_adding; uint32_t peer_device_index; ze_memory_provider_t *source_memory_provider; uint32_t success_changes; @@ -1022,7 +1047,7 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, } ze_result_t result; - if (change_data->isAdding) { + if (change_data->is_adding) { result = g_ze_ops.zeContextMakeMemoryResident( change_data->source_memory_provider->context, peer_device, info->props.base, info->props.base_size); @@ -1048,12 +1073,12 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, static umf_result_t ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, - bool isAdding) { + bool is_adding) { ze_memory_provider_t *ze_provider = provider; LOG_INFO("%s resident device with id:%d, src_provider:%p, existing peers " "count:%d", - (isAdding ? "adding" : "removing"), device_index, provider, + (is_adding ? "adding" : "removing"), device_index, provider, ze_provider->resident_device_count); uint32_t existing_peer_index = 0; @@ -1066,7 +1091,8 @@ ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, if (ze_provider->resident_device_count == 0 || existing_peer_index == ze_provider->resident_device_count) { // not found - if (!isAdding) { // impossible for UR, should be an assertion + if (!is_adding) { + // impossible for UR, should be an assertion LOG_ERR("trying to remove resident device of idx:%d but the device " "is not a peer of provider:%p currently", device_index, provider); @@ -1082,8 +1108,8 @@ ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (ze_provider->device_count <= - device_index) { // impossible for UR, should be an assertion + if (ze_provider->device_count <= device_index) { + // impossible for UR, should be an assertion LOG_ERR("using too large peer device idx:%d, devices count is %d", device_index, ze_provider->device_count); return UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -1095,7 +1121,8 @@ ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, } else { // found - if (isAdding) { // impossible for UR, should be an assertion + if (is_adding) { + // impossible for UR, should be an assertion LOG_ERR("trying to add resident device of idx:%d but the device is " "already a peer of provider:%p", device_index, provider); @@ -1109,7 +1136,7 @@ ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, } struct ze_memory_provider_resident_device_change_data privData = { - .isAdding = isAdding, + .is_adding = is_adding, .peer_device_index = device_index, .source_memory_provider = ze_provider, .success_changes = 0, @@ -1129,7 +1156,8 @@ ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, LOG_ERR("umfMemoryTrackerIterateAll did not manage to do some change " "numFailed:%d, numSuccess:%d", privData.success_changes, privData.failed_changes); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; // probably some other result is better, best just change into assertion + // TODO: change into permanent assertion when avail + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; } LOG_INFO("ze_memory_provider_resident_device_change done, numSuccess:%d", diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 706fc013df..e0ab69736e 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -1336,15 +1336,6 @@ static umf_result_t trackingGetAllocationPropertiesSize( p->hUpstream, memory_property_id, size); } -static umf_result_t trackingResidentDeviceChange(void *provider, - uint32_t device_index, - bool is_adding) { - (void)provider; - (void)device_index; - (void)is_adding; - return UMF_RESULT_SUCCESS; -} - umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = trackingInitialize, @@ -1367,7 +1358,7 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .ext_ctl = NULL, .ext_get_allocation_properties = trackingGetAllocationProperties, .ext_get_allocation_properties_size = trackingGetAllocationPropertiesSize, - .ext_resident_device_change = trackingResidentDeviceChange, + .ext_resident_device_change = NULL, }; static void free_ipc_cache_value(void *unused, void *ipc_cache_value) { From 5983cfa07da01e7f3dcb2bf053332aa425e7d7cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 15 Sep 2025 11:22:06 +0200 Subject: [PATCH 07/49] applied Lukasz's comments --- src/critnib/critnib.c | 2 +- src/libumf.map | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index 1e2e7c14b6..8ff5796684 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -1138,7 +1138,7 @@ void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, } utils_mutex_unlock(&c->mutex); if (!wasIterating) { - LOG_DEBUG("there was no root, iterating critnib:%p was skipped", + LOG_DEBUG("there was no root, iterating critnib: %p was skipped", (void *)c); } } diff --git a/src/libumf.map b/src/libumf.map index 0dd3557c5e..78ecc1ab01 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -54,7 +54,6 @@ UMF_1.0 { umfMemoryProviderPurgeForce; umfMemoryProviderPurgeLazy; umfMemoryProviderPutIPCHandle; - umfMemoryProviderResidentDeviceChange; umfMempolicyCreate; umfMempolicyDestroy; umfMempolicySetCustomSplitPartitions; @@ -138,6 +137,7 @@ UMF_1.0 { umfJemallocPoolParamsCreate; umfJemallocPoolParamsDestroy; umfJemallocPoolParamsSetNumArenas; + umfMemoryProviderResidentDeviceChange; umfPoolGetName; local: *; From 5cbbad2688cb2aece5ba6f40313dbe2e707d9734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 15 Sep 2025 11:24:56 +0200 Subject: [PATCH 08/49] one more missed Lukasz comment applied --- src/libumf.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libumf.def b/src/libumf.def index 114682d3db..f1784e21cc 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -41,7 +41,6 @@ EXPORTS umfLevelZeroMemoryProviderParamsSetContext umfLevelZeroMemoryProviderParamsSetDevice umfLevelZeroMemoryProviderParamsSetMemoryType - umfLevelZeroMemoryProviderParamsSetResidentDevices umfMemoryProviderAlloc umfMemoryProviderAllocationMerge umfMemoryProviderAllocationSplit @@ -144,6 +143,7 @@ EXPORTS umfJemallocPoolParamsCreate umfJemallocPoolParamsDestroy umfJemallocPoolParamsSetNumArenas + umfLevelZeroMemoryProviderParamsSetResidentDevices umfPoolGetName ; Added in UMF_1.1 umfCUDAMemoryProviderParamsSetName From c27407a0d3048aa1c72a439759897184d529ebac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 15 Sep 2025 13:31:40 +0200 Subject: [PATCH 09/49] added some tests for resident device feature --- test/providers/provider_level_zero.cpp | 40 ++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index c7c5cca820..b8ca69d16b 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -204,21 +204,49 @@ TEST_F(LevelZeroProviderInit, FailNonNullDevice) { umfLevelZeroMemoryProviderParamsDestroy(hParams); } -TEST_F(test, FailMismatchedResidentHandlesCount) { +static void invalidResidentDevicesHandlesTestHelper( + ze_device_handle_t *hDevices, uint32_t deviceCount, + uint32_t *residentDevicesIndices, uint32_t residentDevicesCount) { const umf_memory_provider_ops_t *ops = umfLevelZeroMemoryProviderOps(); ASSERT_NE(ops, nullptr); umf_level_zero_memory_provider_params_handle_t hParams = nullptr; - umf_result_t result = umfLevelZeroMemoryProviderParamsCreate(&hParams); - ASSERT_EQ(result, UMF_RESULT_SUCCESS); + const umf_result_t create_result = + umfLevelZeroMemoryProviderParamsCreate(&hParams); + ASSERT_EQ(create_result, UMF_RESULT_SUCCESS); - result = umfLevelZeroMemoryProviderParamsSetResidentDevices( - hParams, nullptr, 99, nullptr, 0); - ASSERT_EQ(result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + const umf_result_t set_resident_result = + umfLevelZeroMemoryProviderParamsSetResidentDevices( + hParams, hDevices, deviceCount, residentDevicesIndices, + residentDevicesCount); + ASSERT_EQ(set_resident_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); umfLevelZeroMemoryProviderParamsDestroy(hParams); } +TEST_F(test, FailMismatchedResidentHandlesCount) { + invalidResidentDevicesHandlesTestHelper(nullptr, 99, nullptr, 0); +} + +TEST_F(test, FailMismatchedResidentDeviceIndicesCount) { + std::vector allDevices(23); + invalidResidentDevicesHandlesTestHelper(allDevices.data(), 23, nullptr, 1); +} + +TEST_F(test, FailRedundantResidentDeviceIndices) { + std::vector allDevices(23); + std::vector residentDevicesIndices{3, 5, 1, 5}; // double 5 + invalidResidentDevicesHandlesTestHelper(allDevices.data(), 23, + residentDevicesIndices.data(), 4); +} + +TEST_F(test, FailTooLargeResidentDeviceIndex) { + std::vector allDevices(23); + std::vector residentDevicesIndices{123}; + invalidResidentDevicesHandlesTestHelper(allDevices.data(), 23, + residentDevicesIndices.data(), 1); +} + class LevelZeroMemoryAccessor : public MemoryAccessor { public: LevelZeroMemoryAccessor(ze_context_handle_t hContext, From 53ec7535dd75b09068906aa3f2f1a129e1edf599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Tue, 16 Sep 2025 07:55:58 +0200 Subject: [PATCH 10/49] add unittests and refactor to be backward compatible --- include/umf/memory_pool_ops.h | 6 +- include/umf/memory_provider.h | 4 +- include/umf/memory_provider_ops.h | 5 +- include/umf/providers/provider_level_zero.h | 9 +- src/libumf.def | 2 +- src/memory_provider.c | 9 +- src/provider/provider_level_zero.c | 256 +++++++--------- src/utils/utils_concurrency.h | 8 +- src/utils/utils_level_zero.cpp | 3 - src/utils/utils_log.h | 30 +- src/utils/utils_posix_concurrency.c | 34 ++- src/utils/utils_windows_concurrency.c | 20 +- test/CMakeLists.txt | 22 +- test/common/CMakeLists.txt | 21 ++ test/common/level_zero_mocks.cpp | 87 ++++++ test/common/level_zero_mocks.h | 77 +++++ test/common/provider_null.c | 5 +- test/common/ze_loopback.cpp | 285 ++++++++++++++++++ test/common/ze_loopback.h | 35 +++ test/pools/pool_residency.cpp | 133 ++++++++ test/providers/provider_level_zero.cpp | 36 +-- .../provider_level_zero_not_impl.cpp | 4 +- .../provider_level_zero_residency.cpp | 79 +++++ 23 files changed, 929 insertions(+), 241 deletions(-) create mode 100644 test/common/level_zero_mocks.cpp create mode 100644 test/common/level_zero_mocks.h create mode 100644 test/common/ze_loopback.cpp create mode 100644 test/common/ze_loopback.h create mode 100644 test/pools/pool_residency.cpp create mode 100644 test/providers/provider_level_zero_residency.cpp diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index af01e7c0c9..aa6db9a1f8 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -167,7 +167,7 @@ typedef struct umf_memory_pool_ops_t { /// /// @return umf_result_t result of the control operation. /// - umf_result_t (*ext_ctl)(void *hPool, umf_ctl_query_source_t source, + umf_result_t (*ext_ctl)(void *pool, umf_ctl_query_source_t source, const char *name, void *arg, size_t size, umf_ctl_query_type_t queryType, va_list args); @@ -196,12 +196,12 @@ typedef struct umf_memory_pool_ops_t { /// @brief Adds or removes devices on which allocations should be made /// resident. /// @param pool pointer to the memory pool - /// @param deviceIndex identifier of device + /// @param device device handle /// @param isAdding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. /// - umf_result_t (*ext_resident_device_change)(void *pool, uint32_t deviceIndex, + umf_result_t (*ext_resident_device_change)(void *pool, void *device, bool isAdding); } umf_memory_pool_ops_t; diff --git a/include/umf/memory_provider.h b/include/umf/memory_provider.h index bff17d18fc..a396a8a9d3 100644 --- a/include/umf/memory_provider.h +++ b/include/umf/memory_provider.h @@ -268,13 +268,13 @@ umfMemoryProviderAllocationMerge(umf_memory_provider_handle_t hProvider, /// @brief Adds or removes devices on which allocations should be made /// resident. /// @param hProvider handle to the memory provider -/// @param deviceIndex identifier of device +/// @param device device handle /// @param isAdding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. umf_result_t umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, - uint32_t deviceIndex, bool isAdding); + void *device, bool isAdding); #ifdef __cplusplus } diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 26b5d616fd..406da06d36 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -326,12 +326,11 @@ typedef struct umf_memory_provider_ops_t { /// @brief Adds or removes devices on which allocations should be made /// resident. /// @param provider handle to the memory provider - /// @param device_index identifier of device + /// @param device device handle /// @param is_adding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. - umf_result_t (*ext_resident_device_change)(void *provider, - uint32_t device_index, + umf_result_t (*ext_resident_device_change)(void *provider, void *device, bool is_adding); } umf_memory_provider_ops_t; diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index 6ea4c6c5d3..65d7e3e781 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -61,15 +61,12 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetMemoryType( /// @brief Set the resident devices in the parameters struct. /// @param hParams handle to the parameters of the Level Zero Memory Provider. -/// @param hDevices array of all devices for which the memory can be made resident. -/// @param deviceCount number of devices for which the memory can be made resident. -/// @param residentDevicesIndices array of indices in all devices array to devices for which the memory should be made resident. -/// @param residentDevicesCount number of items in indices array. +/// @param hDevices array of devices for which the memory should be made resident. +/// @param deviceCount number of devices for which the memory should be made resident. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, - ze_device_handle_t *hDevices, uint32_t deviceCount, - uint32_t *residentDevicesIndices, uint32_t residentDevicesCount); + ze_device_handle_t *hDevices, uint32_t deviceCount); typedef enum umf_level_zero_memory_provider_free_policy_t { UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT = diff --git a/src/libumf.def b/src/libumf.def index f1784e21cc..5955088b8e 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -19,6 +19,7 @@ EXPORTS umfCUDAMemoryProviderParamsSetContext umfCUDAMemoryProviderParamsSetDevice umfCUDAMemoryProviderParamsSetMemoryType + umfLevelZeroMemoryProviderParamsSetResidentDevices umfDevDaxMemoryProviderOps umfDevDaxMemoryProviderParamsCreate umfDevDaxMemoryProviderParamsDestroy @@ -143,7 +144,6 @@ EXPORTS umfJemallocPoolParamsCreate umfJemallocPoolParamsDestroy umfJemallocPoolParamsSetNumArenas - umfLevelZeroMemoryProviderParamsSetResidentDevices umfPoolGetName ; Added in UMF_1.1 umfCUDAMemoryProviderParamsSetName diff --git a/src/memory_provider.c b/src/memory_provider.c index 6c6042174f..6771bffaa4 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -154,11 +154,10 @@ static umf_result_t umfDefaultGetAllocationPropertiesSize( return UMF_RESULT_ERROR_NOT_SUPPORTED; } -static umf_result_t umfDefaultResidentDeviceChange(void *provider, - uint32_t device_index, +static umf_result_t umfDefaultResidentDeviceChange(void *provider, void *device, bool is_adding) { (void)provider; - (void)device_index; + (void)device; (void)is_adding; return UMF_RESULT_ERROR_NOT_SUPPORTED; } @@ -622,12 +621,12 @@ umf_result_t umfMemoryProviderGetAllocationPropertiesSize( umf_result_t umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, - uint32_t deviceIndex, bool isAdding) { + void *device, bool isAdding) { UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); umf_result_t res = hProvider->ops.ext_resident_device_change( - hProvider->provider_priv, deviceIndex, isAdding); + hProvider->provider_priv, device, isAdding); checkErrorAndSetLastProvider(res, hProvider); return res; diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index f534509899..4c7ace96e5 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -52,11 +52,7 @@ typedef struct umf_level_zero_memory_provider_params_t { umf_usm_memory_type_t memory_type; ///< Allocation memory type ze_device_handle_t * - device_handles; ///< Array of all devices, null if resident devices unsupported - uint32_t - device_count; ///< Number of items on device_handles, 0 if resident devices unsupported - uint32_t * - resident_device_indices; ///< Array of indices of devices for which the memory should be made resident + resident_device_handles; ///< Array of devices for which the memory should be made resident uint32_t resident_device_count; ///< Number of devices for which the memory should be made resident @@ -72,10 +68,10 @@ typedef struct ze_memory_provider_t { ze_device_handle_t device; ze_memory_type_t memory_type; - ze_device_handle_t *device_handles; - uint32_t device_count; // just for checks - uint32_t *resident_device_indices; // not sorted + utils_rwlock_t resident_device_rwlock; + ze_device_handle_t *resident_device_handles; uint32_t resident_device_count; + uint32_t resident_device_capacity; ze_device_properties_t device_properties; @@ -181,11 +177,18 @@ static ze_memory_type_t umf2ze_memory_type(umf_usm_memory_type_t memory_type) { } static void init_ze_global_state(void) { + + char *lib_name = getenv("UMF_ZE_LOADER_LIB_NAME"); + if (lib_name != NULL && lib_name[0] != '\0') { + LOG_INFO("Using custom ze_loader library name: %s", lib_name); + } else { #ifdef _WIN32 - const char *lib_name = "ze_loader.dll"; + lib_name = "ze_loader.dll"; #else - const char *lib_name = "libze_loader.so.1"; + lib_name = "libze_loader.so.1"; #endif + LOG_DEBUG("Using default ze_loader library name: %s", lib_name); + } // The Level Zero shared library should be already loaded by the user // of the Level Zero provider. UMF just want to reuse it // and increase the reference count to the Level Zero shared library. @@ -257,9 +260,7 @@ umf_result_t umfLevelZeroMemoryProviderParamsCreate( params->level_zero_context_handle = NULL; params->level_zero_device_handle = NULL; params->memory_type = UMF_MEMORY_TYPE_UNKNOWN; - params->device_handles = NULL; - params->device_count = 0; - params->resident_device_indices = NULL; + params->resident_device_handles = NULL; params->resident_device_count = 0; params->freePolicy = UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT; params->device_ordinal = 0; @@ -353,49 +354,31 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetName( umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, - ze_device_handle_t *hDevices, uint32_t deviceCount, - uint32_t *residentDevicesIndices, uint32_t residentDevicesCount) { + ze_device_handle_t *hDevices, uint32_t deviceCount) { if (!hParams) { LOG_ERR("Level Zero memory provider params handle is NULL"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (residentDevicesCount > 0 && residentDevicesIndices == NULL) { - LOG_ERR("Resident devices indices array is NULL, but " - "residentDevicesCount is not zero"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - if (deviceCount > 0 && hDevices == NULL) { - LOG_ERR("All devices array is NULL, but deviceCount is not zero"); + LOG_ERR("Resident devices array is NULL, but deviceCount is not zero"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - for (uint32_t first_idx = 0; first_idx < residentDevicesCount; - first_idx++) { - if (residentDevicesIndices[first_idx] >= deviceCount) { - LOG_ERR("Resident device index:%u is out of range, should be less " - "than deviceCount:%u", - residentDevicesIndices[first_idx], deviceCount); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } + for (uint32_t first_idx = 0; first_idx < deviceCount; first_idx++) { for (uint32_t second_idx = 0; second_idx < first_idx; second_idx++) { - if (residentDevicesIndices[first_idx] == - residentDevicesIndices[second_idx]) { - LOG_ERR("resident device indices are not unique, idx:%u and " - "idx:%u both point to indice:%u", - first_idx, second_idx, - residentDevicesIndices[first_idx]); + if (hDevices[first_idx] == hDevices[second_idx]) { + LOG_ERR("resident devices are not unique, idx:%u and " + "idx:%u both point to device:%p", + first_idx, second_idx, hDevices[first_idx]); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } } } - hParams->device_handles = hDevices; - hParams->device_count = deviceCount; - hParams->resident_device_indices = residentDevicesIndices; - hParams->resident_device_count = residentDevicesCount; + hParams->resident_device_handles = hDevices; + hParams->resident_device_count = deviceCount; return UMF_RESULT_SUCCESS; } @@ -497,22 +480,24 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, return ze2umf_result(ze_result); } + utils_read_lock(&ze_provider->resident_device_rwlock); for (uint32_t i = 0; i < ze_provider->resident_device_count; i++) { - const uint32_t resident_device_idx = - ze_provider->resident_device_indices[i]; ze_result = g_ze_ops.zeContextMakeMemoryResident( - ze_provider->context, - ze_provider->device_handles[resident_device_idx], *resultPtr, size); + ze_provider->context, ze_provider->resident_device_handles[i], + *resultPtr, size); if (ze_result != ZE_RESULT_SUCCESS) { - LOG_DEBUG("making resident allocation %p of size:%lu on device %u " + utils_read_unlock(&ze_provider->resident_device_rwlock); + LOG_DEBUG("making resident allocation %p of size:%lu on device %p " "failed with %d", - *resultPtr, size, resident_device_idx, ze_result); + *resultPtr, size, ze_provider->resident_device_handles[i], + ze_result); return ze2umf_result(ze_result); } - LOG_DEBUG("allocation %p of size:%lu made resident on device %u", - *resultPtr, size, resident_device_idx); + LOG_DEBUG("allocation %p of size:%lu made resident on device %p", + *resultPtr, size, ze_provider->resident_device_handles[i]); } + utils_read_unlock(&ze_provider->resident_device_rwlock); if (update_stats) { provider_ctl_stats_alloc(ze_provider, size); @@ -591,12 +576,10 @@ static umf_result_t query_min_page_size(ze_memory_provider_t *ze_provider, static umf_result_t ze_memory_provider_finalize(void *provider) { ze_memory_provider_t *ze_provider = provider; - if (ze_provider->device_handles != NULL) { - umf_ba_global_free(ze_provider->device_handles); - } - if (ze_provider->resident_device_indices != NULL) { - umf_ba_global_free(ze_provider->resident_device_indices); + if (ze_provider->resident_device_handles != NULL) { + umf_ba_global_free(ze_provider->resident_device_handles); } + utils_rwlock_destroy_not_free(&ze_provider->resident_device_rwlock); umf_ba_global_free(provider); return UMF_RESULT_SUCCESS; } @@ -621,24 +604,11 @@ static umf_result_t ze_memory_provider_initialize(const void *params, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (ze_params->device_count < ze_params->resident_device_count) { - LOG_ERR( - "Device count should be not less than than resident devices count"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (ze_params->device_count > 0 && ze_params->device_handles == NULL) { + if (ze_params->resident_device_count > 0 && + ze_params->resident_device_handles == NULL) { LOG_ERR("Device handler should be non-NULL if device_count:%d is " "greater than 0", - ze_params->device_count); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (ze_params->resident_device_count > 0 && - (ze_params->resident_device_indices == NULL)) { - LOG_ERR("Resident devices indices array is NULL, but " - "resident_device_count is " - "not zero"); + ze_params->resident_device_count); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -681,46 +651,36 @@ static umf_result_t ze_memory_provider_initialize(const void *params, } } - ze_provider->device_handles = ze_params->device_handles; + if (utils_rwlock_init(&ze_provider->resident_device_rwlock) == NULL) { + LOG_ERR("Cannot initialize resident device rwlock"); + umf_ba_global_free(ze_provider); + return UMF_RESULT_ERROR_OUT_OF_RESOURCES; + } - if (ze_params->device_count > 0) { - // we allocate space for maximum possible set of resident indices - ze_provider->resident_device_indices = - umf_ba_global_alloc(sizeof(uint32_t) * ze_params->device_count); - if (ze_provider->resident_device_indices == NULL) { - LOG_ERR("Cannot allocate memory for resident device indices"); - umf_ba_global_free(ze_provider); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } + ze_provider->resident_device_count = ze_params->resident_device_count; + ze_provider->resident_device_capacity = ze_params->resident_device_count; - ze_provider->device_handles = umf_ba_global_alloc( - sizeof(ze_device_handle_t) * ze_params->device_count); - if (ze_provider->device_handles == NULL) { - LOG_ERR("Cannot allocate memory for device handles"); - umf_ba_global_free(ze_provider->resident_device_indices); + if (ze_params->resident_device_count > 0) { + ze_provider->resident_device_handles = umf_ba_global_alloc( + sizeof(ze_device_handle_t) * ze_params->resident_device_count); + if (ze_provider->resident_device_handles == NULL) { + LOG_ERR("Cannot allocate memory for resident devices"); + utils_rwlock_destroy_not_free(&ze_provider->resident_device_rwlock); umf_ba_global_free(ze_provider); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - ze_provider->resident_device_count = ze_params->resident_device_count; - if (ze_params->resident_device_count > 0) { - memcpy(ze_provider->resident_device_indices, - ze_params->resident_device_indices, - sizeof(uint32_t) * ze_params->resident_device_count); - } - ze_provider->device_count = ze_params->device_count; - memcpy(ze_provider->device_handles, ze_params->device_handles, - sizeof(ze_device_handle_t) * ze_params->device_count); - - LOG_INFO("L0 memory provider:%p have %d device(s) including %d " - "resident device(s)", - (void *)ze_provider, ze_params->device_count, - ze_params->resident_device_count); + memcpy(ze_provider->resident_device_handles, + ze_params->resident_device_handles, + sizeof(ze_device_handle_t) * ze_params->resident_device_count); + + LOG_INFO("L0 memory provider:%p have %d resident device(s)", + (void *)ze_provider, ze_params->resident_device_count); } else { - LOG_INFO("L0 memory provider does not use resident devices"); + LOG_INFO("L0 memory provider has no resident devices"); } - umf_result_t result = + const umf_result_t result = query_min_page_size(ze_provider, &ze_provider->min_page_size); if (result != UMF_RESULT_SUCCESS) { ze_memory_provider_finalize(ze_provider); @@ -1015,7 +975,7 @@ static umf_result_t ze_memory_provider_get_allocation_properties_size( struct ze_memory_provider_resident_device_change_data { bool is_adding; - uint32_t peer_device_index; + ze_device_handle_t peer_device; ze_memory_provider_t *source_memory_provider; uint32_t success_changes; uint32_t failed_changes; @@ -1035,10 +995,6 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, return 0; } - ze_device_handle_t peer_device = - change_data->source_memory_provider - ->device_handles[change_data->peer_device_index]; - // TODO: add assertions to UMF and change it to be an assertion if (info->props.base != (void *)key) { LOG_ERR("key:%p is different than base:%p", (void *)key, @@ -1049,8 +1005,8 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, ze_result_t result; if (change_data->is_adding) { result = g_ze_ops.zeContextMakeMemoryResident( - change_data->source_memory_provider->context, peer_device, - info->props.base, info->props.base_size); + change_data->source_memory_provider->context, + change_data->peer_device, info->props.base, info->props.base_size); } else { result = ZE_RESULT_SUCCESS; // TODO: currently not implemented call evict here @@ -1059,32 +1015,34 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, if (result != ZE_RESULT_SUCCESS) { LOG_ERR("ze_memory_provider_resident_device_change found our pointer " "%p but failed to make it resident on device:%p due to err:%d", - (void *)key, (void *)peer_device, result); + (void *)key, (void *)change_data->peer_device, result); ++change_data->failed_changes; return 1; } LOG_DEBUG("ze_memory_provider_resident_device_change found our pointer %p " "and made it resident on device:%p", - (void *)key, (void *)peer_device); + (void *)key, (void *)change_data->peer_device); ++change_data->success_changes; return 0; } -static umf_result_t -ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, - bool is_adding) { +static umf_result_t ze_memory_provider_resident_device_change(void *provider, + void *device, + bool is_adding) { ze_memory_provider_t *ze_provider = provider; + ze_device_handle_t ze_device = device; - LOG_INFO("%s resident device with id:%d, src_provider:%p, existing peers " + LOG_INFO("%s resident device %p, src_provider:%p, existing peers " "count:%d", - (is_adding ? "adding" : "removing"), device_index, provider, + (is_adding ? "adding" : "removing"), ze_device, provider, ze_provider->resident_device_count); uint32_t existing_peer_index = 0; + utils_write_lock(&ze_provider->resident_device_rwlock); while (existing_peer_index < ze_provider->resident_device_count && - ze_provider->resident_device_indices[existing_peer_index] != - device_index) { + ze_provider->resident_device_handles[existing_peer_index] != + ze_device) { ++existing_peer_index; } @@ -1092,52 +1050,60 @@ ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, existing_peer_index == ze_provider->resident_device_count) { // not found if (!is_adding) { - // impossible for UR, should be an assertion - LOG_ERR("trying to remove resident device of idx:%d but the device " + utils_write_unlock(&ze_provider->resident_device_rwlock); + LOG_ERR("trying to remove resident device %p but the device " "is not a peer of provider:%p currently", - device_index, provider); + ze_device, provider); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } // adding case - if (ze_provider->device_count <= - ze_provider - ->resident_device_count) { // impossible for UR, should be an assertion - LOG_ERR("trying to add resident device of idx:%d while all devices " - "were already added as resident ones", - device_index); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (ze_provider->device_count <= device_index) { - // impossible for UR, should be an assertion - LOG_ERR("using too large peer device idx:%d, devices count is %d", - device_index, ze_provider->device_count); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; + if (ze_provider->resident_device_count == + ze_provider->resident_device_capacity) { + const uint32_t new_capacity = + ze_provider->resident_device_capacity * 2 + + 1; // +1 to work also with old capacity == 0 + ze_device_handle_t *new_handles = + umf_ba_global_alloc(sizeof(ze_device_handle_t) * new_capacity); + if (new_handles == NULL) { + utils_write_unlock(&ze_provider->resident_device_rwlock); + LOG_ERR("enlarging resident devices array from %u to %u failed " + "due to no memory", + ze_provider->resident_device_capacity, new_capacity); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + LOG_DEBUG("enlarging resident devices array from %u to %u", + ze_provider->resident_device_capacity, new_capacity); + memcpy(new_handles, ze_provider->resident_device_handles, + sizeof(ze_device_handle_t) * + ze_provider->resident_device_count); + umf_ba_global_free(ze_provider->resident_device_handles); + ze_provider->resident_device_handles = new_handles; + ze_provider->resident_device_capacity = new_capacity; } - - ze_provider->resident_device_indices[existing_peer_index] = - device_index; + ze_provider->resident_device_handles[existing_peer_index] = ze_device; ++ze_provider->resident_device_count; } else { // found if (is_adding) { + utils_write_unlock(&ze_provider->resident_device_rwlock); // impossible for UR, should be an assertion - LOG_ERR("trying to add resident device of idx:%d but the device is " + LOG_ERR("trying to add resident device:%p but the device is " "already a peer of provider:%p", - device_index, provider); + ze_device, provider); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - // removing case + // removing case, put last in place of removed one --ze_provider->resident_device_count; - ze_provider->resident_device_indices[existing_peer_index] = + ze_provider->resident_device_handles[existing_peer_index] = ze_provider - ->resident_device_indices[ze_provider->resident_device_count]; + ->resident_device_handles[ze_provider->resident_device_count]; } + utils_write_unlock(&ze_provider->resident_device_rwlock); struct ze_memory_provider_resident_device_change_data privData = { .is_adding = is_adding, - .peer_device_index = device_index, + .peer_device = ze_device, .source_memory_provider = ze_provider, .success_changes = 0, .failed_changes = 0, @@ -1156,7 +1122,6 @@ ze_memory_provider_resident_device_change(void *provider, uint32_t device_index, LOG_ERR("umfMemoryTrackerIterateAll did not manage to do some change " "numFailed:%d, numSuccess:%d", privData.success_changes, privData.failed_changes); - // TODO: change into permanent assertion when avail return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; } @@ -1246,13 +1211,10 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetMemoryType( umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, - ze_device_handle_t *hDevices, uint32_t deviceCount, - uint32_t *residentDevicesIndices, uint32_t residentDevicesCount) { + ze_device_handle_t *hDevices, uint32_t deviceCount) { (void)hParams; (void)hDevices; (void)deviceCount; - (void)residentDevicesIndices; - (void)residentDevicesCount; LOG_ERR("L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " "OFF)"); return UMF_RESULT_ERROR_NOT_SUPPORTED; diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index a00b8bc405..922f992e83 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -73,10 +73,10 @@ typedef struct utils_rwlock_t { utils_rwlock_t *utils_rwlock_init(utils_rwlock_t *ptr); void utils_rwlock_destroy_not_free(utils_rwlock_t *rwlock); -int utils_read_lock(utils_rwlock_t *rwlock); -int utils_write_lock(utils_rwlock_t *rwlock); -int utils_read_unlock(utils_rwlock_t *rwlock); -int utils_write_unlock(utils_rwlock_t *rwlock); +void utils_read_lock(utils_rwlock_t *rwlock); +void utils_write_lock(utils_rwlock_t *rwlock); +void utils_read_unlock(utils_rwlock_t *rwlock); +void utils_write_unlock(utils_rwlock_t *rwlock); #if defined(_WIN32) #define UTIL_ONCE_FLAG INIT_ONCE diff --git a/src/utils/utils_level_zero.cpp b/src/utils/utils_level_zero.cpp index 6daab3e691..5f2d16666a 100644 --- a/src/utils/utils_level_zero.cpp +++ b/src/utils/utils_level_zero.cpp @@ -13,8 +13,6 @@ #include "utils_concurrency.h" #include "utils_load_library.h" -#include "ze_api.h" - struct libze_ops { ze_result_t (*zeInit)(ze_init_flags_t flags); ze_result_t (*zeDriverGet)(uint32_t *pCount, ze_driver_handle_t *phDrivers); @@ -338,7 +336,6 @@ static void utils_ze_init_level_zero_once(void) { int utils_ze_init_level_zero(void) { utils_init_once(&level_zero_init_flag, utils_ze_init_level_zero_once); - return InitResult; } diff --git a/src/utils/utils_log.h b/src/utils/utils_log.h index c0e0a95723..eced32395f 100644 --- a/src/utils/utils_log.h +++ b/src/utils/utils_log.h @@ -24,17 +24,25 @@ typedef enum { LOG_FATAL } utils_log_level_t; -#define LOG_DEBUG(...) utils_log(LOG_DEBUG, __func__, __VA_ARGS__); -#define LOG_INFO(...) utils_log(LOG_INFO, __func__, __VA_ARGS__); -#define LOG_WARN(...) utils_log(LOG_WARNING, __func__, __VA_ARGS__); -#define LOG_ERR(...) utils_log(LOG_ERROR, __func__, __VA_ARGS__); -#define LOG_FATAL(...) utils_log(LOG_FATAL, __func__, __VA_ARGS__); - -#define LOG_PDEBUG(...) utils_plog(LOG_DEBUG, __func__, __VA_ARGS__); -#define LOG_PINFO(...) utils_plog(LOG_INFO, __func__, __VA_ARGS__); -#define LOG_PWARN(...) utils_plog(LOG_WARNING, __func__, __VA_ARGS__); -#define LOG_PERR(...) utils_plog(LOG_ERROR, __func__, __VA_ARGS__); -#define LOG_PFATAL(...) utils_plog(LOG_FATAL, __func__, __VA_ARGS__); +#ifdef UMF_DEVELOPER_MODE +#define UMF_STRINGIFY(x) #x +#define UMF_TOSTRING(x) UMF_STRINGIFY(x) +#define UMF_FUNC_DESC() __FILE__ ":" UMF_TOSTRING(__LINE__) +#else +#define UMF_FUNC_DESC() __func__ +#endif + +#define LOG_DEBUG(...) utils_log(LOG_DEBUG, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_INFO(...) utils_log(LOG_INFO, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_WARN(...) utils_log(LOG_WARNING, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_ERR(...) utils_log(LOG_ERROR, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_FATAL(...) utils_log(LOG_FATAL, UMF_FUNC_DESC(), __VA_ARGS__); + +#define LOG_PDEBUG(...) utils_plog(LOG_DEBUG, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_PINFO(...) utils_plog(LOG_INFO, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_PWARN(...) utils_plog(LOG_WARNING, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_PERR(...) utils_plog(LOG_ERROR, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_PFATAL(...) utils_plog(LOG_FATAL, UMF_FUNC_DESC(), __VA_ARGS__); void utils_log_init(void); #ifdef _WIN32 diff --git a/src/utils/utils_posix_concurrency.c b/src/utils/utils_posix_concurrency.c index c6f273bed2..6cbcd908ec 100644 --- a/src/utils/utils_posix_concurrency.c +++ b/src/utils/utils_posix_concurrency.c @@ -54,24 +54,36 @@ utils_rwlock_t *utils_rwlock_init(utils_rwlock_t *ptr) { void utils_rwlock_destroy_not_free(utils_rwlock_t *ptr) { pthread_rwlock_t *rwlock = (pthread_rwlock_t *)ptr; - int ret = pthread_rwlock_destroy(rwlock); - if (ret) { - LOG_ERR("pthread_rwlock_destroy failed"); + if (pthread_rwlock_destroy(rwlock) != 0) { + LOG_FATAL("pthread_rwlock_destroy failed"); + abort(); } } -int utils_read_lock(utils_rwlock_t *rwlock) { - return pthread_rwlock_rdlock((pthread_rwlock_t *)rwlock); +void utils_read_lock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_rdlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_rdlock failed"); + abort(); + } } -int utils_write_lock(utils_rwlock_t *rwlock) { - return pthread_rwlock_wrlock((pthread_rwlock_t *)rwlock); +void utils_write_lock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_wrlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_wrlock failed"); + abort(); + } } -int utils_read_unlock(utils_rwlock_t *rwlock) { - return pthread_rwlock_unlock((pthread_rwlock_t *)rwlock); +void utils_read_unlock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_unlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_unlock failed"); + abort(); + } } -int utils_write_unlock(utils_rwlock_t *rwlock) { - return pthread_rwlock_unlock((pthread_rwlock_t *)rwlock); +void utils_write_unlock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_unlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_unlock failed"); + abort(); + } } diff --git a/src/utils/utils_windows_concurrency.c b/src/utils/utils_windows_concurrency.c index faa302be36..4e2f9c3046 100644 --- a/src/utils/utils_windows_concurrency.c +++ b/src/utils/utils_windows_concurrency.c @@ -46,24 +46,20 @@ void utils_rwlock_destroy_not_free(utils_rwlock_t *rwlock) { (void)rwlock; } -int utils_read_lock(utils_rwlock_t *rwlock) { - AcquireSRWLockShared(&rwlock->lock); - return 0; // never fails +void utils_read_lock(utils_rwlock_t *rwlock) { + AcquireSRWLockShared(&rwlock->lock); // never fails } -int utils_write_lock(utils_rwlock_t *rwlock) { - AcquireSRWLockExclusive(&rwlock->lock); - return 0; // never fails +void utils_write_lock(utils_rwlock_t *rwlock) { + AcquireSRWLockExclusive(&rwlock->lock); // never fails } -int utils_read_unlock(utils_rwlock_t *rwlock) { - ReleaseSRWLockShared(&rwlock->lock); - return 0; // never fails +void utils_read_unlock(utils_rwlock_t *rwlock) { + ReleaseSRWLockShared(&rwlock->lock); // never fails } -int utils_write_unlock(utils_rwlock_t *rwlock) { - ReleaseSRWLockExclusive(&rwlock->lock); - return 0; // never fails +void utils_write_unlock(utils_rwlock_t *rwlock) { + ReleaseSRWLockExclusive(&rwlock->lock); // never fails } static BOOL CALLBACK initOnceCb(PINIT_ONCE InitOnce, PVOID Parameter, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8862b7b88d..a44569bd3a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,8 +35,9 @@ endif() enable_testing() set(UMF_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(UMF_UTILS_DIR ${UMF_CMAKE_SOURCE_DIR}/src/utils) -set(UMF_BA_DIR ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc) +set(UMF_SRC_DIR ${UMF_CMAKE_SOURCE_DIR}/src) +set(UMF_UTILS_DIR ${UMF_SRC_DIR}/utils) +set(UMF_BA_DIR ${UMF_SRC_DIR}/base_alloc) function(build_umf_test) # Parameters: @@ -430,6 +431,19 @@ else() LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) endif() +if (UMF_LEVEL_ZERO_ENABLED) + add_umf_test( + NAME provider_level_zero_residency + SRCS providers/provider_level_zero_residency.cpp + ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock) + add_umf_test( + NAME pool_residency + SRCS pools/pool_residency.cpp + ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock) +endif () + if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) # we have two test binaries here that use the same sources, but differ in # the way they are linked to the Level Zero (statically or at runtime using @@ -466,8 +480,8 @@ endif() if(NOT UMF_BUILD_LEVEL_ZERO_PROVIDER) add_umf_test( NAME provider_level_zero_not_impl - SRCS providers/provider_level_zero_not_impl.cpp - LIBS ${UMF_UTILS_FOR_TEST}) + SRCS providers/provider_level_zero_not_impl.cpp + LIBS ${UMF_UTILS_FOR_TEST}) endif() if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 6cffe5cfe8..5fe3e5b3c3 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -20,3 +20,24 @@ add_umf_library( target_include_directories(umf_test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) + +add_library( + umf_test_mocks + STATIC + level_zero_mocks.cpp) + +target_link_libraries(umf_test_mocks GTest::gmock) + +target_include_directories( + umf_test_mocks + PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include + ${UMF_CMAKE_SOURCE_DIR}/src + ${UMF_CMAKE_SOURCE_DIR}/src/utils) + +add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) +target_include_directories( + umf_ze_loopback + PRIVATE ${ZE_LOADER_INCLUDE_DIRS} + ${UMF_UTILS_DIR} + ${UMF_SRC_DIR} + ${UMF_CMAKE_SOURCE_DIR}/include) \ No newline at end of file diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp new file mode 100644 index 0000000000..1eb0ac3021 --- /dev/null +++ b/test/common/level_zero_mocks.cpp @@ -0,0 +1,87 @@ +#include "level_zero_mocks.h" +#include "umf/providers/provider_level_zero.h" +#include + +using namespace ::testing; + +umf_memory_provider_handle_t +LevelZeroMock::initializeMemoryProviderWithResidentDevices( + ze_device_handle_t device, std::vector residentDevices, + ze_context_handle_t context, ze_device_properties_t device_properties, + ze_memory_allocation_properties_t memory_allocation_properties) { + umf_level_zero_memory_provider_params_handle_t params = nullptr; + EXPECT_EQ(umfLevelZeroMemoryProviderParamsCreate(¶ms), + UMF_RESULT_SUCCESS); + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetContext(params, context), + UMF_RESULT_SUCCESS); + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetDevice(params, device), + UMF_RESULT_SUCCESS); + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetMemoryType( + params, UMF_MEMORY_TYPE_DEVICE), + UMF_RESULT_SUCCESS); + + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetResidentDevices( + params, residentDevices.data(), residentDevices.size()), + UMF_RESULT_SUCCESS); + + // query min page size operation upon provider initialization + EXPECT_CALL(*this, zeDeviceGetProperties(device, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(device_properties), + Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(*this, zeMemAllocDevice(CONTEXT, _, _, _, device, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + for (auto dev : residentDevices) { + EXPECT_CALL(*this, zeContextMakeMemoryResident(context, dev, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + } + EXPECT_CALL(*this, zeMemGetAllocProperties(context, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(memory_allocation_properties), + Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(*this, zeMemFree(CONTEXT, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + umf_memory_provider_handle_t provider = nullptr; + EXPECT_EQ(umfMemoryProviderCreate(umfLevelZeroMemoryProviderOps(), params, + &provider), + UMF_RESULT_SUCCESS); + EXPECT_NE(provider, nullptr); + + umfLevelZeroMemoryProviderParamsDestroy(params); + return provider; +} + +ze_device_properties_t TestCreateDeviceProperties() { + return ze_device_properties_t{ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES, + nullptr, + ZE_DEVICE_TYPE_GPU, + 0, + 0, + 0, + 0, + 0, + 1024, + 100, + 20, + 16, + 256, + 8, + 2, + 4, + 1, + 8, + 8, + {123}, + "TESTGPU"}; +}; + +ze_memory_allocation_properties_t +TestCreateMemoryAllocationProperties(uint32_t modifier) { + return ze_memory_allocation_properties_t{ + ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES, nullptr, + ZE_MEMORY_TYPE_DEVICE, modifier, 2048}; +} + +void MockedLevelZeroTestEnvironment::SetUp() { + setenv("UMF_ZE_LOADER_LIB_NAME", "libumf_ze_loopback.so", 1); +} +void MockedLevelZeroTestEnvironment::TearDown() {} diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h new file mode 100644 index 0000000000..af15e1013f --- /dev/null +++ b/test/common/level_zero_mocks.h @@ -0,0 +1,77 @@ +#ifndef UMF_PROVIDER_LEVEL_ZERO_MOCKS_H +#define UMF_PROVIDER_LEVEL_ZERO_MOCKS_H + +#include "utils/utils_log.h" +#include "ze_loopback.h" +#include +#include + +// TEST CREATE methods for objects + +template constexpr T TestCreatePointer(uint32_t modifier = 0) { + return reinterpret_cast(0x1000 + modifier); +} + +ze_device_properties_t TestCreateDeviceProperties(); + +ze_memory_allocation_properties_t +TestCreateMemoryAllocationProperties(uint32_t modifier = 0); + +// already created common instances for tests writing convenience + +static const auto DEVICE_0 = TestCreatePointer(0); +static const auto DEVICE_1 = TestCreatePointer(1); +static const auto DEVICE_2 = TestCreatePointer(2); +static const auto DEVICE_3 = TestCreatePointer(3); +static const auto DEVICE_4 = TestCreatePointer(4); +static const auto DEVICE_5 = TestCreatePointer(5); + +static const auto CONTEXT = TestCreatePointer(); +static const auto DEVICE_PROPS = TestCreateDeviceProperties(); +static const auto MEM_PROPS = TestCreateMemoryAllocationProperties(); + +static void *POINTER_0 = TestCreatePointer(0x90); +static void *POINTER_1 = TestCreatePointer(0x91); +static void *POINTER_2 = TestCreatePointer(0x92); +static void *POINTER_3 = TestCreatePointer(0x93); +static void *POINTER_4 = TestCreatePointer(0x94); + +class LevelZeroMock : public LevelZero { + public: + MOCK_METHOD3(zeContextCreate, + ze_result_t(ze_driver_handle_t, const ze_context_desc_t *, + ze_context_handle_t *)); + MOCK_METHOD2(zeDeviceGetProperties, + ze_result_t(ze_device_handle_t, ze_device_properties_t *)); + MOCK_METHOD6(zeMemAllocDevice, + ze_result_t(ze_context_handle_t, + const ze_device_mem_alloc_desc_t *, size_t, size_t, + ze_device_handle_t, void **)); + MOCK_METHOD4(zeMemGetAllocProperties, + ze_result_t(ze_context_handle_t, const void *, + ze_memory_allocation_properties_t *, + ze_device_handle_t *)); + MOCK_METHOD4(zeContextMakeMemoryResident, + ze_result_t(ze_context_handle_t, ze_device_handle_t, void *, + size_t)); + MOCK_METHOD2(zeMemFree, + ze_result_t(ze_context_handle_t hContext, void *ptr)); + + // helper setting all expects related to successful l0 provider creation & initialization and calling its creation & initialization + umf_memory_provider_handle_t initializeMemoryProviderWithResidentDevices( + ze_device_handle_t device, + std::vector residentDevices, + ze_context_handle_t context = CONTEXT, + ze_device_properties_t device_properties = DEVICE_PROPS, + ze_memory_allocation_properties_t memory_allocation_properties = + MEM_PROPS); +}; + +// important, makes UMF load ze_loopback instead of regular l0 +class MockedLevelZeroTestEnvironment : public ::testing::Environment { + public: + void SetUp() override; + void TearDown() override; +}; + +#endif //UMF_PROVIDER_LEVEL_ZERO_MOCKS_H diff --git a/test/common/provider_null.c b/test/common/provider_null.c index ae6f8e595e..d91518f125 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -162,11 +162,10 @@ static umf_result_t nullGetAllocationPropertiesSize( return UMF_RESULT_SUCCESS; } -static umf_result_t nullResidentDeviceChange(void *provider, - uint32_t device_index, +static umf_result_t nullResidentDeviceChange(void *provider, void *device, bool is_adding) { (void)provider; - (void)device_index; + (void)device; (void)is_adding; return UMF_RESULT_SUCCESS; } diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp new file mode 100644 index 0000000000..2fba286cde --- /dev/null +++ b/test/common/ze_loopback.cpp @@ -0,0 +1,285 @@ +#include "ze_loopback.h" +#include "utils_log.h" + +#include + +LevelZero *level_zero_mock = nullptr; + +// +// libze_ops from src/utils/utils_level_zero.cpp +// + +void check_mock_present() { + if (level_zero_mock == nullptr) { + LOG_FATAL("level_zero_mock was not set"); + abort(); + } +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeInit(ze_init_flags_t flags) { + (void)flags; + return ZE_RESULT_SUCCESS; +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeDriverGet(uint32_t *pCount, + ze_driver_handle_t *phDrivers) { + (void)phDrivers; + (void)pCount; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeDeviceGet(ze_driver_handle_t hDriver, + uint32_t *pCount, + ze_device_handle_t *phDevices) { + (void)hDriver; + (void)pCount; + (void)phDevices; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeDeviceGetProperties( + ze_device_handle_t hDevice, ze_device_properties_t *pDeviceProperties) { + check_mock_present(); + return level_zero_mock->zeDeviceGetProperties(hDevice, pDeviceProperties); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeContextCreate(ze_driver_handle_t hDriver, const ze_context_desc_t *desc, + ze_context_handle_t *phContext) { + (void)hDriver; + (void)desc; + (void)phContext; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeContextDestroy(ze_context_handle_t hContext) { + (void)hContext; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandQueueCreate(ze_context_handle_t hContext, ze_device_handle_t hDevice, + const ze_command_queue_desc_t *desc, + ze_command_queue_handle_t *phCommandQueue) { + (void)hContext; + (void)hDevice; + (void)desc; + (void)phCommandQueue; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandQueueDestroy(ze_command_queue_handle_t hCommandQueue) { + (void)hCommandQueue; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueExecuteCommandLists( + ze_command_queue_handle_t hCommandQueue, uint32_t numCommandLists, + ze_command_list_handle_t *phCommandLists, ze_fence_handle_t hFence) { + (void)hCommandQueue; + (void)numCommandLists; + (void)phCommandLists; + (void)hFence; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueSynchronize( + ze_command_queue_handle_t hCommandQueue, uint64_t timeout) { + (void)hCommandQueue; + (void)timeout; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandListCreate(ze_context_handle_t hContext, ze_device_handle_t hDevice, + const ze_command_list_desc_t *desc, + ze_command_list_handle_t *phCommandList) { + (void)hContext; + (void)hDevice; + (void)desc; + (void)phCommandList; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandListDestroy(ze_command_list_handle_t hCommandList) { + (void)hCommandList; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandListClose(ze_command_list_handle_t hCommandList) { + (void)hCommandList; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryCopy( + ze_command_list_handle_t hCommandList, void *dstptr, const void *srcptr, + size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents) { + (void)hCommandList; + (void)dstptr; + (void)srcptr; + (void)size; + (void)hSignalEvent; + (void)numWaitEvents; + (void)phWaitEvents; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryFill( + ze_command_list_handle_t hCommandList, void *ptr, const void *pattern, + size_t pattern_size, size_t size, ze_event_handle_t hSignalEvent, + uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents) { + (void)hCommandList; + (void)ptr; + (void)pattern; + (void)pattern_size; + (void)size; + (void)hSignalEvent; + (void)numWaitEvents; + (void)phWaitEvents; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeMemGetAllocProperties(ze_context_handle_t hContext, const void *ptr, + ze_memory_allocation_properties_t *pMemAllocProperties, + ze_device_handle_t *phDevice) { + check_mock_present(); + return level_zero_mock->zeMemGetAllocProperties( + hContext, ptr, pMemAllocProperties, phDevice); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeMemAllocDevice( + ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, + size_t size, size_t alignment, ze_device_handle_t hDevice, void **pptr) { + check_mock_present(); + return level_zero_mock->zeMemAllocDevice(hContext, device_desc, size, + alignment, hDevice, pptr); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeMemFree(ze_context_handle_t hContext, + void *ptr) { + check_mock_present(); + return level_zero_mock->zeMemFree(hContext, ptr); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeDeviceGetMemoryProperties(ze_device_handle_t hDevice, uint32_t *pCount, + ze_device_memory_properties_t *pMemProperties) { + (void)hDevice; + (void)pCount; + (void)pMemProperties; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +// +// ze_ops_t operations from src/provider/provider_level_zero.c +// + +ze_result_t ZE_APICALL zeMemAllocHost(ze_context_handle_t hContext, + const ze_host_mem_alloc_desc_t *host_desc, + size_t size, size_t alignment, + void **pptr) { + (void)hContext; + (void)host_desc; + (void)size; + (void)alignment; + (void)pptr; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ze_result_t ZE_APICALL zeMemAllocShared( + ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, + const ze_host_mem_alloc_desc_t *host_desc, size_t size, size_t alignment, + ze_device_handle_t hDevice, void **pptr) { + (void)hContext; + (void)device_desc; + (void)host_desc; + (void)size; + (void)alignment; + (void)hDevice; + (void)pptr; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ze_result_t ZE_APICALL zeMemGetIpcHandle(ze_context_handle_t hContext, + const void *ptr, + ze_ipc_mem_handle_t *pIpcHandle) { + (void)hContext; + (void)ptr; + (void)pIpcHandle; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ze_result_t ZE_APICALL zeMemPutIpcHandle(ze_context_handle_t hContext, + ze_ipc_mem_handle_t handle) { + (void)hContext; + (void)handle; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ze_result_t ZE_APICALL zeMemOpenIpcHandle(ze_context_handle_t hContext, + ze_device_handle_t hDevice, + ze_ipc_mem_handle_t handle, + ze_ipc_memory_flags_t flags, + void **pptr) { + (void)hContext; + (void)hDevice; + (void)handle; + (void)flags; + (void)pptr; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ze_result_t ZE_APICALL zeMemCloseIpcHandle(ze_context_handle_t hContext, + const void *ptr) { + (void)hContext; + (void)ptr; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +ze_result_t ZE_APICALL zeContextMakeMemoryResident(ze_context_handle_t hContext, + ze_device_handle_t hDevice, + void *ptr, size_t size) { + check_mock_present(); + return level_zero_mock->zeContextMakeMemoryResident(hContext, hDevice, ptr, + size); +} + +ze_result_t ZE_APICALL +zeMemFreeExt(ze_context_handle_t hContext, + const ze_memory_free_ext_desc_t *pMemFreeDesc, void *ptr) { + (void)hContext; + (void)pMemFreeDesc; + (void)ptr; + LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); + abort(); +} + +// defined in both libze_ops from src/utils/utils_level_zero.cpp and +// ze_ops_t operations from src/provider/provider_level_zero.c +// zeDeviceGetProperties, zeMemFree, zeMemGetAllocProperties, zeMemFree, zeMemAllocDevice, find them in the former ones \ No newline at end of file diff --git a/test/common/ze_loopback.h b/test/common/ze_loopback.h new file mode 100644 index 0000000000..d3af3cc649 --- /dev/null +++ b/test/common/ze_loopback.h @@ -0,0 +1,35 @@ +// Copyright (C) 2024-2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef UMF_ZE_LOOPBACK_H +#define UMF_ZE_LOOPBACK_H + +#include + +class LevelZero { + public: + virtual ~LevelZero() = default; + + virtual ze_result_t zeContextCreate(ze_driver_handle_t, + const ze_context_desc_t *, + ze_context_handle_t *) = 0; + virtual ze_result_t zeDeviceGetProperties(ze_device_handle_t, + ze_device_properties_t *) = 0; + virtual ze_result_t zeMemAllocDevice(ze_context_handle_t, + const ze_device_mem_alloc_desc_t *, + size_t, size_t, ze_device_handle_t, + void **) = 0; + virtual ze_result_t + zeMemGetAllocProperties(ze_context_handle_t, const void *, + ze_memory_allocation_properties_t *, + ze_device_handle_t *) = 0; + virtual ze_result_t zeContextMakeMemoryResident(ze_context_handle_t, + ze_device_handle_t, void *, + size_t) = 0; + virtual ze_result_t zeMemFree(ze_context_handle_t hContext, void *ptr) = 0; +}; + +extern LevelZero *level_zero_mock; + +#endif //UMF_ZE_LOOPBACK_H diff --git a/test/pools/pool_residency.cpp b/test/pools/pool_residency.cpp new file mode 100644 index 0000000000..9a2b75e10a --- /dev/null +++ b/test/pools/pool_residency.cpp @@ -0,0 +1,133 @@ +// Copyright (C) 2024-2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "../common/level_zero_mocks.h" +#include "pool.hpp" +#include "umf/pools/pool_disjoint.h" +#include "gtest/gtest.h" + +using namespace testing; + +class PoolResidencyTestFixture : public Test { + protected: + StrictMock l0mock; + umf_memory_pool_handle_t pool = nullptr; + const ze_device_handle_t OUR_DEVICE; + + PoolResidencyTestFixture() + : OUR_DEVICE(TestCreatePointer(777)) {} + + void initializeMemoryPool(umf_memory_provider_handle_t provider) { + + auto *params = static_cast( + umf_test::defaultDisjointPoolConfig()); + + EXPECT_EQ(umfPoolCreate(umfDisjointPoolOps(), provider, params, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool), + UMF_RESULT_SUCCESS); + + umf_test::defaultDisjointPoolConfigDestroy(params); + } + + void SetUp() override { level_zero_mock = &l0mock; } + void TearDown() override { + if (pool != nullptr) { + EXPECT_CALL(l0mock, zeMemFree(CONTEXT, _)) + .WillRepeatedly(Return(ZE_RESULT_SUCCESS)); + umfPoolDestroy(pool); + } + Mock::VerifyAndClearExpectations(&l0mock); + } +}; + +TEST_F(PoolResidencyTestFixture, + initialResidentDevicesShouldBeUsedDuringAllocation) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_0, DEVICE_1})); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_0, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_1, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + umfPoolFree(pool, ptr); +} + +TEST_F(PoolResidencyTestFixture, + addedResidentDevicesShouldBeUsedDuringAllocation) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_0})); + + umf_memory_provider_handle_t provider; + EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); + umfMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_0, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_4, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + umfPoolFree(pool, ptr); +} + +TEST_F(PoolResidencyTestFixture, + existingAllocationsShouldBeMadeResidentOnAddedDevice) { + initializeMemoryPool( + l0mock.initializeMemoryProviderWithResidentDevices(OUR_DEVICE, {})); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_4, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + umf_memory_provider_handle_t provider; + EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); + umfMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); + + umfPoolFree(pool, ptr); +} + +TEST_F(PoolResidencyTestFixture, + allocationShouldNotBeMadeResidentOnRemovedDevice) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_2})); + + umf_memory_provider_handle_t provider; + EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); + umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, false); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_2, _, _)) + .Times(0); // not called + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + umfPoolFree(pool, ptr); +} + +int main(int argc, char **argv) { + InitGoogleTest(&argc, argv); + AddGlobalTestEnvironment(new MockedLevelZeroTestEnvironment); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index b8ca69d16b..9d72e4f367 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -204,9 +204,9 @@ TEST_F(LevelZeroProviderInit, FailNonNullDevice) { umfLevelZeroMemoryProviderParamsDestroy(hParams); } -static void invalidResidentDevicesHandlesTestHelper( - ze_device_handle_t *hDevices, uint32_t deviceCount, - uint32_t *residentDevicesIndices, uint32_t residentDevicesCount) { +static void +invalidResidentDevicesHandlesTestHelper(ze_device_handle_t *hDevices, + uint32_t deviceCount) { const umf_memory_provider_ops_t *ops = umfLevelZeroMemoryProviderOps(); ASSERT_NE(ops, nullptr); @@ -216,35 +216,23 @@ static void invalidResidentDevicesHandlesTestHelper( ASSERT_EQ(create_result, UMF_RESULT_SUCCESS); const umf_result_t set_resident_result = - umfLevelZeroMemoryProviderParamsSetResidentDevices( - hParams, hDevices, deviceCount, residentDevicesIndices, - residentDevicesCount); + umfLevelZeroMemoryProviderParamsSetResidentDevices(hParams, hDevices, + deviceCount); ASSERT_EQ(set_resident_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); umfLevelZeroMemoryProviderParamsDestroy(hParams); } TEST_F(test, FailMismatchedResidentHandlesCount) { - invalidResidentDevicesHandlesTestHelper(nullptr, 99, nullptr, 0); + invalidResidentDevicesHandlesTestHelper(nullptr, 99); } -TEST_F(test, FailMismatchedResidentDeviceIndicesCount) { - std::vector allDevices(23); - invalidResidentDevicesHandlesTestHelper(allDevices.data(), 23, nullptr, 1); -} - -TEST_F(test, FailRedundantResidentDeviceIndices) { - std::vector allDevices(23); - std::vector residentDevicesIndices{3, 5, 1, 5}; // double 5 - invalidResidentDevicesHandlesTestHelper(allDevices.data(), 23, - residentDevicesIndices.data(), 4); -} - -TEST_F(test, FailTooLargeResidentDeviceIndex) { - std::vector allDevices(23); - std::vector residentDevicesIndices{123}; - invalidResidentDevicesHandlesTestHelper(allDevices.data(), 23, - residentDevicesIndices.data(), 1); +TEST_F(test, FailRedundantResidentDeviceHandles) { + std::vector hDevices{ + reinterpret_cast(0x100), + reinterpret_cast(0x101), + reinterpret_cast(0x100)}; + invalidResidentDevicesHandlesTestHelper(hDevices.data(), 3); } class LevelZeroMemoryAccessor : public MemoryAccessor { diff --git a/test/providers/provider_level_zero_not_impl.cpp b/test/providers/provider_level_zero_not_impl.cpp index b255ec5780..dfbd4a2b21 100644 --- a/test/providers/provider_level_zero_not_impl.cpp +++ b/test/providers/provider_level_zero_not_impl.cpp @@ -27,8 +27,8 @@ TEST_F(test, level_zero_provider_not_implemented) { ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); ze_device_handle_t hDevices[1]; - result = umfLevelZeroMemoryProviderParamsSetResidentDevices( - hParams, hDevices, 1, nullptr, 0); + result = umfLevelZeroMemoryProviderParamsSetResidentDevices(hParams, + hDevices, 1); ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); result = umfLevelZeroMemoryProviderParamsSetFreePolicy( diff --git a/test/providers/provider_level_zero_residency.cpp b/test/providers/provider_level_zero_residency.cpp new file mode 100644 index 0000000000..bd511e3b9a --- /dev/null +++ b/test/providers/provider_level_zero_residency.cpp @@ -0,0 +1,79 @@ +// Copyright (C) 2024-2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "../common/level_zero_mocks.h" +#include "utils_level_zero.h" +#include "utils_log.h" + +#include "gtest/gtest.h" + +using namespace testing; + +class LevelZeroResidencyTestFixture : public Test { + protected: + StrictMock l0mock; + umf_memory_provider_handle_t provider = nullptr; + const ze_device_handle_t OUR_DEVICE; + + LevelZeroResidencyTestFixture() + : OUR_DEVICE(TestCreatePointer(777)) {} + + void SetUp() override { level_zero_mock = &l0mock; } + void TearDown() override { + Mock::VerifyAndClearExpectations(&l0mock); + umfMemoryProviderDestroy(provider); + } +}; + +TEST_F(LevelZeroResidencyTestFixture, addNonexistingDeviceShouldSucceed) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, true), + UMF_RESULT_SUCCESS); +} + +TEST_F(LevelZeroResidencyTestFixture, addExistingDeviceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_5, true), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(LevelZeroResidencyTestFixture, removeNonexistingDeviceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_0, false), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(LevelZeroResidencyTestFixture, removeExistingDeviceShouldSucceed) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_1, false), + UMF_RESULT_SUCCESS); +} + +TEST_F(LevelZeroResidencyTestFixture, addDeviceTwiceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, true), + UMF_RESULT_SUCCESS); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, true), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(LevelZeroResidencyTestFixture, removeDeviceTwiceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_3, false), + UMF_RESULT_SUCCESS); + ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_3, false), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +int main(int argc, char **argv) { + InitGoogleTest(&argc, argv); + AddGlobalTestEnvironment(new MockedLevelZeroTestEnvironment); + return RUN_ALL_TESTS(); +} \ No newline at end of file From 01d74bad3d2af34ce231f8982388e460944454f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 19 Sep 2025 14:07:53 +0200 Subject: [PATCH 11/49] moved umfMemoryProviderResidentDeviceChange down --- src/libumf.map | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libumf.map b/src/libumf.map index 78ecc1ab01..0c7436ee5b 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -137,7 +137,6 @@ UMF_1.0 { umfJemallocPoolParamsCreate; umfJemallocPoolParamsDestroy; umfJemallocPoolParamsSetNumArenas; - umfMemoryProviderResidentDeviceChange; umfPoolGetName; local: *; @@ -152,6 +151,7 @@ UMF_1.1 { umfGetMemoryProperty; umfGetMemoryPropertySize; umfJemallocPoolParamsSetName; + umfMemoryProviderResidentDeviceChange; umfLevelZeroMemoryProviderParamsSetName; umfOsMemoryProviderParamsSetName; umfPoolTrimMemory; From 0bc9a29eb329d9d608e6301e6500676c888beea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 19 Sep 2025 14:13:23 +0200 Subject: [PATCH 12/49] added empty line for Rafal :) --- src/memory_provider.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory_provider.c b/src/memory_provider.c index 6771bffaa4..9d595aa636 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -630,4 +630,4 @@ umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, checkErrorAndSetLastProvider(res, hProvider); return res; -} \ No newline at end of file +} From d13a3efa10e636027ab43badf963dfa76607efdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 19 Sep 2025 14:27:08 +0200 Subject: [PATCH 13/49] cmake format fixed --- test/CMakeLists.txt | 19 +++++++++++-------- test/common/CMakeLists.txt | 17 +++++------------ 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a44569bd3a..e08397eb29 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -431,18 +431,21 @@ else() LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) endif() -if (UMF_LEVEL_ZERO_ENABLED) +if(UMF_LEVEL_ZERO_ENABLED) add_umf_test( NAME provider_level_zero_residency SRCS providers/provider_level_zero_residency.cpp - ${UMF_UTILS_DIR}/utils_level_zero.cpp + ${UMF_UTILS_DIR}/utils_level_zero.cpp LIBS ${UMF_UTILS_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock) add_umf_test( NAME pool_residency - SRCS pools/pool_residency.cpp - ${UMF_UTILS_DIR}/utils_level_zero.cpp - LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock) -endif () + SRCS pools/pool_residency.cpp ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} + ${UMF_BA_FOR_TEST} + umf_ze_loopback + umf_test_mocks + GTest::gmock) +endif() if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) # we have two test binaries here that use the same sources, but differ in @@ -480,8 +483,8 @@ endif() if(NOT UMF_BUILD_LEVEL_ZERO_PROVIDER) add_umf_test( NAME provider_level_zero_not_impl - SRCS providers/provider_level_zero_not_impl.cpp - LIBS ${UMF_UTILS_FOR_TEST}) + SRCS providers/provider_level_zero_not_impl.cpp + LIBS ${UMF_UTILS_FOR_TEST}) endif() if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 5fe3e5b3c3..6412b06c07 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -21,23 +21,16 @@ add_umf_library( target_include_directories(umf_test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) -add_library( - umf_test_mocks - STATIC - level_zero_mocks.cpp) +add_library(umf_test_mocks STATIC level_zero_mocks.cpp) target_link_libraries(umf_test_mocks GTest::gmock) target_include_directories( umf_test_mocks - PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include - ${UMF_CMAKE_SOURCE_DIR}/src - ${UMF_CMAKE_SOURCE_DIR}/src/utils) + PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src + ${UMF_CMAKE_SOURCE_DIR}/src/utils) add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) target_include_directories( - umf_ze_loopback - PRIVATE ${ZE_LOADER_INCLUDE_DIRS} - ${UMF_UTILS_DIR} - ${UMF_SRC_DIR} - ${UMF_CMAKE_SOURCE_DIR}/include) \ No newline at end of file + umf_ze_loopback PRIVATE ${ZE_LOADER_INCLUDE_DIRS} ${UMF_UTILS_DIR} + ${UMF_SRC_DIR} ${UMF_CMAKE_SOURCE_DIR}/include) From 272bd66a0c2df68f7b7db5ec325200c85b9b6e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 22 Sep 2025 09:43:53 +0200 Subject: [PATCH 14/49] Lukasz and Rafal comments applied --- src/libumf.def | 2 +- src/provider/provider_level_zero.c | 100 ++++++++++++++++------------- test/common/CMakeLists.txt | 24 +++---- test/common/level_zero_mocks.cpp | 9 +++ test/common/level_zero_mocks.h | 9 +++ test/common/ze_loopback.cpp | 9 +++ test/pools/pool_residency.cpp | 19 ++++++ 7 files changed, 114 insertions(+), 58 deletions(-) diff --git a/src/libumf.def b/src/libumf.def index 5955088b8e..114682d3db 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -19,7 +19,6 @@ EXPORTS umfCUDAMemoryProviderParamsSetContext umfCUDAMemoryProviderParamsSetDevice umfCUDAMemoryProviderParamsSetMemoryType - umfLevelZeroMemoryProviderParamsSetResidentDevices umfDevDaxMemoryProviderOps umfDevDaxMemoryProviderParamsCreate umfDevDaxMemoryProviderParamsDestroy @@ -42,6 +41,7 @@ EXPORTS umfLevelZeroMemoryProviderParamsSetContext umfLevelZeroMemoryProviderParamsSetDevice umfLevelZeroMemoryProviderParamsSetMemoryType + umfLevelZeroMemoryProviderParamsSetResidentDevices umfMemoryProviderAlloc umfMemoryProviderAllocationMerge umfMemoryProviderAllocationSplit diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 4c7ace96e5..2ebf53d358 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -422,6 +422,43 @@ static ze_relaxed_allocation_limits_exp_desc_t relaxed_device_allocation_desc = .pNext = NULL, .flags = ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE}; +static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, + size_t bytes, + int update_stats) { + if (ptr == NULL) { + return UMF_RESULT_SUCCESS; + } + + ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; + umf_result_t ret; + if (ze_provider->freePolicyFlags == 0) { + ret = ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); + } else { + ze_memory_free_ext_desc_t desc = { + .stype = ZE_STRUCTURE_TYPE_MEMORY_FREE_EXT_DESC, + .pNext = NULL, + .freePolicy = ze_provider->freePolicyFlags}; + + ret = ze2umf_result( + g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); + } + + if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + + if (update_stats) { + provider_ctl_stats_free(ze_provider, bytes); + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t ze_memory_provider_free(void *provider, void *ptr, + size_t bytes) { + return ze_memory_provider_free_helper(provider, ptr, bytes, 1); +} + static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, size_t alignment, int update_stats, @@ -488,10 +525,17 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, *resultPtr, size); if (ze_result != ZE_RESULT_SUCCESS) { utils_read_unlock(&ze_provider->resident_device_rwlock); - LOG_DEBUG("making resident allocation %p of size:%lu on device %p " - "failed with %d", - *resultPtr, size, ze_provider->resident_device_handles[i], - ze_result); + LOG_ERR("making resident allocation %p of size:%lu on device %p " + "failed with 0x%x", + *resultPtr, size, ze_provider->resident_device_handles[i], + ze_result); + umf_result_t free_result = + ze_memory_provider_free(ze_provider, *resultPtr, size); + if (free_result != UMF_RESULT_SUCCESS) { + LOG_ERR("failed to free memory with: 0x%x after failed making " + "resident, free fail ignored", + free_result); + } return ze2umf_result(ze_result); } LOG_DEBUG("allocation %p of size:%lu made resident on device %p", @@ -512,43 +556,6 @@ static umf_result_t ze_memory_provider_alloc(void *provider, size_t size, resultPtr); } -static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, - size_t bytes, - int update_stats) { - if (ptr == NULL) { - return UMF_RESULT_SUCCESS; - } - - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; - umf_result_t ret; - if (ze_provider->freePolicyFlags == 0) { - ret = ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); - } else { - ze_memory_free_ext_desc_t desc = { - .stype = ZE_STRUCTURE_TYPE_MEMORY_FREE_EXT_DESC, - .pNext = NULL, - .freePolicy = ze_provider->freePolicyFlags}; - - ret = ze2umf_result( - g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); - } - - if (ret != UMF_RESULT_SUCCESS) { - return ret; - } - - if (update_stats) { - provider_ctl_stats_free(ze_provider, bytes); - } - - return UMF_RESULT_SUCCESS; -} - -static umf_result_t ze_memory_provider_free(void *provider, void *ptr, - size_t bytes) { - return ze_memory_provider_free_helper(provider, ptr, bytes, 1); -} - static umf_result_t query_min_page_size(ze_memory_provider_t *ze_provider, size_t *min_page_size) { assert(min_page_size); @@ -997,8 +1004,8 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, // TODO: add assertions to UMF and change it to be an assertion if (info->props.base != (void *)key) { - LOG_ERR("key:%p is different than base:%p", (void *)key, - info->props.base); + LOG_FATAL("key:%p is different than base:%p", (void *)key, + info->props.base); abort(); } @@ -1060,8 +1067,7 @@ static umf_result_t ze_memory_provider_resident_device_change(void *provider, if (ze_provider->resident_device_count == ze_provider->resident_device_capacity) { const uint32_t new_capacity = - ze_provider->resident_device_capacity * 2 + - 1; // +1 to work also with old capacity == 0 + ze_provider->resident_device_capacity + 1; ze_device_handle_t *new_handles = umf_ba_global_alloc(sizeof(ze_device_handle_t) * new_capacity); if (new_handles == NULL) { @@ -1087,7 +1093,6 @@ static umf_result_t ze_memory_provider_resident_device_change(void *provider, // found if (is_adding) { utils_write_unlock(&ze_provider->resident_device_rwlock); - // impossible for UR, should be an assertion LOG_ERR("trying to add resident device:%p but the device is " "already a peer of provider:%p", ze_device, provider); @@ -1109,6 +1114,9 @@ static umf_result_t ze_memory_provider_resident_device_change(void *provider, .failed_changes = 0, }; + // This is "hacky" and it will not work if someone uses pool without tracker + // or just use provider without pool. It can be solved by keeping track of + // allocations by the provider like in os_provider. umf_result_t result = umfMemoryTrackerIterateAll( &ze_memory_provider_resident_device_change_helper, &privData); if (result != UMF_RESULT_SUCCESS) { diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 6412b06c07..3310470780 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -21,16 +21,18 @@ add_umf_library( target_include_directories(umf_test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) -add_library(umf_test_mocks STATIC level_zero_mocks.cpp) +if(UMF_LEVEL_ZERO_ENABLED) + add_library(umf_test_mocks STATIC level_zero_mocks.cpp) -target_link_libraries(umf_test_mocks GTest::gmock) + target_link_libraries(umf_test_mocks GTest::gmock) -target_include_directories( - umf_test_mocks - PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src - ${UMF_CMAKE_SOURCE_DIR}/src/utils) + target_include_directories( + umf_test_mocks + PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src + ${UMF_CMAKE_SOURCE_DIR}/src/utils) -add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) -target_include_directories( - umf_ze_loopback PRIVATE ${ZE_LOADER_INCLUDE_DIRS} ${UMF_UTILS_DIR} - ${UMF_SRC_DIR} ${UMF_CMAKE_SOURCE_DIR}/include) + add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) + target_include_directories( + umf_ze_loopback PRIVATE ${ZE_LOADER_INCLUDE_DIRS} ${UMF_UTILS_DIR} + ${UMF_SRC_DIR} ${UMF_CMAKE_SOURCE_DIR}/include) +endif() diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index 1eb0ac3021..8aa4144158 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -1,3 +1,12 @@ +/* + * + * Copyright (C) 2023-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + #include "level_zero_mocks.h" #include "umf/providers/provider_level_zero.h" #include diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h index af15e1013f..a6a56b931a 100644 --- a/test/common/level_zero_mocks.h +++ b/test/common/level_zero_mocks.h @@ -1,3 +1,12 @@ +/* + * + * Copyright (C) 2023-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + #ifndef UMF_PROVIDER_LEVEL_ZERO_MOCKS_H #define UMF_PROVIDER_LEVEL_ZERO_MOCKS_H diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp index 2fba286cde..88b7cecba3 100644 --- a/test/common/ze_loopback.cpp +++ b/test/common/ze_loopback.cpp @@ -1,3 +1,12 @@ +/* + * + * Copyright (C) 2023-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + #include "ze_loopback.h" #include "utils_log.h" diff --git a/test/pools/pool_residency.cpp b/test/pools/pool_residency.cpp index 9a2b75e10a..53ebe65638 100644 --- a/test/pools/pool_residency.cpp +++ b/test/pools/pool_residency.cpp @@ -126,6 +126,25 @@ TEST_F(PoolResidencyTestFixture, umfPoolFree(pool, ptr); } +TEST_F(PoolResidencyTestFixture, + allocationThatFailedToBeMadeResidedShouldBeFreed) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_2})); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_2, _, _)) + .WillOnce(Return(ZE_RESULT_ERROR_DEVICE_LOST)); + EXPECT_CALL(l0mock, zeMemFree(CONTEXT, _)) + .WillOnce(Return(ZE_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE)); + + void *ptr = umfPoolMalloc(pool, 16 * 1024 * 1024); + EXPECT_EQ(ptr, nullptr); + + umfPoolFree(pool, ptr); +} + int main(int argc, char **argv) { InitGoogleTest(&argc, argv); AddGlobalTestEnvironment(new MockedLevelZeroTestEnvironment); From cabd5e7c2183c91119494aabc783851b9ed7adf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 22 Sep 2025 13:21:23 +0200 Subject: [PATCH 15/49] fix log test --- test/utils/utils_log.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/utils/utils_log.cpp b/test/utils/utils_log.cpp index 16d0358615..cd23a25b5d 100644 --- a/test/utils/utils_log.cpp +++ b/test/utils/utils_log.cpp @@ -31,9 +31,11 @@ int fput_count = 0; int mock_fputs(const char *s, FILE *stream) { fput_count++; +#ifndef UMF_DEVELOPER_MODE if (!expected_message.empty()) { EXPECT_STREQ(s, expected_message.c_str()); } +#endif EXPECT_EQ(stream, expected_stream); return (int)strlen(s); } From 7de5b1521a2f4563ada0fd20e69456a35cd7d14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 22 Sep 2025 14:20:15 +0200 Subject: [PATCH 16/49] removed usless ext_resident_device_change from memory_pool --- include/umf/memory_pool_ops.h | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index aa6db9a1f8..7754dcbe03 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -191,18 +191,6 @@ typedef struct umf_memory_pool_ops_t { /// failure. /// umf_result_t (*ext_trim_memory)(void *pool, size_t minBytesToKeep); - - /// - /// @brief Adds or removes devices on which allocations should be made - /// resident. - /// @param pool pointer to the memory pool - /// @param device device handle - /// @param isAdding Boolean indicating if peer is to be removed or added - /// @return UMF_RESULT_SUCCESS on success or appropriate error code on - /// failure. - /// - umf_result_t (*ext_resident_device_change)(void *pool, void *device, - bool isAdding); } umf_memory_pool_ops_t; #ifdef __cplusplus From 32ebea7a7582aeec31cd5a2331412a834431c61c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 22 Sep 2025 15:07:07 +0200 Subject: [PATCH 17/49] use umfLevelZeroMemoryProviderResidentDeviceChange instead of ops --- include/umf/memory_provider.h | 11 ----- include/umf/memory_provider_ops.h | 11 ----- include/umf/providers/provider_level_zero.h | 11 +++++ src/libumf.def | 2 +- src/libumf.map | 2 +- src/memory_provider.c | 27 +---------- src/provider/provider_cuda.c | 1 - src/provider/provider_level_zero.c | 45 +++++++++++-------- src/provider/provider_tracking.c | 1 - test/common/provider_null.c | 9 ---- test/pools/pool_residency.cpp | 8 ++-- .../provider_level_zero_residency.cpp | 24 ++++++---- 12 files changed, 62 insertions(+), 90 deletions(-) diff --git a/include/umf/memory_provider.h b/include/umf/memory_provider.h index a396a8a9d3..b98b893468 100644 --- a/include/umf/memory_provider.h +++ b/include/umf/memory_provider.h @@ -265,17 +265,6 @@ umf_result_t umfMemoryProviderAllocationMerge(umf_memory_provider_handle_t hProvider, void *lowPtr, void *highPtr, size_t totalSize); -/// @brief Adds or removes devices on which allocations should be made -/// resident. -/// @param hProvider handle to the memory provider -/// @param device device handle -/// @param isAdding Boolean indicating if peer is to be removed or added -/// @return UMF_RESULT_SUCCESS on success or appropriate error code on -/// failure. -umf_result_t -umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, - void *device, bool isAdding); - #ifdef __cplusplus } #endif diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 406da06d36..ff623bc371 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -322,17 +322,6 @@ typedef struct umf_memory_provider_ops_t { umf_result_t (*ext_get_allocation_properties_size)( void *provider, umf_memory_property_id_t memory_property_id, size_t *size); - - /// @brief Adds or removes devices on which allocations should be made - /// resident. - /// @param provider handle to the memory provider - /// @param device device handle - /// @param is_adding Boolean indicating if peer is to be removed or added - /// @return UMF_RESULT_SUCCESS on success or appropriate error code on - /// failure. - umf_result_t (*ext_resident_device_change)(void *provider, void *device, - bool is_adding); - } umf_memory_provider_ops_t; #ifdef __cplusplus diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index 65d7e3e781..b92560e377 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -101,6 +101,17 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetDeviceOrdinal( umf_result_t umfLevelZeroMemoryProviderParamsSetName( umf_level_zero_memory_provider_params_handle_t hParams, const char *name); +/// @brief Adds or removes devices on which allocations should be made +/// resident. +/// @param hProvider handle to the memory provider +/// @param device device handle +/// @param isAdding Boolean indicating if peer is to be removed or added +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on +/// failure. +umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( + umf_memory_provider_handle_t provider, ze_device_handle_t device, + bool is_adding); + const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void); #ifdef __cplusplus diff --git a/src/libumf.def b/src/libumf.def index 114682d3db..397ff74aeb 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -42,6 +42,7 @@ EXPORTS umfLevelZeroMemoryProviderParamsSetDevice umfLevelZeroMemoryProviderParamsSetMemoryType umfLevelZeroMemoryProviderParamsSetResidentDevices + umfLevelZeroMemoryProviderResidentDeviceChange umfMemoryProviderAlloc umfMemoryProviderAllocationMerge umfMemoryProviderAllocationSplit @@ -60,7 +61,6 @@ EXPORTS umfMemoryProviderPurgeForce umfMemoryProviderPurgeLazy umfMemoryProviderPutIPCHandle - umfMemoryProviderResidentDeviceChange umfMempolicyCreate umfMempolicyDestroy umfMempolicySetCustomSplitPartitions diff --git a/src/libumf.map b/src/libumf.map index 0c7436ee5b..d23c6a2e02 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -151,8 +151,8 @@ UMF_1.1 { umfGetMemoryProperty; umfGetMemoryPropertySize; umfJemallocPoolParamsSetName; - umfMemoryProviderResidentDeviceChange; umfLevelZeroMemoryProviderParamsSetName; + umfLevelZeroMemoryProviderResidentDeviceChange; umfOsMemoryProviderParamsSetName; umfPoolTrimMemory; umfScalablePoolParamsSetName; diff --git a/src/memory_provider.c b/src/memory_provider.c index 9d595aa636..b62d9f961f 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -154,14 +154,6 @@ static umf_result_t umfDefaultGetAllocationPropertiesSize( return UMF_RESULT_ERROR_NOT_SUPPORTED; } -static umf_result_t umfDefaultResidentDeviceChange(void *provider, void *device, - bool is_adding) { - (void)provider; - (void)device; - (void)is_adding; - return UMF_RESULT_ERROR_NOT_SUPPORTED; -} - void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { if (!ops->ext_purge_lazy) { ops->ext_purge_lazy = umfDefaultPurgeLazy; @@ -191,10 +183,6 @@ void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { ops->ext_get_allocation_properties_size = umfDefaultGetAllocationPropertiesSize; } - - if (!ops->ext_resident_device_change) { - ops->ext_resident_device_change = umfDefaultResidentDeviceChange; - } } void assignOpsIpcDefaults(umf_memory_provider_ops_t *ops) { @@ -617,17 +605,4 @@ umf_result_t umfMemoryProviderGetAllocationPropertiesSize( checkErrorAndSetLastProvider(res, hProvider); return res; -} - -umf_result_t -umfMemoryProviderResidentDeviceChange(umf_memory_provider_handle_t hProvider, - void *device, bool isAdding) { - - UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); - - umf_result_t res = hProvider->ops.ext_resident_device_change( - hProvider->provider_priv, device, isAdding); - - checkErrorAndSetLastProvider(res, hProvider); - return res; -} +} \ No newline at end of file diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index 4f4a00a3d5..b6c15d2d1b 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -829,7 +829,6 @@ static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { cu_memory_provider_get_allocation_properties, .ext_get_allocation_properties_size = cu_memory_provider_get_allocation_properties_size, - .ext_resident_device_change = NULL, }; const umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void) { diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 2ebf53d358..17e5dc99db 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -178,7 +178,7 @@ static ze_memory_type_t umf2ze_memory_type(umf_usm_memory_type_t memory_type) { static void init_ze_global_state(void) { - char *lib_name = getenv("UMF_ZE_LOADER_LIB_NAME"); + const char *lib_name = getenv("UMF_ZE_LOADER_LIB_NAME"); if (lib_name != NULL && lib_name[0] != '\0') { LOG_INFO("Using custom ze_loader library name: %s", lib_name); } else { @@ -371,7 +371,7 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( if (hDevices[first_idx] == hDevices[second_idx]) { LOG_ERR("resident devices are not unique, idx:%u and " "idx:%u both point to device:%p", - first_idx, second_idx, hDevices[first_idx]); + first_idx, second_idx, (void *)hDevices[first_idx]); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } } @@ -527,8 +527,8 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, utils_read_unlock(&ze_provider->resident_device_rwlock); LOG_ERR("making resident allocation %p of size:%lu on device %p " "failed with 0x%x", - *resultPtr, size, ze_provider->resident_device_handles[i], - ze_result); + *resultPtr, size, + (void *)ze_provider->resident_device_handles[i], ze_result); umf_result_t free_result = ze_memory_provider_free(ze_provider, *resultPtr, size); if (free_result != UMF_RESULT_SUCCESS) { @@ -539,7 +539,8 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, return ze2umf_result(ze_result); } LOG_DEBUG("allocation %p of size:%lu made resident on device %p", - *resultPtr, size, ze_provider->resident_device_handles[i]); + *resultPtr, size, + (void *)ze_provider->resident_device_handles[i]); } utils_read_unlock(&ze_provider->resident_device_rwlock); @@ -1034,22 +1035,21 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, return 0; } -static umf_result_t ze_memory_provider_resident_device_change(void *provider, - void *device, - bool is_adding) { - ze_memory_provider_t *ze_provider = provider; - ze_device_handle_t ze_device = device; +umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( + umf_memory_provider_handle_t provider, ze_device_handle_t device, + bool is_adding) { + ze_memory_provider_t *ze_provider = umfMemoryProviderGetPriv(provider); LOG_INFO("%s resident device %p, src_provider:%p, existing peers " "count:%d", - (is_adding ? "adding" : "removing"), ze_device, provider, - ze_provider->resident_device_count); + (is_adding ? "adding" : "removing"), (void *)device, + (void *)provider, ze_provider->resident_device_count); uint32_t existing_peer_index = 0; utils_write_lock(&ze_provider->resident_device_rwlock); while (existing_peer_index < ze_provider->resident_device_count && ze_provider->resident_device_handles[existing_peer_index] != - ze_device) { + device) { ++existing_peer_index; } @@ -1060,7 +1060,7 @@ static umf_result_t ze_memory_provider_resident_device_change(void *provider, utils_write_unlock(&ze_provider->resident_device_rwlock); LOG_ERR("trying to remove resident device %p but the device " "is not a peer of provider:%p currently", - ze_device, provider); + (void *)device, (void *)provider); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } // adding case @@ -1086,7 +1086,7 @@ static umf_result_t ze_memory_provider_resident_device_change(void *provider, ze_provider->resident_device_handles = new_handles; ze_provider->resident_device_capacity = new_capacity; } - ze_provider->resident_device_handles[existing_peer_index] = ze_device; + ze_provider->resident_device_handles[existing_peer_index] = device; ++ze_provider->resident_device_count; } else { @@ -1095,7 +1095,7 @@ static umf_result_t ze_memory_provider_resident_device_change(void *provider, utils_write_unlock(&ze_provider->resident_device_rwlock); LOG_ERR("trying to add resident device:%p but the device is " "already a peer of provider:%p", - ze_device, provider); + (void *)device, (void *)provider); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } // removing case, put last in place of removed one @@ -1108,7 +1108,7 @@ static umf_result_t ze_memory_provider_resident_device_change(void *provider, struct ze_memory_provider_resident_device_change_data privData = { .is_adding = is_adding, - .peer_device = ze_device, + .peer_device = device, .source_memory_provider = ze_provider, .success_changes = 0, .failed_changes = 0, @@ -1162,7 +1162,6 @@ static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { ze_memory_provider_get_allocation_properties, .ext_get_allocation_properties_size = ze_memory_provider_get_allocation_properties_size, - .ext_resident_device_change = ze_memory_provider_resident_device_change, }; const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { @@ -1258,4 +1257,14 @@ const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { return NULL; } +umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( + umf_memory_provider_handle_t provider, ze_device_handle_t device, + bool is_adding) { + (void)provider(void) device, + (void)is_adding LOG_ERR( + "L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " + "OFF)"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + #endif // !UMF_BUILD_LEVEL_ZERO_PROVIDER diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index e0ab69736e..b8f3cfa71c 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -1358,7 +1358,6 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .ext_ctl = NULL, .ext_get_allocation_properties = trackingGetAllocationProperties, .ext_get_allocation_properties_size = trackingGetAllocationPropertiesSize, - .ext_resident_device_change = NULL, }; static void free_ipc_cache_value(void *unused, void *ipc_cache_value) { diff --git a/test/common/provider_null.c b/test/common/provider_null.c index d91518f125..2ce8c78ddb 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -162,14 +162,6 @@ static umf_result_t nullGetAllocationPropertiesSize( return UMF_RESULT_SUCCESS; } -static umf_result_t nullResidentDeviceChange(void *provider, void *device, - bool is_adding) { - (void)provider; - (void)device; - (void)is_adding; - return UMF_RESULT_SUCCESS; -} - umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = nullInitialize, @@ -191,5 +183,4 @@ umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .ext_close_ipc_handle = nullCloseIpcHandle, .ext_get_allocation_properties = nullGetAllocationProperties, .ext_get_allocation_properties_size = nullGetAllocationPropertiesSize, - .ext_resident_device_change = nullResidentDeviceChange, }; diff --git a/test/pools/pool_residency.cpp b/test/pools/pool_residency.cpp index 53ebe65638..1d94383a0a 100644 --- a/test/pools/pool_residency.cpp +++ b/test/pools/pool_residency.cpp @@ -5,6 +5,8 @@ #include "../common/level_zero_mocks.h" #include "pool.hpp" #include "umf/pools/pool_disjoint.h" +#include "umf/providers/provider_level_zero.h" + #include "gtest/gtest.h" using namespace testing; @@ -67,7 +69,7 @@ TEST_F(PoolResidencyTestFixture, umf_memory_provider_handle_t provider; EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); - umfMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); + umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) .WillOnce( @@ -100,7 +102,7 @@ TEST_F(PoolResidencyTestFixture, umf_memory_provider_handle_t provider; EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); - umfMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); + umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); umfPoolFree(pool, ptr); } @@ -112,7 +114,7 @@ TEST_F(PoolResidencyTestFixture, umf_memory_provider_handle_t provider; EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); - umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, false); + umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, false); EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) .WillOnce( diff --git a/test/providers/provider_level_zero_residency.cpp b/test/providers/provider_level_zero_residency.cpp index bd511e3b9a..95cbe9a3b5 100644 --- a/test/providers/provider_level_zero_residency.cpp +++ b/test/providers/provider_level_zero_residency.cpp @@ -29,46 +29,54 @@ class LevelZeroResidencyTestFixture : public Test { TEST_F(LevelZeroResidencyTestFixture, addNonexistingDeviceShouldSucceed) { provider = l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, true), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, + true), UMF_RESULT_SUCCESS); } TEST_F(LevelZeroResidencyTestFixture, addExistingDeviceShouldFail) { provider = l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_5, true), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_5, + true), UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_F(LevelZeroResidencyTestFixture, removeNonexistingDeviceShouldFail) { provider = l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_0, false), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_0, + false), UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_F(LevelZeroResidencyTestFixture, removeExistingDeviceShouldSucceed) { provider = l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_1, false), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_1, + false), UMF_RESULT_SUCCESS); } TEST_F(LevelZeroResidencyTestFixture, addDeviceTwiceShouldFail) { provider = l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, true), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, + true), UMF_RESULT_SUCCESS); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_2, true), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, + true), UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_F(LevelZeroResidencyTestFixture, removeDeviceTwiceShouldFail) { provider = l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_3, false), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_3, + false), UMF_RESULT_SUCCESS); - ASSERT_EQ(umfMemoryProviderResidentDeviceChange(provider, DEVICE_3, false), + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_3, + false), UMF_RESULT_ERROR_INVALID_ARGUMENT); } From 50db5c58bd46b54d4c90319cf098b1c8fc5b0529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 22 Sep 2025 15:20:28 +0200 Subject: [PATCH 18/49] reverted log change as it was moved to different PR --- src/utils/utils_log.h | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/utils/utils_log.h b/src/utils/utils_log.h index eced32395f..c0e0a95723 100644 --- a/src/utils/utils_log.h +++ b/src/utils/utils_log.h @@ -24,25 +24,17 @@ typedef enum { LOG_FATAL } utils_log_level_t; -#ifdef UMF_DEVELOPER_MODE -#define UMF_STRINGIFY(x) #x -#define UMF_TOSTRING(x) UMF_STRINGIFY(x) -#define UMF_FUNC_DESC() __FILE__ ":" UMF_TOSTRING(__LINE__) -#else -#define UMF_FUNC_DESC() __func__ -#endif - -#define LOG_DEBUG(...) utils_log(LOG_DEBUG, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_INFO(...) utils_log(LOG_INFO, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_WARN(...) utils_log(LOG_WARNING, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_ERR(...) utils_log(LOG_ERROR, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_FATAL(...) utils_log(LOG_FATAL, UMF_FUNC_DESC(), __VA_ARGS__); - -#define LOG_PDEBUG(...) utils_plog(LOG_DEBUG, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_PINFO(...) utils_plog(LOG_INFO, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_PWARN(...) utils_plog(LOG_WARNING, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_PERR(...) utils_plog(LOG_ERROR, UMF_FUNC_DESC(), __VA_ARGS__); -#define LOG_PFATAL(...) utils_plog(LOG_FATAL, UMF_FUNC_DESC(), __VA_ARGS__); +#define LOG_DEBUG(...) utils_log(LOG_DEBUG, __func__, __VA_ARGS__); +#define LOG_INFO(...) utils_log(LOG_INFO, __func__, __VA_ARGS__); +#define LOG_WARN(...) utils_log(LOG_WARNING, __func__, __VA_ARGS__); +#define LOG_ERR(...) utils_log(LOG_ERROR, __func__, __VA_ARGS__); +#define LOG_FATAL(...) utils_log(LOG_FATAL, __func__, __VA_ARGS__); + +#define LOG_PDEBUG(...) utils_plog(LOG_DEBUG, __func__, __VA_ARGS__); +#define LOG_PINFO(...) utils_plog(LOG_INFO, __func__, __VA_ARGS__); +#define LOG_PWARN(...) utils_plog(LOG_WARNING, __func__, __VA_ARGS__); +#define LOG_PERR(...) utils_plog(LOG_ERROR, __func__, __VA_ARGS__); +#define LOG_PFATAL(...) utils_plog(LOG_FATAL, __func__, __VA_ARGS__); void utils_log_init(void); #ifdef _WIN32 From 780794cdc8394392d5d52ba0c8190d38f751950a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 22 Sep 2025 15:32:18 +0200 Subject: [PATCH 19/49] removed accidental changes --- include/umf/memory_provider_ops.h | 2 +- include/umf/providers/provider_level_zero.h | 2 ++ src/memory_provider.c | 2 +- src/provider/provider_level_zero.c | 4 +--- src/utils/utils_level_zero.cpp | 1 + test/providers/provider_level_zero_residency.cpp | 2 +- test/utils/utils_log.cpp | 2 -- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index ff623bc371..a520ed8891 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -11,7 +11,6 @@ #define UMF_MEMORY_PROVIDER_OPS_H 1 #include -#include #include @@ -322,6 +321,7 @@ typedef struct umf_memory_provider_ops_t { umf_result_t (*ext_get_allocation_properties_size)( void *provider, umf_memory_property_id_t memory_property_id, size_t *size); + } umf_memory_provider_ops_t; #ifdef __cplusplus diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index b92560e377..297d3122a3 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -8,6 +8,8 @@ #ifndef UMF_LEVEL_ZERO_PROVIDER_H #define UMF_LEVEL_ZERO_PROVIDER_H +#include + #include #ifdef __cplusplus diff --git a/src/memory_provider.c b/src/memory_provider.c index b62d9f961f..324fa751bd 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -605,4 +605,4 @@ umf_result_t umfMemoryProviderGetAllocationPropertiesSize( checkErrorAndSetLastProvider(res, hProvider); return res; -} \ No newline at end of file +} diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 17e5dc99db..5397ff7e3b 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -275,6 +275,7 @@ umf_result_t umfLevelZeroMemoryProviderParamsCreate( umf_result_t umfLevelZeroMemoryProviderParamsDestroy( umf_level_zero_memory_provider_params_handle_t hParams) { umf_ba_global_free(hParams); + return UMF_RESULT_SUCCESS; } @@ -355,12 +356,10 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetName( umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, ze_device_handle_t *hDevices, uint32_t deviceCount) { - if (!hParams) { LOG_ERR("Level Zero memory provider params handle is NULL"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (deviceCount > 0 && hDevices == NULL) { LOG_ERR("Resident devices array is NULL, but deviceCount is not zero"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -519,7 +518,6 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, utils_read_lock(&ze_provider->resident_device_rwlock); for (uint32_t i = 0; i < ze_provider->resident_device_count; i++) { - ze_result = g_ze_ops.zeContextMakeMemoryResident( ze_provider->context, ze_provider->resident_device_handles[i], *resultPtr, size); diff --git a/src/utils/utils_level_zero.cpp b/src/utils/utils_level_zero.cpp index 5f2d16666a..ee7aa85eae 100644 --- a/src/utils/utils_level_zero.cpp +++ b/src/utils/utils_level_zero.cpp @@ -336,6 +336,7 @@ static void utils_ze_init_level_zero_once(void) { int utils_ze_init_level_zero(void) { utils_init_once(&level_zero_init_flag, utils_ze_init_level_zero_once); + return InitResult; } diff --git a/test/providers/provider_level_zero_residency.cpp b/test/providers/provider_level_zero_residency.cpp index 95cbe9a3b5..098d5f5535 100644 --- a/test/providers/provider_level_zero_residency.cpp +++ b/test/providers/provider_level_zero_residency.cpp @@ -84,4 +84,4 @@ int main(int argc, char **argv) { InitGoogleTest(&argc, argv); AddGlobalTestEnvironment(new MockedLevelZeroTestEnvironment); return RUN_ALL_TESTS(); -} \ No newline at end of file +} diff --git a/test/utils/utils_log.cpp b/test/utils/utils_log.cpp index cd23a25b5d..16d0358615 100644 --- a/test/utils/utils_log.cpp +++ b/test/utils/utils_log.cpp @@ -31,11 +31,9 @@ int fput_count = 0; int mock_fputs(const char *s, FILE *stream) { fput_count++; -#ifndef UMF_DEVELOPER_MODE if (!expected_message.empty()) { EXPECT_STREQ(s, expected_message.c_str()); } -#endif EXPECT_EQ(stream, expected_stream); return (int)strlen(s); } From 4f75bca28d5fa1a999d49ad978844efb7d7f1352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 22 Sep 2025 15:34:31 +0200 Subject: [PATCH 20/49] fix doc --- include/umf/providers/provider_level_zero.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index 297d3122a3..22c3513351 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -105,9 +105,9 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetName( /// @brief Adds or removes devices on which allocations should be made /// resident. -/// @param hProvider handle to the memory provider +/// @param provider handle to the memory provider /// @param device device handle -/// @param isAdding Boolean indicating if peer is to be removed or added +/// @param is_adding Boolean indicating if peer is to be removed or added /// @return UMF_RESULT_SUCCESS on success or appropriate error code on /// failure. umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( From 9f1c5887121cda065851b3833e8c1d70a8f3a20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Tue, 23 Sep 2025 11:40:50 +0200 Subject: [PATCH 21/49] applied Lukasz S. comments --- src/provider/provider_level_zero.c | 14 +++++--------- test/common/level_zero_mocks.cpp | 2 +- test/common/level_zero_mocks.h | 11 ++++++----- test/common/ze_loopback.cpp | 6 +----- test/common/ze_loopback.h | 8 ++++---- test/pools/pool_residency.cpp | 4 ++-- test/providers/provider_level_zero_residency.cpp | 2 +- 7 files changed, 20 insertions(+), 27 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 5397ff7e3b..126df53478 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -1001,12 +1001,7 @@ static int ze_memory_provider_resident_device_change_helper(uintptr_t key, return 0; } - // TODO: add assertions to UMF and change it to be an assertion - if (info->props.base != (void *)key) { - LOG_FATAL("key:%p is different than base:%p", (void *)key, - info->props.base); - abort(); - } + assert(info->props.base == (void *)key); ze_result_t result; if (change_data->is_adding) { @@ -1258,9 +1253,10 @@ const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( umf_memory_provider_handle_t provider, ze_device_handle_t device, bool is_adding) { - (void)provider(void) device, - (void)is_adding LOG_ERR( - "L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " + (void)provider; + (void)device; + (void)is_adding; + LOG_ERR("L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " "OFF)"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index 8aa4144158..f2d212ca78 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2025 Intel Corporation + * Copyright (C) 2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h index a6a56b931a..52902a92bc 100644 --- a/test/common/level_zero_mocks.h +++ b/test/common/level_zero_mocks.h @@ -1,14 +1,14 @@ /* * - * Copyright (C) 2023-2025 Intel Corporation + * Copyright (C) 2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * */ -#ifndef UMF_PROVIDER_LEVEL_ZERO_MOCKS_H -#define UMF_PROVIDER_LEVEL_ZERO_MOCKS_H +#ifndef UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H +#define UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H #include "utils/utils_log.h" #include "ze_loopback.h" @@ -66,7 +66,8 @@ class LevelZeroMock : public LevelZero { MOCK_METHOD2(zeMemFree, ze_result_t(ze_context_handle_t hContext, void *ptr)); - // helper setting all expects related to successful l0 provider creation & initialization and calling its creation & initialization + // helper setting all expects related to successful l0 provider creation + // and initialization and calling its creation and initialization umf_memory_provider_handle_t initializeMemoryProviderWithResidentDevices( ze_device_handle_t device, std::vector residentDevices, @@ -83,4 +84,4 @@ class MockedLevelZeroTestEnvironment : public ::testing::Environment { void TearDown() override; }; -#endif //UMF_PROVIDER_LEVEL_ZERO_MOCKS_H +#endif //UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp index 88b7cecba3..33b8a7d177 100644 --- a/test/common/ze_loopback.cpp +++ b/test/common/ze_loopback.cpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2025 Intel Corporation + * Copyright (C) 2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -288,7 +288,3 @@ zeMemFreeExt(ze_context_handle_t hContext, LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); abort(); } - -// defined in both libze_ops from src/utils/utils_level_zero.cpp and -// ze_ops_t operations from src/provider/provider_level_zero.c -// zeDeviceGetProperties, zeMemFree, zeMemGetAllocProperties, zeMemFree, zeMemAllocDevice, find them in the former ones \ No newline at end of file diff --git a/test/common/ze_loopback.h b/test/common/ze_loopback.h index d3af3cc649..a58462316f 100644 --- a/test/common/ze_loopback.h +++ b/test/common/ze_loopback.h @@ -1,9 +1,9 @@ -// Copyright (C) 2024-2025 Intel Corporation +// Copyright (C) 2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_ZE_LOOPBACK_H -#define UMF_ZE_LOOPBACK_H +#ifndef UMF_TEST_ZE_LOOPBACK_H +#define UMF_TEST_ZE_LOOPBACK_H #include @@ -32,4 +32,4 @@ class LevelZero { extern LevelZero *level_zero_mock; -#endif //UMF_ZE_LOOPBACK_H +#endif //UMF_TEST_ZE_LOOPBACK_H diff --git a/test/pools/pool_residency.cpp b/test/pools/pool_residency.cpp index 1d94383a0a..3a939bc147 100644 --- a/test/pools/pool_residency.cpp +++ b/test/pools/pool_residency.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024-2025 Intel Corporation +// Copyright (C) 2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -151,4 +151,4 @@ int main(int argc, char **argv) { InitGoogleTest(&argc, argv); AddGlobalTestEnvironment(new MockedLevelZeroTestEnvironment); return RUN_ALL_TESTS(); -} \ No newline at end of file +} diff --git a/test/providers/provider_level_zero_residency.cpp b/test/providers/provider_level_zero_residency.cpp index 098d5f5535..9e7dcafda4 100644 --- a/test/providers/provider_level_zero_residency.cpp +++ b/test/providers/provider_level_zero_residency.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024-2025 Intel Corporation +// Copyright (C) 2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception From 3f4db4fe94809cd3ac8eb76c0617b5c19ac3a2d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Tue, 23 Sep 2025 12:05:13 +0200 Subject: [PATCH 22/49] change ze_api include due to windows compile error --- test/common/ze_loopback.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/ze_loopback.h b/test/common/ze_loopback.h index a58462316f..9a999edc5d 100644 --- a/test/common/ze_loopback.h +++ b/test/common/ze_loopback.h @@ -5,7 +5,7 @@ #ifndef UMF_TEST_ZE_LOOPBACK_H #define UMF_TEST_ZE_LOOPBACK_H -#include +#include "ze_api.h" class LevelZero { public: From cf4254ab5f46e44bc02ee158ca66b2be4defcbbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Tue, 23 Sep 2025 13:23:01 +0200 Subject: [PATCH 23/49] fix link in windows --- test/common/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 3310470780..5ae6d342be 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -22,6 +22,12 @@ target_include_directories(umf_test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) if(UMF_LEVEL_ZERO_ENABLED) + + pkg_check_modules(ZE_LOADER ze_loader) + if(NOT ZE_LOADER_FOUND) + find_package(ZE_LOADER REQUIRED ze_loader) + endif() + add_library(umf_test_mocks STATIC level_zero_mocks.cpp) target_link_libraries(umf_test_mocks GTest::gmock) @@ -35,4 +41,5 @@ if(UMF_LEVEL_ZERO_ENABLED) target_include_directories( umf_ze_loopback PRIVATE ${ZE_LOADER_INCLUDE_DIRS} ${UMF_UTILS_DIR} ${UMF_SRC_DIR} ${UMF_CMAKE_SOURCE_DIR}/include) + target_link_libraries(umf_ze_loopback ${UMF_UTILS_FOR_TEST}) endif() From 7bd5c249205739f0dc7904e67d0d326012d0d3ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Wed, 24 Sep 2025 14:25:23 +0200 Subject: [PATCH 24/49] not using utils_log in ze_loopback --- test/common/CMakeLists.txt | 1 - test/common/ze_loopback.cpp | 81 +++++++++++++++---------------------- 2 files changed, 32 insertions(+), 50 deletions(-) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 5ae6d342be..7b08237bd9 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -41,5 +41,4 @@ if(UMF_LEVEL_ZERO_ENABLED) target_include_directories( umf_ze_loopback PRIVATE ${ZE_LOADER_INCLUDE_DIRS} ${UMF_UTILS_DIR} ${UMF_SRC_DIR} ${UMF_CMAKE_SOURCE_DIR}/include) - target_link_libraries(umf_ze_loopback ${UMF_UTILS_FOR_TEST}) endif() diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp index 33b8a7d177..3ca4cd4c73 100644 --- a/test/common/ze_loopback.cpp +++ b/test/common/ze_loopback.cpp @@ -8,23 +8,27 @@ */ #include "ze_loopback.h" -#include "utils_log.h" #include +#include LevelZero *level_zero_mock = nullptr; -// -// libze_ops from src/utils/utils_level_zero.cpp -// - -void check_mock_present() { +static void check_mock_present() { if (level_zero_mock == nullptr) { - LOG_FATAL("level_zero_mock was not set"); + std::cerr << "level_zero_mock was not set\n"; abort(); } } +#define FAIL_NOT_IMPLEMENTED \ + std::cerr << __func__ << " not implemented in ze_loopback.cpp\n"; \ + abort(); + +// +// libze_ops from src/utils/utils_level_zero.cpp +// + ZE_APIEXPORT ze_result_t ZE_APICALL zeInit(ze_init_flags_t flags) { (void)flags; return ZE_RESULT_SUCCESS; @@ -34,8 +38,7 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeDriverGet(uint32_t *pCount, ze_driver_handle_t *phDrivers) { (void)phDrivers; (void)pCount; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeDeviceGet(ze_driver_handle_t hDriver, @@ -44,8 +47,7 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeDeviceGet(ze_driver_handle_t hDriver, (void)hDriver; (void)pCount; (void)phDevices; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeDeviceGetProperties( @@ -60,15 +62,13 @@ zeContextCreate(ze_driver_handle_t hDriver, const ze_context_desc_t *desc, (void)hDriver; (void)desc; (void)phContext; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeContextDestroy(ze_context_handle_t hContext) { (void)hContext; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED; } ZE_APIEXPORT ze_result_t ZE_APICALL @@ -79,15 +79,13 @@ zeCommandQueueCreate(ze_context_handle_t hContext, ze_device_handle_t hDevice, (void)hDevice; (void)desc; (void)phCommandQueue; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueDestroy(ze_command_queue_handle_t hCommandQueue) { (void)hCommandQueue; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueExecuteCommandLists( @@ -97,16 +95,14 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueExecuteCommandLists( (void)numCommandLists; (void)phCommandLists; (void)hFence; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueSynchronize( ze_command_queue_handle_t hCommandQueue, uint64_t timeout) { (void)hCommandQueue; (void)timeout; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL @@ -117,22 +113,19 @@ zeCommandListCreate(ze_context_handle_t hContext, ze_device_handle_t hDevice, (void)hDevice; (void)desc; (void)phCommandList; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListDestroy(ze_command_list_handle_t hCommandList) { (void)hCommandList; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListClose(ze_command_list_handle_t hCommandList) { (void)hCommandList; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryCopy( @@ -146,8 +139,7 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryCopy( (void)hSignalEvent; (void)numWaitEvents; (void)phWaitEvents; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryFill( @@ -162,8 +154,7 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryFill( (void)hSignalEvent; (void)numWaitEvents; (void)phWaitEvents; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ZE_APIEXPORT ze_result_t ZE_APICALL @@ -195,8 +186,7 @@ zeDeviceGetMemoryProperties(ze_device_handle_t hDevice, uint32_t *pCount, (void)hDevice; (void)pCount; (void)pMemProperties; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } // @@ -212,8 +202,7 @@ ze_result_t ZE_APICALL zeMemAllocHost(ze_context_handle_t hContext, (void)size; (void)alignment; (void)pptr; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ze_result_t ZE_APICALL zeMemAllocShared( @@ -227,8 +216,7 @@ ze_result_t ZE_APICALL zeMemAllocShared( (void)alignment; (void)hDevice; (void)pptr; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ze_result_t ZE_APICALL zeMemGetIpcHandle(ze_context_handle_t hContext, @@ -237,16 +225,14 @@ ze_result_t ZE_APICALL zeMemGetIpcHandle(ze_context_handle_t hContext, (void)hContext; (void)ptr; (void)pIpcHandle; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ze_result_t ZE_APICALL zeMemPutIpcHandle(ze_context_handle_t hContext, ze_ipc_mem_handle_t handle) { (void)hContext; (void)handle; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ze_result_t ZE_APICALL zeMemOpenIpcHandle(ze_context_handle_t hContext, @@ -259,16 +245,14 @@ ze_result_t ZE_APICALL zeMemOpenIpcHandle(ze_context_handle_t hContext, (void)handle; (void)flags; (void)pptr; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ze_result_t ZE_APICALL zeMemCloseIpcHandle(ze_context_handle_t hContext, const void *ptr) { (void)hContext; (void)ptr; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } ze_result_t ZE_APICALL zeContextMakeMemoryResident(ze_context_handle_t hContext, @@ -285,6 +269,5 @@ zeMemFreeExt(ze_context_handle_t hContext, (void)hContext; (void)pMemFreeDesc; (void)ptr; - LOG_FATAL("not implemented, call in ze_loader_mock not passed to gmock"); - abort(); + FAIL_NOT_IMPLEMENTED } From dbb8298b7c198f95f4d7aef526d181887c92c656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Wed, 24 Sep 2025 16:13:07 +0200 Subject: [PATCH 25/49] fix includes for level_zero_mocks.cpp --- test/common/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 7b08237bd9..d86abe5892 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -35,7 +35,7 @@ if(UMF_LEVEL_ZERO_ENABLED) target_include_directories( umf_test_mocks PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src - ${UMF_CMAKE_SOURCE_DIR}/src/utils) + ${UMF_CMAKE_SOURCE_DIR}/src/utils ${ZE_LOADER_INCLUDE_DIRS}) add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) target_include_directories( From 58fae70b7dedb9fdb6cc17f03d80258549015c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Thu, 25 Sep 2025 14:43:17 +0200 Subject: [PATCH 26/49] windows use _putenv_s instead of setenv --- test/common/level_zero_mocks.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index f2d212ca78..c36780e8e3 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -91,6 +91,10 @@ TestCreateMemoryAllocationProperties(uint32_t modifier) { } void MockedLevelZeroTestEnvironment::SetUp() { +#ifdef _WIN32 + _putenv_s("UMF_ZE_LOADER_LIB_NAME", "umf_ze_loopback.dll"); +#else setenv("UMF_ZE_LOADER_LIB_NAME", "libumf_ze_loopback.so", 1); +#endif } void MockedLevelZeroTestEnvironment::TearDown() {} From 53bfcb0c276cdb7f8f8ee32b920a4344c1a8ea7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Thu, 25 Sep 2025 15:14:39 +0200 Subject: [PATCH 27/49] TestCreatePointer fix --- test/common/level_zero_mocks.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h index 52902a92bc..f72995fa79 100644 --- a/test/common/level_zero_mocks.h +++ b/test/common/level_zero_mocks.h @@ -17,8 +17,8 @@ // TEST CREATE methods for objects -template constexpr T TestCreatePointer(uint32_t modifier = 0) { - return reinterpret_cast(0x1000 + modifier); +template constexpr T TestCreatePointer(uintptr_t modifier = 0) { + return reinterpret_cast(static_cast(0x1000) + modifier); } ze_device_properties_t TestCreateDeviceProperties(); From 822f278de6927ee0ffc0aae1efedaa66affd8774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Thu, 25 Sep 2025 16:05:08 +0200 Subject: [PATCH 28/49] more windows fixes --- test/common/ze_loopback.cpp | 2 -- test/pools/pool_residency.cpp | 8 +++++--- test/providers/provider_level_zero_residency.cpp | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp index 3ca4cd4c73..9253050bde 100644 --- a/test/common/ze_loopback.cpp +++ b/test/common/ze_loopback.cpp @@ -12,8 +12,6 @@ #include #include -LevelZero *level_zero_mock = nullptr; - static void check_mock_present() { if (level_zero_mock == nullptr) { std::cerr << "level_zero_mock was not set\n"; diff --git a/test/pools/pool_residency.cpp b/test/pools/pool_residency.cpp index 3a939bc147..c99c401625 100644 --- a/test/pools/pool_residency.cpp +++ b/test/pools/pool_residency.cpp @@ -11,6 +11,8 @@ using namespace testing; +LevelZero *level_zero_mock = nullptr; + class PoolResidencyTestFixture : public Test { protected: StrictMock l0mock; @@ -67,7 +69,7 @@ TEST_F(PoolResidencyTestFixture, initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_0})); - umf_memory_provider_handle_t provider; + umf_memory_provider_handle_t provider = nullptr; EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); @@ -100,7 +102,7 @@ TEST_F(PoolResidencyTestFixture, EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_4, _, _)) .WillOnce(Return(ZE_RESULT_SUCCESS)); - umf_memory_provider_handle_t provider; + umf_memory_provider_handle_t provider = nullptr; EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); @@ -112,7 +114,7 @@ TEST_F(PoolResidencyTestFixture, initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( OUR_DEVICE, {DEVICE_2})); - umf_memory_provider_handle_t provider; + umf_memory_provider_handle_t provider = nullptr; EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, false); diff --git a/test/providers/provider_level_zero_residency.cpp b/test/providers/provider_level_zero_residency.cpp index 9e7dcafda4..81ed4dc5ce 100644 --- a/test/providers/provider_level_zero_residency.cpp +++ b/test/providers/provider_level_zero_residency.cpp @@ -8,6 +8,8 @@ #include "gtest/gtest.h" +LevelZero *level_zero_mock = nullptr; + using namespace testing; class LevelZeroResidencyTestFixture : public Test { From 340bfb159e15103197751730a45d8b145eb0842f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Thu, 25 Sep 2025 22:42:17 +0200 Subject: [PATCH 29/49] windows next fix attempt --- test/common/level_zero_mocks.cpp | 19 +++++++++++++++++-- test/common/level_zero_mocks.h | 2 ++ test/common/ze_loopback.cpp | 2 ++ test/common/ze_loopback.h | 2 -- test/pools/pool_residency.cpp | 10 +++++----- .../provider_level_zero_residency.cpp | 8 ++++---- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index c36780e8e3..f5a5e59443 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -8,7 +8,9 @@ */ #include "level_zero_mocks.h" + #include "umf/providers/provider_level_zero.h" +#include "utils_load_library.h" #include using namespace ::testing; @@ -92,9 +94,22 @@ TestCreateMemoryAllocationProperties(uint32_t modifier) { void MockedLevelZeroTestEnvironment::SetUp() { #ifdef _WIN32 - _putenv_s("UMF_ZE_LOADER_LIB_NAME", "umf_ze_loopback.dll"); + const char *lib_name = "umf_ze_loopback.dll"; + _putenv_s("UMF_ZE_LOADER_LIB_NAME", lib_name); #else - setenv("UMF_ZE_LOADER_LIB_NAME", "libumf_ze_loopback.so", 1); + const char *lib_name = "libumf_ze_loopback.so"; + setenv("UMF_ZE_LOADER_LIB_NAME", lib_name, 1); #endif + + void *lib_handle = + utils_open_library(lib_name, UMF_UTIL_OPEN_LIBRARY_NO_LOAD); + ASSERT_NE(lib_handle, nullptr); + + l0interface = static_cast( + utils_get_symbol_addr(lib_handle, "level_zero_mock", lib_name)); + ASSERT_NE(l0interface, nullptr); + ASSERT_EQ(*l0interface, nullptr); } void MockedLevelZeroTestEnvironment::TearDown() {} + +LevelZero **MockedLevelZeroTestEnvironment::l0interface; \ No newline at end of file diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h index f72995fa79..eeaecc4c36 100644 --- a/test/common/level_zero_mocks.h +++ b/test/common/level_zero_mocks.h @@ -80,6 +80,8 @@ class LevelZeroMock : public LevelZero { // important, makes UMF load ze_loopback instead of regular l0 class MockedLevelZeroTestEnvironment : public ::testing::Environment { public: + static LevelZero **l0interface; + void SetUp() override; void TearDown() override; }; diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp index 9253050bde..3ca4cd4c73 100644 --- a/test/common/ze_loopback.cpp +++ b/test/common/ze_loopback.cpp @@ -12,6 +12,8 @@ #include #include +LevelZero *level_zero_mock = nullptr; + static void check_mock_present() { if (level_zero_mock == nullptr) { std::cerr << "level_zero_mock was not set\n"; diff --git a/test/common/ze_loopback.h b/test/common/ze_loopback.h index 9a999edc5d..2bfa441be5 100644 --- a/test/common/ze_loopback.h +++ b/test/common/ze_loopback.h @@ -30,6 +30,4 @@ class LevelZero { virtual ze_result_t zeMemFree(ze_context_handle_t hContext, void *ptr) = 0; }; -extern LevelZero *level_zero_mock; - #endif //UMF_TEST_ZE_LOOPBACK_H diff --git a/test/pools/pool_residency.cpp b/test/pools/pool_residency.cpp index c99c401625..8f4db27a3f 100644 --- a/test/pools/pool_residency.cpp +++ b/test/pools/pool_residency.cpp @@ -11,16 +11,16 @@ using namespace testing; -LevelZero *level_zero_mock = nullptr; - class PoolResidencyTestFixture : public Test { protected: - StrictMock l0mock; umf_memory_pool_handle_t pool = nullptr; const ze_device_handle_t OUR_DEVICE; + StrictMock l0mock; PoolResidencyTestFixture() - : OUR_DEVICE(TestCreatePointer(777)) {} + : OUR_DEVICE(TestCreatePointer(777)) { + *MockedLevelZeroTestEnvironment::l0interface = &l0mock; + } void initializeMemoryPool(umf_memory_provider_handle_t provider) { @@ -34,7 +34,7 @@ class PoolResidencyTestFixture : public Test { umf_test::defaultDisjointPoolConfigDestroy(params); } - void SetUp() override { level_zero_mock = &l0mock; } + void SetUp() override {} void TearDown() override { if (pool != nullptr) { EXPECT_CALL(l0mock, zeMemFree(CONTEXT, _)) diff --git a/test/providers/provider_level_zero_residency.cpp b/test/providers/provider_level_zero_residency.cpp index 81ed4dc5ce..808547b213 100644 --- a/test/providers/provider_level_zero_residency.cpp +++ b/test/providers/provider_level_zero_residency.cpp @@ -8,8 +8,6 @@ #include "gtest/gtest.h" -LevelZero *level_zero_mock = nullptr; - using namespace testing; class LevelZeroResidencyTestFixture : public Test { @@ -19,9 +17,11 @@ class LevelZeroResidencyTestFixture : public Test { const ze_device_handle_t OUR_DEVICE; LevelZeroResidencyTestFixture() - : OUR_DEVICE(TestCreatePointer(777)) {} + : OUR_DEVICE(TestCreatePointer(777)) { + *MockedLevelZeroTestEnvironment::l0interface = &l0mock; + } - void SetUp() override { level_zero_mock = &l0mock; } + void SetUp() override {} void TearDown() override { Mock::VerifyAndClearExpectations(&l0mock); umfMemoryProviderDestroy(provider); From fad510c8d385525ef20c4dcb8a0ebdb2cb6225ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 26 Sep 2025 09:46:16 +0200 Subject: [PATCH 30/49] fixed sharedlib compilation --- test/CMakeLists.txt | 2 +- test/common/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e08397eb29..b3c2ecd703 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -431,7 +431,7 @@ else() LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) endif() -if(UMF_LEVEL_ZERO_ENABLED) +if(UMF_BUILD_LEVEL_ZERO_PROVIDER) add_umf_test( NAME provider_level_zero_residency SRCS providers/provider_level_zero_residency.cpp diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index d86abe5892..82b8ae5942 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -21,7 +21,7 @@ add_umf_library( target_include_directories(umf_test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) -if(UMF_LEVEL_ZERO_ENABLED) +if(UMF_BUILD_LEVEL_ZERO_PROVIDER) pkg_check_modules(ZE_LOADER ze_loader) if(NOT ZE_LOADER_FOUND) @@ -30,7 +30,7 @@ if(UMF_LEVEL_ZERO_ENABLED) add_library(umf_test_mocks STATIC level_zero_mocks.cpp) - target_link_libraries(umf_test_mocks GTest::gmock) + target_link_libraries(umf_test_mocks GTest::gmock umf_utils) target_include_directories( umf_test_mocks From 636477a11119a4f301e8b6b18ab4b2fc8d03e395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 26 Sep 2025 10:16:01 +0200 Subject: [PATCH 31/49] removed pkg_check_modules ZE_LOADER --- test/common/CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 82b8ae5942..dc290bf922 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -22,12 +22,6 @@ target_include_directories(umf_test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) if(UMF_BUILD_LEVEL_ZERO_PROVIDER) - - pkg_check_modules(ZE_LOADER ze_loader) - if(NOT ZE_LOADER_FOUND) - find_package(ZE_LOADER REQUIRED ze_loader) - endif() - add_library(umf_test_mocks STATIC level_zero_mocks.cpp) target_link_libraries(umf_test_mocks GTest::gmock umf_utils) From a6e456e18447bd90707b9a03592eb9c81732675c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 26 Sep 2025 10:28:10 +0200 Subject: [PATCH 32/49] change ZE_LOADER_INCLUDE_DIRS into LEVEL_ZERO_INCLUDE_DIRS --- test/common/CMakeLists.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index dc290bf922..0b9a8b9d15 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -23,16 +23,14 @@ target_include_directories(umf_test_common if(UMF_BUILD_LEVEL_ZERO_PROVIDER) add_library(umf_test_mocks STATIC level_zero_mocks.cpp) - target_link_libraries(umf_test_mocks GTest::gmock umf_utils) - target_include_directories( umf_test_mocks PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src - ${UMF_CMAKE_SOURCE_DIR}/src/utils ${ZE_LOADER_INCLUDE_DIRS}) + ${UMF_CMAKE_SOURCE_DIR}/src/utils ${LEVEL_ZERO_INCLUDE_DIRS}) add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) target_include_directories( - umf_ze_loopback PRIVATE ${ZE_LOADER_INCLUDE_DIRS} ${UMF_UTILS_DIR} + umf_ze_loopback PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS} ${UMF_UTILS_DIR} ${UMF_SRC_DIR} ${UMF_CMAKE_SOURCE_DIR}/include) endif() From 877a36eaa38a43d9b1e1f0c75384b0d5a5681bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 26 Sep 2025 11:07:05 +0200 Subject: [PATCH 33/49] fix includes for umf_test_mocks and umf_ze_loopback --- test/common/CMakeLists.txt | 10 ++++------ test/common/level_zero_mocks.h | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 0b9a8b9d15..67f20ecd28 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -25,12 +25,10 @@ if(UMF_BUILD_LEVEL_ZERO_PROVIDER) add_library(umf_test_mocks STATIC level_zero_mocks.cpp) target_link_libraries(umf_test_mocks GTest::gmock umf_utils) target_include_directories( - umf_test_mocks - PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src - ${UMF_CMAKE_SOURCE_DIR}/src/utils ${LEVEL_ZERO_INCLUDE_DIRS}) + umf_test_mocks PUBLIC ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_SRC_DIR} + ${UMF_UTILS_DIR} ${LEVEL_ZERO_INCLUDE_DIRS}) add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) - target_include_directories( - umf_ze_loopback PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS} ${UMF_UTILS_DIR} - ${UMF_SRC_DIR} ${UMF_CMAKE_SOURCE_DIR}/include) + target_include_directories(umf_ze_loopback + PUBLIC ${LEVEL_ZERO_INCLUDE_DIRS}) endif() diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h index eeaecc4c36..94fef6c538 100644 --- a/test/common/level_zero_mocks.h +++ b/test/common/level_zero_mocks.h @@ -10,7 +10,7 @@ #ifndef UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H #define UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H -#include "utils/utils_log.h" +#include "utils_log.h" #include "ze_loopback.h" #include #include From 6b0e07022c5717bc7b0ea24cd9c466a0594fdf35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Sat, 27 Sep 2025 16:42:51 +0200 Subject: [PATCH 34/49] maybe UMF_UTIL_OPEN_LIBRARY_NO_LOAD was wrong? --- test/common/level_zero_mocks.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index f5a5e59443..7b73c833f2 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -101,8 +101,7 @@ void MockedLevelZeroTestEnvironment::SetUp() { setenv("UMF_ZE_LOADER_LIB_NAME", lib_name, 1); #endif - void *lib_handle = - utils_open_library(lib_name, UMF_UTIL_OPEN_LIBRARY_NO_LOAD); + void *lib_handle = utils_open_library(lib_name, 0); ASSERT_NE(lib_handle, nullptr); l0interface = static_cast( From 0498da7e6c79e2106a30dff90ccab19329736158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 09:22:36 +0200 Subject: [PATCH 35/49] ze_dll or ze_api export missing? --- test/common/ze_loopback.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp index 3ca4cd4c73..5346640e59 100644 --- a/test/common/ze_loopback.cpp +++ b/test/common/ze_loopback.cpp @@ -12,7 +12,7 @@ #include #include -LevelZero *level_zero_mock = nullptr; +ZE_APIEXPORT ZE_DLLEXPORT LevelZero *level_zero_mock = nullptr; static void check_mock_present() { if (level_zero_mock == nullptr) { From e9ccf45813d25f6cac3dace4c2aa49050ee3f914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 10:19:26 +0200 Subject: [PATCH 36/49] dll path fix --- test/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b3c2ecd703..59574cb523 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -149,8 +149,11 @@ function(add_umf_test) if(WINDOWS) # add PATH to DLL on Windows set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${CMAKE_BINARY_DIR}/bin/;PATH=path_list_append:${CMAKE_BINARY_DIR}/bin/$/" - ) + "${DLL_PATH_LIST};" + "PATH=path_list_append:${CMAKE_BINARY_DIR}/bin/;" + "PATH=path_list_append:${CMAKE_BINARY_DIR}/bin/$/;" + "PATH=path_list_append:${CMAKE_BINARY_DIR}/test/common/;" + "PATH=path_list_append:${CMAKE_BINARY_DIR}/test/common/$/") # append PATH to DLLs NOTE: this would work only for the CMake ver >= # # 3.22. For the older versions, the PATH variable should be set in the From d30da266e342c3178ff64347549cf2f7b01c1e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 10:40:10 +0200 Subject: [PATCH 37/49] back with UMF_UTIL_OPEN_LIBRARY_NO_LOAD? --- test/common/level_zero_mocks.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index 7b73c833f2..b0d78991a8 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -101,12 +101,17 @@ void MockedLevelZeroTestEnvironment::SetUp() { setenv("UMF_ZE_LOADER_LIB_NAME", lib_name, 1); #endif - void *lib_handle = utils_open_library(lib_name, 0); + void *lib_handle = + utils_open_library(lib_name, UMF_UTIL_OPEN_LIBRARY_NO_LOAD); ASSERT_NE(lib_handle, nullptr); - l0interface = static_cast( - utils_get_symbol_addr(lib_handle, "level_zero_mock", lib_name)); + void *l0interface_sym = + utils_get_symbol_addr(lib_handle, "level_zero_mock", lib_name); + ASSERT_NE(l0interface_sym, nullptr); + + l0interface = static_cast(l0interface_sym); ASSERT_NE(l0interface, nullptr); + ASSERT_EQ(*l0interface, nullptr); } void MockedLevelZeroTestEnvironment::TearDown() {} From 29a706dce837e39de3a2592c68362b4c36db6d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 11:32:21 +0200 Subject: [PATCH 38/49] add_library changed to add_umf_library --- test/common/CMakeLists.txt | 8 ++++++- test/common/level_zero_mocks.cpp | 3 +-- test/common/ze_loopback.def | 37 ++++++++++++++++++++++++++++++ test/common/ze_loopback.map | 39 ++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 test/common/ze_loopback.def create mode 100644 test/common/ze_loopback.map diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 67f20ecd28..bbe45446d9 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -28,7 +28,13 @@ if(UMF_BUILD_LEVEL_ZERO_PROVIDER) umf_test_mocks PUBLIC ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_SRC_DIR} ${UMF_UTILS_DIR} ${LEVEL_ZERO_INCLUDE_DIRS}) - add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) + add_umf_library( + NAME umf_ze_loopback + TYPE SHARED + SRCS ze_loopback.h ze_loopback.cpp + LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/ze_loopback.map + WINDOWS_DEF_FILE ${CMAKE_CURRENT_SOURCE_DIR}/ze_loopback.def) + target_include_directories(umf_ze_loopback PUBLIC ${LEVEL_ZERO_INCLUDE_DIRS}) endif() diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index b0d78991a8..37a353354e 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -101,8 +101,7 @@ void MockedLevelZeroTestEnvironment::SetUp() { setenv("UMF_ZE_LOADER_LIB_NAME", lib_name, 1); #endif - void *lib_handle = - utils_open_library(lib_name, UMF_UTIL_OPEN_LIBRARY_NO_LOAD); + void *lib_handle = utils_open_library(lib_name, 0); ASSERT_NE(lib_handle, nullptr); void *l0interface_sym = diff --git a/test/common/ze_loopback.def b/test/common/ze_loopback.def new file mode 100644 index 0000000000..0b13bab8c1 --- /dev/null +++ b/test/common/ze_loopback.def @@ -0,0 +1,37 @@ +;;;; Begin Copyright Notice +; Copyright (C) 2025 Intel Corporation +; Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +;;;; End Copyright Notice + +LIBRARY UMF_ZE_LOOPBACK + +EXPORTS + level_zero_mock + zeInit + zeDriverGet + zeDeviceGet + zeDeviceGetProperties + zeContextCreate + zeContextDestroy + zeCommandQueueCreate + zeCommandQueueDestroy + zeCommandQueueExecuteCommandLists + zeCommandQueueSynchronize + zeCommandListCreate + zeCommandListDestroy + zeCommandListClose + zeCommandListAppendMemoryCopy + zeCommandListAppendMemoryFill + zeContextMakeMemoryResident + zeMemGetAllocProperties + zeMemAllocDevice + zeMemAllocHost + zeMemAllocShared + zeMemFree + zeMemFreeExt + zeMemGetIpcHandle + zeMemPutIpcHandle + zeMemOpenIpcHandle + zeMemCloseIpcHandle + zeDeviceGetMemoryProperties diff --git a/test/common/ze_loopback.map b/test/common/ze_loopback.map new file mode 100644 index 0000000000..08782782f7 --- /dev/null +++ b/test/common/ze_loopback.map @@ -0,0 +1,39 @@ +# Copyright (C) 2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# These functions are meant to be in unnamed scope. They are also not named +# with any umf prefix, as they should override functions with the same name. +{ + global: + level_zero_mock; + zeInit; + zeDriverGet; + zeDeviceGet; + zeDeviceGetProperties; + zeContextCreate; + zeContextDestroy; + zeCommandQueueCreate; + zeCommandQueueDestroy; + zeCommandQueueExecuteCommandLists; + zeCommandQueueSynchronize; + zeCommandListCreate; + zeCommandListDestroy; + zeCommandListClose; + zeCommandListAppendMemoryCopy; + zeCommandListAppendMemoryFill; + zeContextMakeMemoryResident; + zeMemGetAllocProperties; + zeMemAllocDevice; + zeMemAllocHost; + zeMemAllocShared; + zeMemFree; + zeMemFreeExt; + zeMemGetIpcHandle; + zeMemPutIpcHandle; + zeMemOpenIpcHandle; + zeMemCloseIpcHandle; + zeDeviceGetMemoryProperties; + local: + *; +}; From 5713a966be1ed371616805db2c980d3e288b7299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 11:57:05 +0200 Subject: [PATCH 39/49] fix dllexport used more than once --- test/common/ze_loopback.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp index 5346640e59..c1272e474e 100644 --- a/test/common/ze_loopback.cpp +++ b/test/common/ze_loopback.cpp @@ -12,7 +12,7 @@ #include #include -ZE_APIEXPORT ZE_DLLEXPORT LevelZero *level_zero_mock = nullptr; +ZE_APIEXPORT LevelZero *level_zero_mock = nullptr; static void check_mock_present() { if (level_zero_mock == nullptr) { From 134e16b547a1a297db21d5bd1e4bd49fb3b05263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 13:14:26 +0200 Subject: [PATCH 40/49] see logs in case of residency tests failure --- test/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 59574cb523..acfc9db373 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -439,7 +439,8 @@ if(UMF_BUILD_LEVEL_ZERO_PROVIDER) NAME provider_level_zero_residency SRCS providers/provider_level_zero_residency.cpp ${UMF_UTILS_DIR}/utils_level_zero.cpp - LIBS ${UMF_UTILS_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock) + LIBS ${UMF_UTILS_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock + ENVS "UMF_LOG=level:debug\\\\\\\\;flush:debug\\\\\\\\;output:stdout") add_umf_test( NAME pool_residency SRCS pools/pool_residency.cpp ${UMF_UTILS_DIR}/utils_level_zero.cpp @@ -447,7 +448,8 @@ if(UMF_BUILD_LEVEL_ZERO_PROVIDER) ${UMF_BA_FOR_TEST} umf_ze_loopback umf_test_mocks - GTest::gmock) + GTest::gmock + ENVS "UMF_LOG=level:debug\\\\\\\\;flush:debug\\\\\\\\;output:stdout") endif() if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) From 5adb37cbece40708b3a042bb17e567fd4f2fe01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 14:00:53 +0200 Subject: [PATCH 41/49] utils_rwlock_init fix --- src/provider/provider_level_zero.c | 2 +- src/utils/utils_concurrency.h | 2 +- src/utils/utils_posix_concurrency.c | 5 ++--- src/utils/utils_windows_concurrency.c | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 126df53478..88994665c8 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -657,7 +657,7 @@ static umf_result_t ze_memory_provider_initialize(const void *params, } } - if (utils_rwlock_init(&ze_provider->resident_device_rwlock) == NULL) { + if (utils_rwlock_init(&ze_provider->resident_device_rwlock) != 0) { LOG_ERR("Cannot initialize resident device rwlock"); umf_ba_global_free(ze_provider); return UMF_RESULT_ERROR_OUT_OF_RESOURCES; diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index 922f992e83..74c7fc8ae1 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -71,7 +71,7 @@ typedef struct utils_rwlock_t { #endif } utils_rwlock_t; -utils_rwlock_t *utils_rwlock_init(utils_rwlock_t *ptr); +int utils_rwlock_init(utils_rwlock_t *ptr); void utils_rwlock_destroy_not_free(utils_rwlock_t *rwlock); void utils_read_lock(utils_rwlock_t *rwlock); void utils_write_lock(utils_rwlock_t *rwlock); diff --git a/src/utils/utils_posix_concurrency.c b/src/utils/utils_posix_concurrency.c index 6cbcd908ec..0149263df3 100644 --- a/src/utils/utils_posix_concurrency.c +++ b/src/utils/utils_posix_concurrency.c @@ -46,10 +46,9 @@ void utils_init_once(UTIL_ONCE_FLAG *flag, void (*oneCb)(void)) { pthread_once(flag, oneCb); } -utils_rwlock_t *utils_rwlock_init(utils_rwlock_t *ptr) { +int utils_rwlock_init(utils_rwlock_t *ptr) { pthread_rwlock_t *rwlock = (pthread_rwlock_t *)ptr; - int ret = pthread_rwlock_init(rwlock, NULL); - return ret == 0 ? ((utils_rwlock_t *)rwlock) : NULL; + return pthread_rwlock_init(rwlock, NULL); } void utils_rwlock_destroy_not_free(utils_rwlock_t *ptr) { diff --git a/src/utils/utils_windows_concurrency.c b/src/utils/utils_windows_concurrency.c index 4e2f9c3046..645d87b106 100644 --- a/src/utils/utils_windows_concurrency.c +++ b/src/utils/utils_windows_concurrency.c @@ -36,7 +36,7 @@ int utils_mutex_unlock(utils_mutex_t *mutex) { return 0; } -utils_rwlock_t *utils_rwlock_init(utils_rwlock_t *rwlock) { +int utils_rwlock_init(utils_rwlock_t *rwlock) { InitializeSRWLock(&rwlock->lock); return 0; // never fails } From 058bf8d3f4d561b001e5fe5fb41b8c79f3baba6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 14:39:20 +0200 Subject: [PATCH 42/49] some logs upon free and add closing library --- src/provider/provider_level_zero.c | 8 +++++--- test/common/level_zero_mocks.cpp | 6 ++++-- test/common/level_zero_mocks.h | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 88994665c8..9d99004b99 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -425,12 +425,14 @@ static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, size_t bytes, int update_stats) { if (ptr == NULL) { + LOG_DEBUG("Free skipped as ptr is NULL"); return UMF_RESULT_SUCCESS; } - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; + ze_memory_provider_t *ze_provider = provider; umf_result_t ret; if (ze_provider->freePolicyFlags == 0) { + LOG_DEBUG("No free policy set, caling zeMemFree with %p", ptr); ret = ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); } else { ze_memory_free_ext_desc_t desc = { @@ -438,6 +440,7 @@ static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, .pNext = NULL, .freePolicy = ze_provider->freePolicyFlags}; + LOG_DEBUG("Free policy was set, caling zeMemFreeExt with %p", ptr); ret = ze2umf_result( g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); } @@ -559,8 +562,6 @@ static umf_result_t query_min_page_size(ze_memory_provider_t *ze_provider, size_t *min_page_size) { assert(min_page_size); - LOG_DEBUG("Querying minimum page size"); - void *ptr; umf_result_t result = ze_memory_provider_alloc_helper(ze_provider, 1, 0, 0, &ptr); @@ -574,6 +575,7 @@ static umf_result_t query_min_page_size(ze_memory_provider_t *ze_provider, ze_provider->context, ptr, &properties, NULL); *min_page_size = properties.pageSize; + LOG_DEBUG("Querying minimum page size, got:%lu", properties.pageSize); ze_memory_provider_free_helper(ze_provider, ptr, 1, 0); diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index 37a353354e..dc3dfdc76b 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -101,7 +101,7 @@ void MockedLevelZeroTestEnvironment::SetUp() { setenv("UMF_ZE_LOADER_LIB_NAME", lib_name, 1); #endif - void *lib_handle = utils_open_library(lib_name, 0); + lib_handle = utils_open_library(lib_name, 0); ASSERT_NE(lib_handle, nullptr); void *l0interface_sym = @@ -113,6 +113,8 @@ void MockedLevelZeroTestEnvironment::SetUp() { ASSERT_EQ(*l0interface, nullptr); } -void MockedLevelZeroTestEnvironment::TearDown() {} +void MockedLevelZeroTestEnvironment::TearDown() { + utils_close_library(lib_handle); +} LevelZero **MockedLevelZeroTestEnvironment::l0interface; \ No newline at end of file diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h index 94fef6c538..33be40c433 100644 --- a/test/common/level_zero_mocks.h +++ b/test/common/level_zero_mocks.h @@ -79,6 +79,9 @@ class LevelZeroMock : public LevelZero { // important, makes UMF load ze_loopback instead of regular l0 class MockedLevelZeroTestEnvironment : public ::testing::Environment { + + void *lib_handle; + public: static LevelZero **l0interface; From 4803c4796e3b43c47281ec1ac8284a3809b847b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 14:52:02 +0200 Subject: [PATCH 43/49] typo fix --- src/provider/provider_level_zero.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 9d99004b99..e4c3438c8f 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -432,7 +432,7 @@ static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, ze_memory_provider_t *ze_provider = provider; umf_result_t ret; if (ze_provider->freePolicyFlags == 0) { - LOG_DEBUG("No free policy set, caling zeMemFree with %p", ptr); + LOG_DEBUG("No free policy set, calling zeMemFree with %p", ptr); ret = ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); } else { ze_memory_free_ext_desc_t desc = { @@ -440,7 +440,7 @@ static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, .pNext = NULL, .freePolicy = ze_provider->freePolicyFlags}; - LOG_DEBUG("Free policy was set, caling zeMemFreeExt with %p", ptr); + LOG_DEBUG("Free policy was set, calling zeMemFreeExt with %p", ptr); ret = ze2umf_result( g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); } From a3cc32ebae11c98623ddf39cd9823c81ebee2eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 15:47:26 +0200 Subject: [PATCH 44/49] fix LevelZeroMock::initializeMemoryProviderWithResidentDevices with POINTER_TESTING_PROPS --- test/common/level_zero_mocks.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index dc3dfdc76b..8bcdd619c0 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -39,8 +39,11 @@ LevelZeroMock::initializeMemoryProviderWithResidentDevices( EXPECT_CALL(*this, zeDeviceGetProperties(device, _)) .WillRepeatedly(DoAll(SetArgPointee<1>(device_properties), Return(ZE_RESULT_SUCCESS))); + + void *POINTER_TESTING_PROPS = TestCreatePointer(0x77); EXPECT_CALL(*this, zeMemAllocDevice(CONTEXT, _, _, _, device, _)) - .WillOnce(Return(ZE_RESULT_SUCCESS)); + .WillOnce(DoAll(SetArgPointee<5>(POINTER_TESTING_PROPS), + Return(ZE_RESULT_SUCCESS))); for (auto dev : residentDevices) { EXPECT_CALL(*this, zeContextMakeMemoryResident(context, dev, _, _)) .WillOnce(Return(ZE_RESULT_SUCCESS)); @@ -48,7 +51,7 @@ LevelZeroMock::initializeMemoryProviderWithResidentDevices( EXPECT_CALL(*this, zeMemGetAllocProperties(context, _, _, _)) .WillOnce(DoAll(SetArgPointee<2>(memory_allocation_properties), Return(ZE_RESULT_SUCCESS))); - EXPECT_CALL(*this, zeMemFree(CONTEXT, _)) + EXPECT_CALL(*this, zeMemFree(CONTEXT, POINTER_TESTING_PROPS)) .WillOnce(Return(ZE_RESULT_SUCCESS)); umf_memory_provider_handle_t provider = nullptr; From 5b7f61a4affb0c1a636729e4ddb88aae80345649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Mon, 29 Sep 2025 15:49:12 +0200 Subject: [PATCH 45/49] remove not needed free logs --- src/provider/provider_level_zero.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index e4c3438c8f..64171c059e 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -425,14 +425,12 @@ static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, size_t bytes, int update_stats) { if (ptr == NULL) { - LOG_DEBUG("Free skipped as ptr is NULL"); return UMF_RESULT_SUCCESS; } ze_memory_provider_t *ze_provider = provider; umf_result_t ret; if (ze_provider->freePolicyFlags == 0) { - LOG_DEBUG("No free policy set, calling zeMemFree with %p", ptr); ret = ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); } else { ze_memory_free_ext_desc_t desc = { @@ -440,7 +438,6 @@ static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, .pNext = NULL, .freePolicy = ze_provider->freePolicyFlags}; - LOG_DEBUG("Free policy was set, calling zeMemFreeExt with %p", ptr); ret = ze2umf_result( g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); } From 0bf7cbdbd7feae0c05e1d13b250c95c7758bf85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Tue, 30 Sep 2025 11:03:26 +0200 Subject: [PATCH 46/49] UBA does not accept NULL in memcpy source even with size==0 --- src/provider/provider_level_zero.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 64171c059e..698550ae8e 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -1071,9 +1071,12 @@ umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( } LOG_DEBUG("enlarging resident devices array from %u to %u", ze_provider->resident_device_capacity, new_capacity); - memcpy(new_handles, ze_provider->resident_device_handles, - sizeof(ze_device_handle_t) * - ze_provider->resident_device_count); + if (ze_provider->resident_device_count > 0) { + ASSERT(ze_provider->resident_device_handles != NULL); + memcpy(new_handles, ze_provider->resident_device_handles, + sizeof(ze_device_handle_t) * + ze_provider->resident_device_count); + } umf_ba_global_free(ze_provider->resident_device_handles); ze_provider->resident_device_handles = new_handles; ze_provider->resident_device_capacity = new_capacity; From e0af7eb6538ab38fd0a0f37cded0e2b4d733722f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Tue, 30 Sep 2025 09:56:33 +0200 Subject: [PATCH 47/49] fix wrong type in zeMemAlloc* functions --- src/provider/provider_level_zero.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 698550ae8e..4d5c1ec901 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -88,14 +88,14 @@ typedef struct ze_memory_provider_t { typedef struct ze_ops_t { ze_result_t (*zeMemAllocHost)(ze_context_handle_t, const ze_host_mem_alloc_desc_t *, size_t, - size_t, void *); + size_t, void **); ze_result_t (*zeMemAllocDevice)(ze_context_handle_t, const ze_device_mem_alloc_desc_t *, size_t, - size_t, ze_device_handle_t, void *); + size_t, ze_device_handle_t, void **); ze_result_t (*zeMemAllocShared)(ze_context_handle_t, const ze_device_mem_alloc_desc_t *, const ze_host_mem_alloc_desc_t *, size_t, - size_t, ze_device_handle_t, void *); + size_t, ze_device_handle_t, void **); ze_result_t (*zeMemFree)(ze_context_handle_t, void *); ze_result_t (*zeMemGetIpcHandle)(ze_context_handle_t, const void *, ze_ipc_mem_handle_t *); @@ -462,7 +462,7 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, size_t alignment, int update_stats, void **resultPtr) { - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; + ze_memory_provider_t *ze_provider = provider; ze_result_t ze_result = ZE_RESULT_SUCCESS; switch (ze2umf_memory_type(ze_provider->memory_type)) { @@ -714,7 +714,7 @@ ze_memory_provider_get_last_native_error(void *provider, const char **ppMessage, static umf_result_t ze_memory_provider_get_min_page_size(void *provider, const void *ptr, size_t *pageSize) { - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; + ze_memory_provider_t *ze_provider = provider; if (!ptr) { *pageSize = ze_provider->min_page_size; @@ -770,7 +770,7 @@ static umf_result_t ze_memory_provider_get_name(void *provider, *name = DEFAULT_NAME; return UMF_RESULT_SUCCESS; } - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; + ze_memory_provider_t *ze_provider = provider; *name = ze_provider->name; return UMF_RESULT_SUCCESS; } From 57acba760b3e2257e9d304ea6feab33cdf03feb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Wed, 1 Oct 2025 09:02:59 +0200 Subject: [PATCH 48/49] simplify cmake and UMF_ZE_LOADER_LIB_NAME use --- src/provider/provider_level_zero.c | 3 +++ test/CMakeLists.txt | 38 ++++++++++++++++-------------- test/common/level_zero_mocks.cpp | 11 ++++----- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 4d5c1ec901..d1dfb77ac2 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -513,6 +513,9 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, } if (ze_result != ZE_RESULT_SUCCESS) { + LOG_ERR("failed to allocate memory, type:%d, size:%lu, alignment:%lu, " + "result:%d", + ze_provider->memory_type, size, alignment, ze_result); return ze2umf_result(ze_result); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index acfc9db373..2d45136ea4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -151,9 +151,7 @@ function(add_umf_test) set(DLL_PATH_LIST "${DLL_PATH_LIST};" "PATH=path_list_append:${CMAKE_BINARY_DIR}/bin/;" - "PATH=path_list_append:${CMAKE_BINARY_DIR}/bin/$/;" - "PATH=path_list_append:${CMAKE_BINARY_DIR}/test/common/;" - "PATH=path_list_append:${CMAKE_BINARY_DIR}/test/common/$/") + "PATH=path_list_append:${CMAKE_BINARY_DIR}/bin/$/") # append PATH to DLLs NOTE: this would work only for the CMake ver >= # # 3.22. For the older versions, the PATH variable should be set in the @@ -163,6 +161,22 @@ function(add_umf_test) endif() endfunction() +function(add_umf_mocked_test test_name test_source_file) + add_umf_test( + NAME ${test_name} + SRCS ${test_source_file} ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} umf_test_mocks + GTest::gmock) + + set_tests_properties( + test_${test_name} PROPERTIES + ENVIRONMENT + "UMF_LOG=level:debug\\\\\\\\;flush:debug\\\\\\\\;output:stdout;UMF_ZE_LOADER_LIB_NAME=$" + ) + target_compile_definitions(test_${test_name} PUBLIC USE_DLOPEN=1) + add_dependencies(test_${test_name} umf_ze_loopback) +endfunction() + add_subdirectory(common) if(UMF_BUILD_SHARED_LIBRARY) @@ -435,21 +449,9 @@ else() endif() if(UMF_BUILD_LEVEL_ZERO_PROVIDER) - add_umf_test( - NAME provider_level_zero_residency - SRCS providers/provider_level_zero_residency.cpp - ${UMF_UTILS_DIR}/utils_level_zero.cpp - LIBS ${UMF_UTILS_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock - ENVS "UMF_LOG=level:debug\\\\\\\\;flush:debug\\\\\\\\;output:stdout") - add_umf_test( - NAME pool_residency - SRCS pools/pool_residency.cpp ${UMF_UTILS_DIR}/utils_level_zero.cpp - LIBS ${UMF_UTILS_FOR_TEST} - ${UMF_BA_FOR_TEST} - umf_ze_loopback - umf_test_mocks - GTest::gmock - ENVS "UMF_LOG=level:debug\\\\\\\\;flush:debug\\\\\\\\;output:stdout") + add_umf_mocked_test(provider_level_zero_residency + providers/provider_level_zero_residency.cpp) + add_umf_mocked_test(pool_residency pools/pool_residency.cpp) endif() if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp index 8bcdd619c0..0bb69dd0f3 100644 --- a/test/common/level_zero_mocks.cpp +++ b/test/common/level_zero_mocks.cpp @@ -96,13 +96,9 @@ TestCreateMemoryAllocationProperties(uint32_t modifier) { } void MockedLevelZeroTestEnvironment::SetUp() { -#ifdef _WIN32 - const char *lib_name = "umf_ze_loopback.dll"; - _putenv_s("UMF_ZE_LOADER_LIB_NAME", lib_name); -#else - const char *lib_name = "libumf_ze_loopback.so"; - setenv("UMF_ZE_LOADER_LIB_NAME", lib_name, 1); -#endif + const char *lib_name = getenv("UMF_ZE_LOADER_LIB_NAME"); + ASSERT_NE(lib_name, nullptr); + ASSERT_NE(lib_name[0], '\0'); lib_handle = utils_open_library(lib_name, 0); ASSERT_NE(lib_handle, nullptr); @@ -116,6 +112,7 @@ void MockedLevelZeroTestEnvironment::SetUp() { ASSERT_EQ(*l0interface, nullptr); } + void MockedLevelZeroTestEnvironment::TearDown() { utils_close_library(lib_handle); } From 56c9e9af3b87d1ece21e2abe16d14cf4c9eae276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Wed, 1 Oct 2025 09:39:53 +0200 Subject: [PATCH 49/49] skip mocked tests in valgrind for now --- test/test_valgrind.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index c877c4f440..fa2610dd37 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -143,9 +143,17 @@ for test in $TESTS; do ./test/test_memspace_highest_capacity) FILTER='--gtest_filter="-*highestCapacityVerify*"' ;; + ./test/test_pool_residency) + echo "- SKIPPED" + continue; # TODO: set UMF_ZE_LOADER_LIB_NAME as ctest does + ;; ./test/test_provider_os_memory_multiple_numa_nodes) FILTER='--gtest_filter="-testNuma.checkModeInterleave*:testNumaNodesAllocations/testNumaOnEachNode.checkNumaNodesAllocations*:testNumaNodesAllocations/testNumaOnEachNode.checkModePreferred*:testNumaNodesAllocations/testNumaOnEachNode.checkModeInterleaveSingleNode*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModePreferredEmptyNodeset*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModeLocal*"' ;; + ./test/test_provider_level_zero_residency) + echo "- SKIPPED" + continue; # TODO: set UMF_ZE_LOADER_LIB_NAME as ctest does + ;; ./test/test_memspace_highest_bandwidth) FILTER='--gtest_filter="-*allocLocalMt*"' ;;