diff --git a/unified-runtime/include/ur_api.h b/unified-runtime/include/ur_api.h index 2f3a26a9b988e..567f350aef8ad 100644 --- a/unified-runtime/include/ur_api.h +++ b/unified-runtime/include/ur_api.h @@ -7331,6 +7331,8 @@ typedef enum ur_command_t { UR_COMMAND_READ_HOST_PIPE = 25, /// Event created by ::urEnqueueWriteHostPipe UR_COMMAND_WRITE_HOST_PIPE = 26, + /// Command type used by events created by ::urEventCreateWithNativeHandle + UR_COMMAND_FROM_NATIVE = 27, /// Event created by ::urEnqueueCommandBufferExp UR_COMMAND_ENQUEUE_COMMAND_BUFFER_EXP = 0x1000, /// Event created by ::urBindlessImagesWaitExternalSemaphoreExp @@ -7377,7 +7379,10 @@ typedef enum ur_event_status_t { /////////////////////////////////////////////////////////////////////////////// /// @brief Event query information type typedef enum ur_event_info_t { - /// [::ur_queue_handle_t] Command queue information of an event object + /// [::ur_queue_handle_t] Command queue information of an event object. + /// This may be NULL if the event was created via + /// ::urEventCreateWithNativeHandle and the adapter has no associated UR + /// queue UR_EVENT_INFO_COMMAND_QUEUE = 0, /// [::ur_context_handle_t] Context information of an event object UR_EVENT_INFO_CONTEXT = 1, diff --git a/unified-runtime/include/ur_print.hpp b/unified-runtime/include/ur_print.hpp index 62f653a640e08..5e656795a00b2 100644 --- a/unified-runtime/include/ur_print.hpp +++ b/unified-runtime/include/ur_print.hpp @@ -10569,6 +10569,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_command_t value) { case UR_COMMAND_WRITE_HOST_PIPE: os << "UR_COMMAND_WRITE_HOST_PIPE"; break; + case UR_COMMAND_FROM_NATIVE: + os << "UR_COMMAND_FROM_NATIVE"; + break; case UR_COMMAND_ENQUEUE_COMMAND_BUFFER_EXP: os << "UR_COMMAND_ENQUEUE_COMMAND_BUFFER_EXP"; break; diff --git a/unified-runtime/scripts/core/event.yml b/unified-runtime/scripts/core/event.yml index 1b0eeca23e9de..9d3b81a9aeb2a 100644 --- a/unified-runtime/scripts/core/event.yml +++ b/unified-runtime/scripts/core/event.yml @@ -77,6 +77,8 @@ etors: desc: Event created by $xEnqueueReadHostPipe - name: WRITE_HOST_PIPE desc: Event created by $xEnqueueWriteHostPipe + - name: FROM_NATIVE + desc: Command type used by events created by $xEventCreateWithNativeHandle --- #-------------------------------------------------------------------------- type: enum desc: "Event Status" @@ -101,7 +103,7 @@ name: $x_event_info_t typed_etors: True etors: - name: COMMAND_QUEUE - desc: "[$x_queue_handle_t] Command queue information of an event object" + desc: "[$x_queue_handle_t] Command queue information of an event object. This may be NULL if the event was created via $xEventCreateWithNativeHandle and the adapter has no associated UR queue" - name: CONTEXT desc: "[$x_context_handle_t] Context information of an event object" - name: COMMAND_TYPE diff --git a/unified-runtime/source/adapters/cuda/event.cpp b/unified-runtime/source/adapters/cuda/event.cpp index 46ac116bab322..0007b7ee959ba 100644 --- a/unified-runtime/source/adapters/cuda/event.cpp +++ b/unified-runtime/source/adapters/cuda/event.cpp @@ -42,7 +42,7 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type, ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t Context, CUevent EventNative) - : handle_base(), CommandType{UR_COMMAND_EVENTS_WAIT}, HasOwnership{false}, + : handle_base(), CommandType{UR_COMMAND_FROM_NATIVE}, HasOwnership{false}, IsInterop{true}, StreamToken{std::numeric_limits::max()}, EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr}, Stream{nullptr}, Context{Context} { @@ -168,11 +168,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent, // If the runtime owns the native handle, we have reference to the queue. // Otherwise, the event handle comes from an interop API with no RT refs. if (!hEvent->getQueue()) { - setErrorMessage("Command queue info cannot be queried for the event. The " - "event object was created from a native event and has no " - "valid reference to a command queue.", - UR_RESULT_ERROR_INVALID_VALUE); - return UR_RESULT_ERROR_ADAPTER_SPECIFIC; + return ReturnValue(nullptr); } return ReturnValue(hEvent->getQueue()); } diff --git a/unified-runtime/source/adapters/hip/event.cpp b/unified-runtime/source/adapters/hip/event.cpp index b6a4070b8eca7..153724fefc5ce 100644 --- a/unified-runtime/source/adapters/hip/event.cpp +++ b/unified-runtime/source/adapters/hip/event.cpp @@ -35,7 +35,7 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type, ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t Context, hipEvent_t EventNative) - : handle_base(), CommandType{UR_COMMAND_EVENTS_WAIT}, HasOwnership{false}, + : handle_base(), CommandType{UR_COMMAND_FROM_NATIVE}, HasOwnership{false}, IsInterop{true}, StreamToken{std::numeric_limits::max()}, EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr}, Stream{nullptr}, Context{Context} { @@ -178,11 +178,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent, // If the runtime owns the native handle, we have reference to the queue. // Otherwise, the event handle comes from an interop API with no RT refs. if (!hEvent->getQueue()) { - setErrorMessage("Command queue info cannot be queried for the event. The " - "event object was created from a native event and has no " - "valid reference to a command queue.", - UR_RESULT_ERROR_INVALID_VALUE); - return UR_RESULT_ERROR_ADAPTER_SPECIFIC; + return ReturnValue(nullptr); } return ReturnValue(hEvent->getQueue()); } diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index 7e71058dda7d3..adf794fba71a5 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -972,9 +972,9 @@ ur_result_t urEventCreateWithNativeHandle( auto ZeEvent = ur_cast(NativeEvent); ur_event_handle_t_ *UREvent{}; try { - UREvent = new ur_event_handle_t_(ZeEvent, nullptr /* ZeEventPool */, - Context, UR_EXT_COMMAND_TYPE_USER, - Properties->isNativeHandleOwned); + UREvent = new ur_event_handle_t_( + ZeEvent, nullptr /* ZeEventPool */, Context, UR_COMMAND_FROM_NATIVE, + Properties ? Properties->isNativeHandleOwned : false); UREvent->RefCountExternal++; } catch (const std::bad_alloc &) { diff --git a/unified-runtime/source/adapters/offload/event.cpp b/unified-runtime/source/adapters/offload/event.cpp index f217fd5321034..a432785063738 100644 --- a/unified-runtime/source/adapters/offload/event.cpp +++ b/unified-runtime/source/adapters/offload/event.cpp @@ -24,7 +24,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent, switch (propName) { case UR_EVENT_INFO_CONTEXT: - return ReturnValue(hEvent->UrQueue->UrContext); + return ReturnValue(hEvent->UrContext); case UR_EVENT_INFO_COMMAND_QUEUE: return ReturnValue(hEvent->UrQueue); case UR_EVENT_INFO_COMMAND_TYPE: @@ -103,7 +103,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventRetain(ur_event_handle_t hEvent) { UR_APIEXPORT ur_result_t UR_APICALL urEventRelease(ur_event_handle_t hEvent) { if (--hEvent->RefCount == 0) { - if (hEvent->OffloadEvent) { + if (hEvent->OffloadEvent && hEvent->isNativeHandleOwned) { auto Res = olDestroyEvent(hEvent->OffloadEvent); if (Res) { return offloadResultToUR(Res); @@ -115,13 +115,32 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventRelease(ur_event_handle_t hEvent) { return UR_RESULT_SUCCESS; } -UR_APIEXPORT ur_result_t UR_APICALL -urEventGetNativeHandle(ur_event_handle_t, ur_native_handle_t *) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +UR_APIEXPORT ur_result_t UR_APICALL urEventGetNativeHandle( + ur_event_handle_t hEvent, ur_native_handle_t *phNativeEvent) { + if (!hEvent->OffloadEvent) { + // There is no associated event - rather than returning nullptr, create an + // empty one + ol_queue_handle_t TmpQueue; + OL_RETURN_ON_ERR(olCreateQueue(hEvent->UrQueue->OffloadDevice, &TmpQueue)); + OL_RETURN_ON_ERR(olCreateEvent( + TmpQueue, reinterpret_cast(phNativeEvent))); + OL_RETURN_ON_ERR(olDestroyQueue(TmpQueue)); + } else { + *phNativeEvent = reinterpret_cast(hEvent->OffloadEvent); + } + + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urEventCreateWithNativeHandle( - ur_native_handle_t, ur_context_handle_t, - const ur_event_native_properties_t *, ur_event_handle_t *) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + ur_native_handle_t hNativeEvent, ur_context_handle_t hContext, + const ur_event_native_properties_t *pProperties, + ur_event_handle_t *hEvent) { + bool IsNativeHandleOwned = + pProperties ? pProperties->isNativeHandleOwned : false; + + *hEvent = + new ur_event_handle_t_(reinterpret_cast(hNativeEvent), + hContext, IsNativeHandleOwned); + return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/offload/event.hpp b/unified-runtime/source/adapters/offload/event.hpp index 42860e68bed36..1fa4098aba59d 100644 --- a/unified-runtime/source/adapters/offload/event.hpp +++ b/unified-runtime/source/adapters/offload/event.hpp @@ -14,14 +14,23 @@ #include #include "common.hpp" +#include "queue.hpp" struct ur_event_handle_t_ : RefCounted { ol_event_handle_t OffloadEvent; ur_command_t Type; ur_queue_handle_t UrQueue; + ur_context_handle_t UrContext; + bool isNativeHandleOwned; ur_event_handle_t_(ur_command_t Type, ur_queue_handle_t Queue) - : Type(Type), UrQueue(Queue) {} + : Type(Type), UrQueue(Queue), UrContext(Queue->UrContext), + isNativeHandleOwned(true) {} + ur_event_handle_t_(ol_event_handle_t NativeHandle, + ur_context_handle_t Context, bool isNativeHandleOwned) + : OffloadEvent(NativeHandle), Type(UR_COMMAND_FROM_NATIVE), + UrQueue(nullptr), UrContext(Context), + isNativeHandleOwned(isNativeHandleOwned) {} static ur_event_handle_t createEmptyEvent(ur_command_t Type, ur_queue_handle_t Queue) { diff --git a/unified-runtime/source/adapters/offload/queue.cpp b/unified-runtime/source/adapters/offload/queue.cpp index 0d296793651e9..1aeaf97ff2a2f 100644 --- a/unified-runtime/source/adapters/offload/queue.cpp +++ b/unified-runtime/source/adapters/offload/queue.cpp @@ -14,9 +14,29 @@ #include "context.hpp" #include "device.hpp" +#include "event.hpp" #include "queue.hpp" #include "ur2offload.hpp" +ol_result_t ur_queue_handle_t_::nextQueueNoLock(ol_queue_handle_t &Handle) { + auto &Slot = OffloadQueues[(QueueOffset++) % OffloadQueues.size()]; + + if (!Slot) { + if (auto Res = olCreateQueue(OffloadDevice, &Slot)) { + return Res; + } + + if (auto Event = Barrier) { + if (auto Res = olWaitEvents(Slot, &Event->OffloadEvent, 1)) { + return Res; + } + } + } + + Handle = Slot; + return nullptr; +} + UR_APIEXPORT ur_result_t UR_APICALL urQueueCreate( [[maybe_unused]] ur_context_handle_t hContext, ur_device_handle_t hDevice, const ur_queue_properties_t *pProps, ur_queue_handle_t *phQueue) { diff --git a/unified-runtime/source/adapters/offload/queue.hpp b/unified-runtime/source/adapters/offload/queue.hpp index 8561812187e8f..2ece757708a96 100644 --- a/unified-runtime/source/adapters/offload/queue.hpp +++ b/unified-runtime/source/adapters/offload/queue.hpp @@ -14,7 +14,6 @@ #include #include "common.hpp" -#include "event.hpp" constexpr size_t OOO_QUEUE_POOL_SIZE = 32; @@ -66,24 +65,7 @@ struct ur_queue_handle_t_ : RefCounted { return OL_SUCCESS; } - ol_result_t nextQueueNoLock(ol_queue_handle_t &Handle) { - auto &Slot = OffloadQueues[(QueueOffset++) % OffloadQueues.size()]; - - if (!Slot) { - if (auto Res = olCreateQueue(OffloadDevice, &Slot)) { - return Res; - } - - if (auto Event = Barrier) { - if (auto Res = olWaitEvents(Slot, &Event->OffloadEvent, 1)) { - return Res; - } - } - } - - Handle = Slot; - return nullptr; - } + ol_result_t nextQueueNoLock(ol_queue_handle_t &Handle); ol_result_t nextQueue(ol_queue_handle_t &Handle) { std::lock_guard Lock(OooMutex); diff --git a/unified-runtime/test/conformance/event/urEventGetInfo.cpp b/unified-runtime/test/conformance/event/urEventGetInfo.cpp index 134b094821cf4..dd8e9ecd4cfe7 100644 --- a/unified-runtime/test/conformance/event/urEventGetInfo.cpp +++ b/unified-runtime/test/conformance/event/urEventGetInfo.cpp @@ -48,10 +48,6 @@ TEST_P(urEventGetInfoTest, SuccessRoundtripContext) { } TEST_P(urEventGetInfoTest, SuccessRoundtripCommandQueue) { - UUR_KNOWN_FAILURE_ON(uur::HIP{}, uur::CUDA{}); - // Segfaults - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); - const ur_event_info_t property_name = UR_EVENT_INFO_COMMAND_QUEUE; size_t property_size = sizeof(ur_queue_handle_t); @@ -66,6 +62,10 @@ TEST_P(urEventGetInfoTest, SuccessRoundtripCommandQueue) { ur_queue_handle_t property_value = nullptr; ASSERT_SUCCESS(urEventGetInfo(from_native_event, property_name, property_size, &property_value, nullptr)); + if (property_value == nullptr) { + GTEST_SKIP() + << "Backend cannot regenerate command queues from native handles"; + } // We can't assume that the two queue handles are equal (since creating the // link to the UR structures has been severed by going through native handle,