Skip to content

Make TRT plugins optional #25261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 10, 2025
Merged

Make TRT plugins optional #25261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
651f6f0
make TRT plugins optional
gedoensmax Jul 2, 2025
edf5415
fix linux path
gedoensmax Jul 3, 2025
1414bc4
lintrunner
gedoensmax Jul 3, 2025
cb2257c
CUDA pinned memory fixes
gedoensmax Jul 3, 2025
d22e079
revert allocator changes
gedoensmax Jul 7, 2025
4f98ef8
do not limit to binary directory for plugins
gedoensmax Jul 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 5 additions & 18 deletions cmake/onnxruntime_providers_tensorrt.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,26 +72,21 @@
endif()

# TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows,
# for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ...
# for example, nvinfer_10.dll, nvonnxparser_10.dll ...
if (WIN32 AND TRT_GREATER_OR_EQUAL_TRT_10_GA)
set(NVINFER_LIB "nvinfer_${NV_TENSORRT_MAJOR}")
set(NVINFER_PLUGIN_LIB "nvinfer_plugin_${NV_TENSORRT_MAJOR}")
set(PARSER_LIB "nvonnxparser_${NV_TENSORRT_MAJOR}")
endif()

if (NOT NVINFER_LIB)
set(NVINFER_LIB "nvinfer")
endif()

if (NOT NVINFER_PLUGIN_LIB)
set(NVINFER_PLUGIN_LIB "nvinfer_plugin")
endif()

if (NOT PARSER_LIB)
set(PARSER_LIB "nvonnxparser")
endif()

MESSAGE(STATUS "Looking for ${NVINFER_LIB} and ${NVINFER_PLUGIN_LIB}")
MESSAGE(STATUS "Looking for ${NVINFER_LIB}")

find_library(TENSORRT_LIBRARY_INFER ${NVINFER_LIB}
HINTS ${TENSORRT_ROOT}
Expand All @@ -101,14 +96,6 @@
MESSAGE(STATUS "Can't find ${NVINFER_LIB}")
endif()

find_library(TENSORRT_LIBRARY_INFER_PLUGIN ${NVINFER_PLUGIN_LIB}
HINTS ${TENSORRT_ROOT}
PATH_SUFFIXES lib lib64 lib/x64)

if (NOT TENSORRT_LIBRARY_INFER_PLUGIN)
MESSAGE(STATUS "Can't find ${NVINFER_PLUGIN_LIB}")
endif()

if (onnxruntime_USE_TENSORRT_BUILTIN_PARSER)
MESSAGE(STATUS "Looking for ${PARSER_LIB}")

Expand All @@ -120,7 +107,7 @@
MESSAGE(STATUS "Can't find ${PARSER_LIB}")
endif()

set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_NVONNXPARSER})
set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_NVONNXPARSER})
MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}")
else()
if (TRT_GREATER_OR_EQUAL_TRT_10_GA)
Expand Down Expand Up @@ -153,15 +140,15 @@
endif()
# Static libraries are just nvonnxparser_static on all platforms
set(onnxparser_link_libs nvonnxparser_static)
set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN})
set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER})
MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}")
endif()

# ${TENSORRT_LIBRARY} is empty if we link nvonnxparser_static.
# nvonnxparser_static is linked against tensorrt libraries in onnx-tensorrt
# See https://github.com/onnx/onnx-tensorrt/blob/8af13d1b106f58df1e98945a5e7c851ddb5f0791/CMakeLists.txt#L121
# However, starting from TRT 10 GA, nvonnxparser_static doesn't link against tensorrt libraries.
# Therefore, the above code finds ${TENSORRT_LIBRARY_INFER} and ${TENSORRT_LIBRARY_INFER_PLUGIN}.
# Therefore, the above code finds ${TENSORRT_LIBRARY_INFER}.
if(onnxruntime_CUDA_MINIMAL)
set(trt_link_libs ${CMAKE_DL_LIBS} ${TENSORRT_LIBRARY})
else()
Expand Down
2 changes: 1 addition & 1 deletion onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1304,7 +1304,7 @@ std::vector<AllocatorPtr> NvExecutionProvider::CreatePreferredAllocators() {

AllocatorCreationInfo pinned_allocator_info(
[](OrtDevice::DeviceId device_id) {
return std::make_unique<CUDAPinnedAllocator>(device_id, CUDA_PINNED);
return std::make_unique<CUDAPinnedAllocator>(CUDA_PINNED, device_id);
},
narrow<OrtDevice::DeviceId>(device_id_));

Expand Down
44 changes: 43 additions & 1 deletion onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_custom_ops.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,25 @@
#include "tensorrt_execution_provider_custom_ops.h"
#include "tensorrt_execution_provider.h"

// The filename extension for a shared library is different per platform
#ifdef _WIN32
#define LIBRARY_PREFIX
#define LIBRARY_EXTENSION ORT_TSTR(".dll")
#elif defined(__APPLE__)
#define LIBRARY_PREFIX "lib"
#define LIBRARY_EXTENSION ".dylib"
#else
#define LIBRARY_PREFIX "lib"
#define LIBRARY_EXTENSION ".so"
#endif

#ifdef _WIN32
#define ORT_DEF2STR_HELPER(x) L#x
#else
#define ORT_DEF2STR_HELPER(X) #X
#endif
#define ORT_DEF2STR(x) ORT_DEF2STR_HELPER(x)

namespace onnxruntime {
extern TensorrtLogger& GetTensorrtLogger(bool verbose);

Expand Down Expand Up @@ -58,8 +77,31 @@
// Get all registered TRT plugins from registry
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Getting all registered TRT plugins from TRT plugin registry ...";
TensorrtLogger trt_logger = GetTensorrtLogger(false);
initLibNvInferPlugins(&trt_logger, "");
try {
void* library_handle = nullptr;
const auto& env = onnxruntime::GetDefaultEnv();
#if NV_TENSORRT_MAJOR < 10
auto full_path = env.GetRuntimePath() +
PathString(LIBRARY_PREFIX ORT_TSTR("nvinfer_plugin") LIBRARY_EXTENSION);
#else
#ifdef _WIN32
auto full_path = PathString(LIBRARY_PREFIX ORT_TSTR("nvinfer_plugin_" ORT_DEF2STR(NV_TENSORRT_MAJOR)) LIBRARY_EXTENSION);
#else
auto full_path = PathString(LIBRARY_PREFIX ORT_TSTR("nvinfer_plugin") LIBRARY_EXTENSION ORT_TSTR("." ORT_DEF2STR(NV_TENSORRT_MAJOR)));
#endif
#endif

ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, false, &library_handle));

bool (*dyn_initLibNvInferPlugins)(void* logger, char const* libNamespace);
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(library_handle, "initLibNvInferPlugins", (void**)&dyn_initLibNvInferPlugins));

Check warning on line 97 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_custom_ops.cc

View workflow job for this annotation

GitHub Actions / Optional Lint C++ 

[cpplint] reported by reviewdog 🐶
 Using C-style cast.  Use reinterpret_cast<void**>(...) instead  [readability/casting] [4]

Raw Output:
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_custom_ops.cc:97:  Using C-style cast.  Use reinterpret_cast<void**>(...) instead  [readability/casting] [4]
if (!dyn_initLibNvInferPlugins(&trt_logger, "")) {
LOGS_DEFAULT(INFO) << "[TensorRT EP] Default plugin library was found but was not able to initialize default plugins.";
}
LOGS_DEFAULT(INFO) << "[TensorRT EP] Default plugins successfully loaded.";
} catch (const std::exception&) {
LOGS_DEFAULT(INFO) << "[TensorRT EP] Default plugin library is not on the path and is therefore ignored";
}
int num_plugin_creator = 0;
auto plugin_creators = getPluginRegistry()->getAllCreators(&num_plugin_creator);
std::unordered_set<std::string> registered_plugin_names;
Expand Down
Loading