Add support for dynmaic workload type

Author: preetha-intel

onnxruntime/core/providers/openvino/backend_manager.cc

@@ -502,6 +502,15 @@ void BackendManager::Compute(OrtKernelContext* context) {
 
     dynamic_backend->Infer(context);
   } else {
+    if (dynamic_workload_type == "EFFICIENT") {
+      auto ov_execution_network = concrete_backend_->GetOVCompiledModel();
+      ov_execution_network.set_property({{"WORKLOAD_TYPE", "EFFICIENT"}});
+      LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << " OV WORKLOAD_TYPE property is set to EFFICIENT mode for OV inference";
+    } else if (dynamic_workload_type == "DEFAULT") {
+      auto ov_execution_network = concrete_backend_->GetOVCompiledModel();
+      ov_execution_network.set_property({{"WORKLOAD_TYPE", "DEFAULT"}});
+      LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << " OV WORKLOAD_TYPE property is set to DEFAULT mode for OV inference";
+    }
     concrete_backend_->Infer(context);
   }
 #ifdef OPENVINO_FIL_ENABLED

onnxruntime/core/providers/openvino/backend_manager.h

@@ -30,6 +30,7 @@ class BackendManager {
   GlobalContext& GetGlobalContext();
   Status ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphViewer& subgraph,
                                        const logging::Logger& logger);
+  std::string dynamic_workload_type = "DEFAULT";
 
  private:
   std::unique_ptr<ONNX_NAMESPACE::ModelProto> GetModelProtoFromFusedNode(

onnxruntime/core/providers/openvino/openvino_execution_provider.cc

@@ -150,7 +150,7 @@ common::Status OpenVINOExecutionProvider::Compile(
                                                       graph_body_viewer,
                                                       *GetLogger(),
                                                       ep_ctx_handle_);
-
+    backend_manager_ = backend_manager;
     compute_info.create_state_func =
         [backend_manager](ComputeContext* context, FunctionState* state) {
           OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState();
@@ -198,4 +198,18 @@ std::vector<AllocatorPtr> OpenVINOExecutionProvider::CreatePreferredAllocators()
 }
 #endif
 
+common::Status OpenVINOExecutionProvider::SetEpDynamicOptions(gsl::span<const char* const> keys,
+                                                              gsl::span<const char* const> values) {
+  if ((std::find(keys.begin(), keys.end(), "ep.dynamic.workload_type") != keys.end()) &&
+      (std::find(values.begin(), values.end(), "Efficient") != values.end())) {
+    backend_manager_->dynamic_workload_type = "EFFICIENT";
+    LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << backend_manager_->dynamic_workload_type << " mode is set for OV inference";
+  }
+  if ((std::find(keys.begin(), keys.end(), "ep.dynamic.workload_type") != keys.end()) &&
+      (std::find(values.begin(), values.end(), "Default") != values.end())) {
+    backend_manager_->dynamic_workload_type = "DEFAULT";
+    LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << backend_manager_->dynamic_workload_type << " mode is set for OV inference";
+  }
+  return Status::OK();
+}
 }  // namespace onnxruntime

onnxruntime/core/providers/openvino/openvino_execution_provider.h

@@ -188,6 +188,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider {
   Status Compile(const std::vector<FusedNodeAndGraph>& fused_nodes,
                  std::vector<NodeComputeInfo>& node_compute_funcs) override;
 
+  Status SetEpDynamicOptions(gsl::span<const char* const> /*keys*/,
+                             gsl::span<const char* const> /*values*/) override;
+
   const void* GetExecutionHandle() const noexcept override {
     return nullptr;
   }
@@ -196,6 +199,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider {
 #endif
  private:
   std::unique_ptr<openvino_ep::GlobalContext> global_context_;
+  std::shared_ptr<openvino_ep::BackendManager> backend_manager_;
   openvino_ep::EPCtxHandler ep_ctx_handle_{};
 };