Image generation multiconcurrency (#2190) (#2284)

Cherry pick from release branch:
#2190

Author: dkalinowski

samples/cpp/image_generation/CMakeLists.txt

@@ -38,6 +38,22 @@ install(TARGETS text2image
         COMPONENT samples_bin
         EXCLUDE_FROM_ALL)
 
+# create text2image concurrent sample executable
+
+add_executable(text2image_concurrency text2image_concurrency.cpp imwrite.cpp)
+
+target_include_directories(text2image_concurrency PRIVATE ${CMAKE_BINARY_DIR} "${CMAKE_CURRENT_SOURCE_DIR}")
+target_link_libraries(text2image_concurrency PRIVATE openvino::genai indicators::indicators)
+
+set_target_properties(text2image_concurrency PROPERTIES
+    # Ensure out of box LC_RPATH on macOS with SIP
+    INSTALL_RPATH_USE_LINK_PATH ON)
+
+install(TARGETS text2image_concurrency
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)
+
 # create LoRA sample executable
 
 add_executable(lora_text2image lora_text2image.cpp imwrite.cpp)
@@ -88,6 +104,22 @@ install(TARGETS image2image
         COMPONENT samples_bin
         EXCLUDE_FROM_ALL)
 
+# create image2image concurrent sample executable
+
+add_executable(image2image_concurrency image2image_concurrency.cpp load_image.cpp imwrite.cpp)
+
+target_include_directories(image2image_concurrency PRIVATE ${CMAKE_BINARY_DIR} "${CMAKE_CURRENT_SOURCE_DIR}")
+target_link_libraries(image2image_concurrency PRIVATE openvino::genai indicators::indicators)
+
+set_target_properties(image2image_concurrency PROPERTIES
+    # Ensure out of box LC_RPATH on macOS with SIP
+    INSTALL_RPATH_USE_LINK_PATH ON)
+
+install(TARGETS image2image_concurrency
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)
+
 # create inpainting executable
 
 add_executable(inpainting inpainting.cpp load_image.cpp imwrite.cpp)

samples/cpp/image_generation/README.md

@@ -4,9 +4,11 @@ Examples in this folder showcase inference of text to image models like Stable D
 
 There are several sample files:
  - [`text2image.cpp`](./text2image.cpp) demonstrates basic usage of the text to image pipeline
+ - [`text2image_concurrency.cpp`](./text2image_concurrency.cpp) demonstrates concurrent usage of the text to image pipeline to create multiple images with different prompts
  - [`lora_text2image.cpp`](./lora_text2image.cpp) shows how to apply LoRA adapters to the pipeline
  - [`heterogeneous_stable_diffusion.cpp`](./heterogeneous_stable_diffusion.cpp) shows how to assemble a heterogeneous txt2image pipeline from individual subcomponents (scheduler, text encoder, unet, vae decoder)
  - [`image2image.cpp`](./image2image.cpp) demonstrates basic usage of the image to image pipeline
+ - [`image2image_concurrency.cpp.cpp`](./image2image_concurrency.cpp) demonstrates concurrent usage of the image to image pipeline to create multiple images with different prompts
  - [`inpainting.cpp`](./inpainting.cpp) demonstrates basic usage of the inpainting pipeline
  - [`benchmark_image_gen.cpp`](./benchmark_image_gen.cpp) demonstrates how to benchmark the text to image / image to image / inpainting pipeline
 
@@ -210,3 +212,42 @@ Test finish, load time: 9356.00 ms
 Warmup number:1, first generate warmup time:85008.00 ms, infer warmup time:84999.88 ms
 Generate iteration number:3, for one iteration, generate avg time: 84372.34 ms, infer avg time:84363.95 ms, all text encoders infer avg time:76.67 ms, vae encoder infer avg time:0.00 ms, vae decoder infer avg time:4470.33 ms
 ```
+
+### Run multiple generations with different prompt in parallel
+
+It is highly recommended to use `ov::genai::num_images_per_prompt(X)` parameter to generate multiple images in parallel. However, when the generation options differ (prompt, height, width), it is recommended to clone the pipeline.
+It is possible to re-use models compiled into device for concurrent generation with different prompts in separate threads.
+
+Here in this example we load and compile the entire pipeline once, and then use `clone()` to create separate generation requests to be reused in separate threads:
+
+
+```cpp
+std::vector<ov::genai::Text2ImagePipeline> pipelines;
+
+// Prepare initial pipeline and compiled models into device
+pipelines.emplace_back(models_path, device);
+// Clone pipeline for concurrent usage
+for (size_t i = 1; i < 4; i++)
+   pipelines.emplace_back(pipelines.begin()->clone());
+
+std::vector<std::thread> threads;
+
+for (size_t i = 0; i < 4; i++) {
+  auto& pipe = pipelines.at(i);
+  threads.emplace_back([&pipe, i] {
+    std::string prompt = "A card with number " + std::to_string(i);
+
+    ov::Tensor image = pipe.generate(prompt,
+      ov::AnyMap{
+        ov::genai::width(512),
+        ov::genai::height(512),
+        ov::genai::num_inference_steps(25)});
+
+    // save image
+  });
+}
+
+for (auto& thread : threads) {
+   thread.join();
+}
+```

samples/cpp/image_generation/image2image_concurrency.cpp

@@ -0,0 +1,68 @@
+// Copyright (C) 2023-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include <vector>
+#include <string>
+
+#include "openvino/genai/image_generation/image2image_pipeline.hpp"
+
+#include "imwrite.hpp"
+#include "load_image.hpp"
+#include "progress_bar.hpp"
+
+int32_t main(int32_t argc, char* argv[]) try {
+    OPENVINO_ASSERT(argc >= 4, "Usage: ", argv[0], " <MODEL_DIR> '<PROMPT>' '<PROMPT>' ... <IMAGE>");
+
+    const std::string models_path = argv[1];
+    const std::string device = "CPU";  // GPU can be used as well
+    const std::string image_path = argv[argc - 1];
+    ov::Tensor image = utils::load_image(image_path);
+
+    std::vector<std::thread> threads;
+    std::vector<std::string> prompts;
+    std::vector<ov::genai::Image2ImagePipeline> pipelines;
+
+    for (int32_t i = 2; i < argc - 1; ++i)
+        prompts.push_back(argv[i]);
+
+    // Prepare initial pipeline and compiled models into device
+    pipelines.emplace_back(models_path, device);
+
+    // Clone pipeline for concurrent usage
+    for (size_t i = 1; i < prompts.size(); ++i)
+        pipelines.emplace_back(pipelines.begin()->clone());
+
+    for (size_t i = 0; i < prompts.size(); ++i) {
+        std::string prompt = prompts[i];
+        auto& pipe = pipelines.at(i);
+
+        std::cout << "Starting to generate with prompt: '" << prompt << "'..." << std::endl;
+
+        threads.emplace_back([i, &pipe, prompt, image] () {
+
+            ov::Tensor generated_image = pipe.generate(prompt, image,
+                // controls how initial image is noised after being converted to latent space. `1` means initial image is fully noised
+                ov::genai::strength(0.8f),
+                ov::genai::num_inference_steps(4));
+
+            // writes `num_images_per_prompt` images by pattern name
+            imwrite("image_" + std::to_string(i) + "_%d.bmp", generated_image, true);
+        });
+    }
+
+    for (auto& thread : threads) {
+        thread.join();
+    }
+
+    return EXIT_SUCCESS;
+} catch (const std::exception& error) {
+    try {
+        std::cerr << error.what() << '\n';
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+} catch (...) {
+    try {
+        std::cerr << "Non-exception object thrown\n";
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+}

samples/cpp/image_generation/text2image_concurrency.cpp

@@ -0,0 +1,65 @@
+// Copyright (C) 2023-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+#include <iostream>
+
+#include "openvino/genai/image_generation/text2image_pipeline.hpp"
+
+
+#include "imwrite.hpp"
+#include "progress_bar.hpp"
+
+int32_t main(int32_t argc, char* argv[]) try {
+    OPENVINO_ASSERT(argc >= 3, "Usage: ", argv[0], " <MODEL_DIR> '<PROMPT>' '<PROMPT>' ...");
+
+    const std::string models_path = argv[1];
+    const std::string device = "CPU";  // GPU can be used as well
+
+    std::vector<std::thread> threads;
+    std::vector<std::string> prompts;
+    std::vector<ov::genai::Text2ImagePipeline> pipelines;
+
+    for (int i = 2; i < argc; ++i)
+        prompts.push_back(argv[i]);
+
+    // Prepare initial pipeline and compiled models into device
+    pipelines.emplace_back(models_path, device);
+
+    // Clone pipeline for concurrent usage
+    for (size_t i = 1; i < prompts.size(); ++i)
+        pipelines.emplace_back(pipelines.begin()->clone());
+
+    for (size_t i = 0; i < prompts.size(); ++i) {
+        std::string prompt = prompts[i];
+        auto& pipe = pipelines.at(i);
+
+        std::cout << "Starting to generate with prompt: '" << prompt << "'..." << std::endl;
+
+        threads.emplace_back([i, &pipe, prompt] () {
+
+            ov::Tensor image = pipe.generate(prompt,
+                ov::AnyMap{
+                    ov::genai::width(512),
+                    ov::genai::height(512),
+                    ov::genai::num_inference_steps(2),
+                    ov::genai::num_images_per_prompt(1)});
+
+            imwrite("image_" + std::to_string(i) + "_%d.bmp", image, true);
+        });
+    }
+
+    for (auto& thread : threads) {
+        thread.join();
+    }
+
+    return EXIT_SUCCESS;
+} catch (const std::exception& error) {
+    try {
+        std::cerr << error.what() << '\n';
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+} catch (...) {
+    try {
+        std::cerr << "Non-exception object thrown\n";
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+}

src/cpp/include/openvino/genai/image_generation/autoencoder_kl.hpp

@@ -116,6 +116,8 @@ class OPENVINO_GENAI_EXPORTS AutoencoderKL {
 
     AutoencoderKL(const AutoencoderKL&);
 
+    AutoencoderKL clone();
+
     AutoencoderKL& reshape(int batch_size, int height, int width);
 
     AutoencoderKL& compile(const std::string& device, const ov::AnyMap& properties = {});

src/cpp/include/openvino/genai/image_generation/clip_text_model.hpp

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <filesystem>
+#include <memory>
 #include <string>
 
 #include "openvino/genai/visibility.hpp"
@@ -69,6 +70,8 @@ class OPENVINO_GENAI_EXPORTS CLIPTextModel {
 
     CLIPTextModel(const CLIPTextModel&);
 
+    std::shared_ptr<CLIPTextModel> clone();
+
     const Config& get_config() const;
 
     CLIPTextModel& reshape(int batch_size);
@@ -91,12 +94,12 @@ class OPENVINO_GENAI_EXPORTS CLIPTextModel {
 private:
     Config m_config;
     AdapterController m_adapter_controller;
-    ov::InferRequest m_request;
-    std::shared_ptr<ov::Model> m_model;
-
     Tokenizer m_clip_tokenizer;
-
     bool m_slice_batch1_output = false;
+
+protected:
+    ov::InferRequest m_request;
+    std::shared_ptr<ov::Model> m_model;
 };
 
 } // namespace genai

src/cpp/include/openvino/genai/image_generation/clip_text_model_with_projection.hpp

@@ -11,6 +11,21 @@ namespace genai {
 class CLIPTextModelWithProjection : public CLIPTextModel {
 public:
     using CLIPTextModel::CLIPTextModel;
+
+    std::shared_ptr<CLIPTextModel> clone() {
+        OPENVINO_ASSERT((m_model != nullptr) ^ static_cast<bool>(m_request), "CLIPTextModelWithProjection must have exactly one of m_model or m_request initialized");
+
+        std::shared_ptr<CLIPTextModelWithProjection> cloned = std::make_shared<CLIPTextModelWithProjection>(*this);
+
+        if (m_model) {
+            cloned->m_model = m_model->clone();
+        } else {
+            cloned->m_request = m_request.get_compiled_model().create_infer_request();
+        }
+
+        return cloned;
+    }
+
 };
 
 } // namespace genai

src/cpp/include/openvino/genai/image_generation/flux_transformer_2d_model.hpp

@@ -62,6 +62,8 @@ class OPENVINO_GENAI_EXPORTS FluxTransformer2DModel {
 
     FluxTransformer2DModel(const FluxTransformer2DModel&);
 
+    FluxTransformer2DModel clone();
+
     const Config& get_config() const;
 
     FluxTransformer2DModel& reshape(int batch_size, int height, int width, int tokenizer_model_max_length);

src/cpp/include/openvino/genai/image_generation/image2image_pipeline.hpp

@@ -73,6 +73,13 @@ class OPENVINO_GENAI_EXPORTS Image2ImagePipeline {
         const CLIPTextModelWithProjection& clip_text_model_2,
         const SD3Transformer2DModel& transformer,
         const AutoencoderKL& vae);
+    
+    /**
+     * Method to clone the pipeline to be used in parallel by another thread.
+     * Reuses underlying models and recreates scheduler and generation config.
+     * @returns A new pipeline for concurrent usage
+     */
+    Image2ImagePipeline clone();
 
     ImageGenerationConfig get_generation_config() const;
     void set_generation_config(const ImageGenerationConfig& generation_config);

src/cpp/include/openvino/genai/image_generation/sd3_transformer_2d_model.hpp

@@ -63,6 +63,8 @@ class OPENVINO_GENAI_EXPORTS SD3Transformer2DModel {
 
     SD3Transformer2DModel(const SD3Transformer2DModel&);
 
+    SD3Transformer2DModel clone();
+
     const Config& get_config() const;
 
     SD3Transformer2DModel& reshape(int batch_size, int height, int width, int tokenizer_model_max_length);