Support multi images for vlm benchmarking in samples and llm_bench

samples/cpp/text_generation/CMakeLists.txt

@@ -46,7 +46,7 @@ FetchContent_Declare(cxxopts
     URL_HASH SHA256=523175f792eb0ff04f9e653c90746c12655f10cb70f1d5e6d6d9491420298a08)
 FetchContent_MakeAvailable(cxxopts)
 
-add_executable(benchmark_genai benchmark_genai.cpp)
+add_executable(benchmark_genai benchmark_genai.cpp read_prompt_from_file.cpp)
 target_link_libraries(benchmark_genai PRIVATE openvino::genai cxxopts::cxxopts)
 set_target_properties(benchmark_genai PROPERTIES
     # Ensure out of box LC_RPATH on macOS with SIP

samples/cpp/text_generation/README.md

@@ -161,7 +161,8 @@ For more information how performance metrics are calculated please follow [perfo
   ```
   #### Options
 - `-m, --model`: Path to the model and tokenizers base directory.
-- `-p, --prompt` (default: `"The Sky is blue because"`): The prompt to generate text.
+- `-p, --prompt` (default: ''): The prompt to generate text. If without `-p` and `--pf`, the default prompt is `"The Sky is blue because"`
+- `--pf, --prompt_file` Read prompt from file.
 - `--nw, --num_warmup` (default: `1`): Number of warmup iterations.
 - `--mt, --max_new_tokens` (default: `20`): Maximal number of new tokens.
 - `-n, --num_iter` (default: `3`): Number of iterations.

samples/cpp/text_generation/benchmark_genai.cpp

@@ -3,13 +3,15 @@
 
 #include "openvino/genai/llm_pipeline.hpp"
 #include <cxxopts.hpp>
+#include "read_prompt_from_file.h"
 
 int main(int argc, char* argv[]) try {
     cxxopts::Options options("benchmark_vanilla_genai", "Help command");
 
     options.add_options()
     ("m,model", "Path to model and tokenizers base directory", cxxopts::value<std::string>())
-    ("p,prompt", "Prompt", cxxopts::value<std::string>()->default_value("The Sky is blue because"))
+    ("p,prompt", "Prompt", cxxopts::value<std::string>()->default_value(""))
+    ("pf,prompt_file", "Read prompt from file", cxxopts::value<std::string>())
     ("nw,num_warmup", "Number of warmup iterations", cxxopts::value<size_t>()->default_value(std::to_string(1)))
     ("n,num_iter", "Number of iterations", cxxopts::value<size_t>()->default_value(std::to_string(3)))
     ("mt,max_new_tokens", "Maximal number of new tokens", cxxopts::value<size_t>()->default_value(std::to_string(20)))
@@ -30,7 +32,22 @@ int main(int argc, char* argv[]) try {
         return EXIT_SUCCESS;
     }
 
-    std::string prompt = result["prompt"].as<std::string>();
+    std::string prompt;
+    if (result.count("prompt") && result.count("prompt_file")) {
+        std::cout << "Prompt and prompt file should not exist together!" << std::endl;
+        return EXIT_FAILURE;
+    } else {
+        if (result.count("prompt_file")) {
+            prompt = utils::read_prompt(result["prompt_file"].as<std::string>());
+        } else {
+            prompt = result["prompt"].as<std::string>().empty() ? "The Sky is blue because" : result["prompt"].as<std::string>();
+        }
+    }
+    if (prompt.empty()) {
+        std::cout << "Prompt is empty!" << std::endl;
+        return EXIT_FAILURE;
+    }
+
     const std::string models_path = result["model"].as<std::string>();
     std::string device = result["device"].as<std::string>();
     size_t num_warmup = result["num_warmup"].as<size_t>();
@@ -39,7 +56,17 @@ int main(int argc, char* argv[]) try {
     ov::genai::GenerationConfig config;
     config.max_new_tokens = result["max_new_tokens"].as<size_t>();
 
-    ov::genai::LLMPipeline pipe(models_path, device);
+    ov::genai::SchedulerConfig scheduler_config;
+    scheduler_config.enable_prefix_caching = false;
+    scheduler_config.max_num_batched_tokens = std::numeric_limits<std::size_t>::max();
+
+    std::cout << ov::get_openvino_version() << std::endl;
+
+    ov::genai::LLMPipeline pipe(models_path, device, ov::genai::scheduler_config(scheduler_config));
+
+    auto input_data = pipe.get_tokenizer().encode(prompt);
+    size_t prompt_token_size = input_data.input_ids.get_shape()[1];
+    std::cout << "Prompt token size:" << prompt_token_size << std::endl;
 
     for (size_t i = 0; i < num_warmup; i++)
         pipe.generate(prompt, config);
@@ -52,6 +79,7 @@ int main(int argc, char* argv[]) try {
     }
 
     std::cout << std::fixed << std::setprecision(2);
+    std::cout << "Output token size:" << res.perf_metrics.get_num_generated_tokens() << std::endl;
     std::cout << "Load time: " << metrics.get_load_time() << " ms" << std::endl;
     std::cout << "Generate time: " << metrics.get_generate_duration().mean << " ± " << metrics.get_generate_duration().std << " ms" << std::endl;
     std::cout << "Tokenization time: " << metrics.get_tokenization_duration().mean << " ± " << metrics.get_tokenization_duration().std << " ms" << std::endl;

samples/cpp/text_generation/read_prompt_from_file.cpp

@@ -0,0 +1,19 @@
+// Copyright (C) 2023-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include <iostream>
+#include <fstream>
+#include "read_prompt_from_file.h"
+
+std::string utils::read_prompt(const std::string& file_path) {
+    std::ifstream file(file_path);
+    if (file.is_open()) {
+        std::stringstream buffer;
+        buffer << file.rdbuf();
+        return buffer.str();
+    } else {
+        std::stringstream error_message;
+        error_message << "Error opening prompt file: '" << file_path << "'";
+        throw std::runtime_error{error_message.str()};
+    }
+}

samples/cpp/text_generation/read_prompt_from_file.h

@@ -0,0 +1,11 @@
+
+// Copyright (C) 2023-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <sstream>
+
+namespace utils {
+std::string read_prompt(const std::string& file_path);
+}

samples/cpp/visual_language_chat/CMakeLists.txt

@@ -44,8 +44,7 @@ install(TARGETS encrypted_model_vlm
         EXCLUDE_FROM_ALL)
 
 # create benchmark executable
-
-add_executable(benchmark_vlm benchmark_vlm.cpp load_image.cpp)
+add_executable(benchmark_vlm benchmark_vlm.cpp load_image.cpp ../text_generation/read_prompt_from_file.cpp)
 target_include_directories(benchmark_vlm PRIVATE "${CMAKE_BINARY_DIR}")
 target_link_libraries(benchmark_vlm PRIVATE openvino::genai cxxopts::cxxopts)
 set_target_properties(benchmark_vlm PROPERTIES

samples/cpp/visual_language_chat/README.md

@@ -40,7 +40,8 @@ benchmark_vlm [OPTIONS]
 ### Options
 
 - `-m, --model`(default: `.`): Path to the model and tokenizers base directory.
-- `-p, --prompt` (default: `What is on the image?`): The prompt to generate text.
+- `-p, --prompt` (default: ''): The prompt to generate text. If without `-p` and `--pf`, the default prompt is `"What is on the image?"`
+- `--pf, --prompt_file` Read prompt from file.
 - `-i, --image` (default: `image.jpg`): Path to the image.
 - `-nw, --num_warmup` (default: `1`): Number of warmup iterations.
 - `-mt, --max_new_tokens` (default: `20`): Maximal number of new tokens.

samples/cpp/visual_language_chat/benchmark_vlm.cpp

@@ -6,14 +6,15 @@
 
 #include "load_image.hpp"
 #include <openvino/genai/visual_language/pipeline.hpp>
-
+#include "../text_generation/read_prompt_from_file.h"
 
 int main(int argc, char* argv[]) try {
     cxxopts::Options options("benchmark_vlm", "Help command");
 
     options.add_options()
     ("m,model", "Path to model and tokenizers base directory", cxxopts::value<std::string>()->default_value("."))
-    ("p,prompt", "Prompt", cxxopts::value<std::string>()->default_value("What is on the image?"))
+    ("p,prompt", "Prompt", cxxopts::value<std::string>()->default_value(""))
+    ("pf,prompt_file", "Read prompt from file", cxxopts::value<std::string>())
     ("i,image", "Image", cxxopts::value<std::string>()->default_value("image.jpg"))
     ("nw,num_warmup", "Number of warmup iterations", cxxopts::value<size_t>()->default_value(std::to_string(1)))
     ("n,num_iter", "Number of iterations", cxxopts::value<size_t>()->default_value(std::to_string(3)))
@@ -35,30 +36,57 @@ int main(int argc, char* argv[]) try {
         return EXIT_SUCCESS;
     }
 
-    std::string prompt = result["prompt"].as<std::string>();
+    std::string prompt;
+    if (result.count("prompt") && result.count("prompt_file")) {
+        std::cout << "Prompt and prompt file should not exist together!" << std::endl;
+        return EXIT_FAILURE;
+    } else {
+        if (result.count("prompt_file")) {
+            prompt = utils::read_prompt(result["prompt_file"].as<std::string>());
+        } else {
+            prompt = result["prompt"].as<std::string>().empty() ? "What is on the image?" : result["prompt"].as<std::string>();
+        }
+    }
+    if (prompt.empty()) {
+        std::cout << "Prompt is empty!" << std::endl;
+        return EXIT_FAILURE;
+    } 
+
     const std::string models_path = result["model"].as<std::string>();
     const std::string image_path = result["image"].as<std::string>();
     std::string device = result["device"].as<std::string>();
     size_t num_warmup = result["num_warmup"].as<size_t>();
     size_t num_iter = result["num_iter"].as<size_t>();
-    ov::Tensor image = utils::load_image(image_path);
-  
+    std::vector<ov::Tensor> images = utils::load_images(image_path);
+
     ov::genai::GenerationConfig config;
     config.max_new_tokens = result["max_new_tokens"].as<size_t>();
+    config.ignore_eos = true;
+
+    ov::genai::SchedulerConfig scheduler_config;
+    scheduler_config.enable_prefix_caching = false;
+    scheduler_config.max_num_batched_tokens = std::numeric_limits<std::size_t>::max();
+
+    std::cout << ov::get_openvino_version() << std::endl;
+
+    ov::genai::VLMPipeline pipe(models_path, device, ov::genai::scheduler_config(scheduler_config));
+
+    auto input_data = pipe.get_tokenizer().encode(prompt);
+    size_t prompt_token_size = input_data.input_ids.get_shape()[1];
+    std::cout << "Number of images:" << images.size() << ", prompt token size:" << prompt_token_size << std::endl;
 
-    ov::genai::VLMPipeline pipe(models_path, device);
-
     for (size_t i = 0; i < num_warmup; i++)
-        pipe.generate(prompt, ov::genai::image(image), ov::genai::generation_config(config));
+        pipe.generate(prompt, ov::genai::images(images), ov::genai::generation_config(config));
 
-    auto res = pipe.generate(prompt, ov::genai::image(image), ov::genai::generation_config(config));
+    auto res = pipe.generate(prompt, ov::genai::images(images), ov::genai::generation_config(config));
     auto metrics = res.perf_metrics;
     for (size_t i = 0; i < num_iter - 1; i++) {
-        res = pipe.generate(prompt, ov::genai::image(image), ov::genai::generation_config(config));
+        res = pipe.generate(prompt, ov::genai::images(images), ov::genai::generation_config(config));
         metrics = metrics + res.perf_metrics;
     }
 
     std::cout << std::fixed << std::setprecision(2);
+    std::cout << "Output token size:" << res.perf_metrics.get_num_generated_tokens() << std::endl;
     std::cout << "Load time: " << metrics.get_load_time() << " ms" << std::endl;
     std::cout << "Generate time: " << metrics.get_generate_duration().mean << " ± " << metrics.get_generate_duration().std << " ms" << std::endl;
     std::cout << "Tokenization time: " << metrics.get_tokenization_duration().mean << " ± " << metrics.get_tokenization_duration().std << " ms" << std::endl;

samples/python/text_generation/README.md

@@ -153,7 +153,8 @@ For more information how performance metrics are calculated please follow [perfo
   ```
   #### Options
 - `-m, --model`: Path to the model and tokenizers base directory.
-- `-p, --prompt` (default: `"The Sky is blue because"`): The prompt to generate text.
+- `-p, --prompt` (default: `None`): The prompt to generate text. If without `-p` and `-pf`, the default prompt is `"The Sky is blue because"`
+- `-pf, --prompt_file` Read prompt from file.
 - `-nw, --num_warmup` (default: `1`): Number of warmup iterations.
 - `-mt, --max_new_tokens` (default: `20`): Maximal number of new tokens.
 - `-n, --num_iter` (default: `3`): Number of iterations.

samples/python/text_generation/benchmark_genai.py

@@ -1,23 +1,38 @@
 # Copyright (C) 2023-2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+import sys
 import argparse
 import openvino_genai as ov_genai
+from openvino import get_version
 
 def main():
     parser = argparse.ArgumentParser(description="Help command")
     parser.add_argument("-m", "--model", type=str, required=True, help="Path to model and tokenizers base directory")
-    parser.add_argument("-p", "--prompt", type=str, default="The Sky is blue because", help="Prompt")
+    parser.add_argument("-p", "--prompt", type=str, default=None, help="Prompt")
+    parser.add_argument("-pf", "--prompt_file", type=str, help="Read prompt from file")
     parser.add_argument("-nw", "--num_warmup", type=int, default=1, help="Number of warmup iterations")
     parser.add_argument("-n", "--num_iter", type=int, default=2, help="Number of iterations")
     parser.add_argument("-mt", "--max_new_tokens", type=int, default=20, help="Maximal number of new tokens")
     parser.add_argument("-d", "--device", type=str, default="CPU", help="Device")
 
     args = parser.parse_args()
 
+    if args.prompt is not None and args.prompt_file is not None:
+        raise RuntimeError(f'Prompt and prompt file should not exist together!')
+    else:
+        if args.prompt_file is not None:
+            with open(args.prompt_file, 'r', encoding='utf-8') as f:
+                prompt = [f.read()]
+        else:
+            prompt = ['The Sky is blue because'] if args.prompt is None else [args.prompt]
+    if len(prompt) == 0:
+        raise RuntimeError(f'Prompt is empty!')
+
+    print(f'openvino runtime version: {get_version()}')
+
     # Perf metrics is stored in DecodedResults. 
     # In order to get DecodedResults instead of a string input should be a list.
-    prompt = [args.prompt]
     models_path = args.model
     device = args.device
     num_warmup = args.num_warmup
@@ -26,8 +41,16 @@ def main():
     config = ov_genai.GenerationConfig()
     config.max_new_tokens = args.max_new_tokens
 
-    pipe = ov_genai.LLMPipeline(models_path, device)
+    scheduler_config = ov_genai.SchedulerConfig()
+    scheduler_config.enable_prefix_caching = False
+    scheduler_config.max_num_batched_tokens = sys.maxsize
+
+    pipe = ov_genai.LLMPipeline(models_path, device, scheduler_config=scheduler_config)
 
+    input_data = pipe.get_tokenizer().encode(prompt)
+    prompt_token_size = input_data.input_ids.get_shape()[1]
+    print(f"Prompt token size: {prompt_token_size}")
+
     for _ in range(num_warmup):
         pipe.generate(prompt, config)
 
@@ -37,6 +60,7 @@ def main():
         res = pipe.generate(prompt, config)
         perf_metrics += res.perf_metrics
 
+    print(f"Output token size: {res.perf_metrics.get_num_generated_tokens()}")
     print(f"Load time: {perf_metrics.get_load_time():.2f} ms")
     print(f"Generate time: {perf_metrics.get_generate_duration().mean:.2f} ± {perf_metrics.get_generate_duration().std:.2f} ms")
     print(f"Tokenization time: {perf_metrics.get_tokenization_duration().mean:.2f} ± {perf_metrics.get_tokenization_duration().std:.2f} ms")

samples/python/visual_language_chat/README.md

@@ -40,7 +40,8 @@ python benchmark_vlm.py [OPTIONS]
 ### Options
 
 - `-m, --model`(default: `.`): Path to the model and tokenizers base directory.
-- `-p, --prompt` (default: `What is on the image?`): The prompt to generate text.
+- `-p, --prompt` (default: `None`): The prompt to generate text. If without `-p` and `-pf`, the default prompt is `"What is on the image?"`
+- `-pf, --prompt_file` Read prompt from file.
 - `-i, --image` (default: `image.jpg`): Path to the image.
 - `-nw, --num_warmup` (default: `1`): Number of warmup iterations.
 - `-mt, --max_new_tokens` (default: `20`): Maximal number of new tokens.