Fix paths with unicode for tokenizers (#2337)

Ticket: CVS-169069

Author: yatarkan

src/cpp/src/tokenizer/tokenizer.cpp

@@ -38,9 +38,17 @@ constexpr char pad_token_key_name[] = "pad_token";
 
 ov::Core core_with_extension() {
     ov::Core core;
+
+#ifdef _WIN32
+    const wchar_t* ov_tokenizer_path_w = _wgetenv(ScopedVar::ENVIRONMENT_VARIABLE_NAME_W);
+    OPENVINO_ASSERT(ov_tokenizer_path_w, "openvino_tokenizers path is not set");
+    core.add_extension(std::wstring(ov_tokenizer_path_w));
+#else
     const char* ov_tokenizer_path = getenv(ScopedVar::ENVIRONMENT_VARIABLE_NAME);
     OPENVINO_ASSERT(ov_tokenizer_path, "openvino_tokenizers path is not set");
     core.add_extension(ov_tokenizer_path);
+#endif
+    
     return core;
 }
 
@@ -252,8 +260,14 @@ class Tokenizer::TokenizerImpl {
         filtered_properties = {};
         if (is_gguf_model(models_path)) {
             std::map<std::string, GGUFMetaData> tokenizer_config{};
+            std::filesystem::path ov_tokenizer_filesystem_path;
+#ifdef _WIN32
+            const wchar_t* ov_tokenizer_path_w = _wgetenv(ScopedVar::ENVIRONMENT_VARIABLE_NAME_W);
+            ov_tokenizer_filesystem_path = std::filesystem::path(std::wstring(ov_tokenizer_path_w));
+#else
             const char* ov_tokenizer_path = getenv(ScopedVar::ENVIRONMENT_VARIABLE_NAME);
-            auto ov_tokenizer_filesystem_path = std::filesystem::path(ov_tokenizer_path);
+            ov_tokenizer_filesystem_path = std::filesystem::path(ov_tokenizer_path);
+#endif
             m_shared_object_ov_tokenizers = load_shared_object(ov_tokenizer_filesystem_path);
             std::tie(ov_tokenizer, ov_detokenizer, tokenizer_config) =
                 create_tokenizer_from_config(m_shared_object_ov_tokenizers, models_path);

src/cpp/src/tokenizer/tokenizers_path.cpp

@@ -4,6 +4,7 @@
 #include "tokenizer/tokenizers_path.hpp"
 
 #include <sstream>
+
 #ifdef _WIN32
 #    include <windows.h>
 #    define MAX_ABS_PATH _MAX_PATH
@@ -34,23 +35,28 @@ std::string get_absolute_file_path(const std::string& path) {
 }
 #endif
 
-std::string get_ov_genai_library_path() {
+std::filesystem::path get_ov_genai_library_path() {
 #ifdef _WIN32
-    CHAR genai_library_path[MAX_PATH];
+    WCHAR genai_library_path_w[MAX_PATH];
     HMODULE hm = NULL;
-    if (!GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                            reinterpret_cast<LPSTR>(get_ov_genai_library_path),
+    if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                            reinterpret_cast<LPCWSTR>(get_ov_genai_library_path),
                             &hm)) {
         std::stringstream ss;
-        ss << "GetModuleHandle returned " << GetLastError();
+        ss << "GetModuleHandleExW returned " << GetLastError();
+        throw std::runtime_error(ss.str());
+    }
+    DWORD result = GetModuleFileNameW(hm, (LPWSTR)genai_library_path_w, sizeof(genai_library_path_w) / sizeof(genai_library_path_w[0]));
+    if (result == 0) {
+        std::stringstream ss;
+        ss << "GetModuleFileNameW failed with error " << GetLastError();
         throw std::runtime_error(ss.str());
     }
-    GetModuleFileNameA(hm, (LPSTR)genai_library_path, sizeof(genai_library_path));
-    return std::string(genai_library_path);
+    return std::filesystem::path(std::wstring(genai_library_path_w));
 #elif defined(__APPLE__) || defined(__linux__) || defined(__EMSCRIPTEN__)
     Dl_info info;
     dladdr(reinterpret_cast<void*>(get_ov_genai_library_path), &info);
-    return get_absolute_file_path(info.dli_fname).c_str();
+    return std::filesystem::path(get_absolute_file_path(info.dli_fname));
 #else
 #    error "Unsupported OS"
 #endif  // _WIN32

src/cpp/src/tokenizer/tokenizers_path.hpp

@@ -25,25 +25,33 @@ class ScopedVar {
     bool was_already_set{false};
 public:
     static constexpr char ENVIRONMENT_VARIABLE_NAME[] = "OPENVINO_TOKENIZERS_PATH_GENAI";
+    static constexpr wchar_t ENVIRONMENT_VARIABLE_NAME_W[] = L"OPENVINO_TOKENIZERS_PATH_GENAI";
 
     explicit ScopedVar(const std::filesystem::path& environment_variable_value) {
+
 #ifdef _WIN32
-        char* value = nullptr;
+        wchar_t* value = nullptr;
         size_t len = 0;
-        _dupenv_s(&value, &len, ENVIRONMENT_VARIABLE_NAME);
-        if (value == nullptr)
-            _putenv_s(ENVIRONMENT_VARIABLE_NAME, environment_variable_value.string().c_str());
+        _wdupenv_s(&value, &len, ENVIRONMENT_VARIABLE_NAME_W);
+        if (value == nullptr) {
+            _wputenv_s(ENVIRONMENT_VARIABLE_NAME_W, environment_variable_value.wstring().c_str());
+        } else {
+            was_already_set = true;
+            free(value);
+        }
 #else
-        if (!getenv(ENVIRONMENT_VARIABLE_NAME))
+        if (!getenv(ENVIRONMENT_VARIABLE_NAME)) {
             setenv(ENVIRONMENT_VARIABLE_NAME, environment_variable_value.string().c_str(), 1);
-#endif
-        else
+        } else {
             was_already_set = true;
+        }
+#endif
     }
+    
     ~ScopedVar() {
         if (!was_already_set) {
 #ifdef _WIN32
-            _putenv_s(ENVIRONMENT_VARIABLE_NAME, "");
+            _wputenv_s(ENVIRONMENT_VARIABLE_NAME_W, L"");
 #else
             unsetenv(ENVIRONMENT_VARIABLE_NAME);
 #endif