Wide character support for env vars on windows (#27626)

* Base: Move wstring conversion into Base::Tools

* App: Switch Windows calls from getenv to GetEnvironmentVariableW

* Apply suggestion from @kadet1090

---------

Co-authored-by: Kacper Donat <[email protected]>

src/App/Application.cpp

@@ -34,15 +34,18 @@
 #  define WINVER 0x502 // needed for SetDllDirectory
 #  include <Windows.h>
 # endif
+
 # include <boost/algorithm/string.hpp>
 # include <boost/program_options.hpp>
 # include <boost/date_time/posix_time/posix_time.hpp>
 # include <boost/scope_exit.hpp>
 # include <chrono>
+# include <optional>
 # include <random>
 # include <memory>
 # include <utility>
 # include <set>
+# include <string>
 # include <list>
 # include <algorithm>
 # include <iostream>
@@ -173,6 +176,38 @@
 # include <new>
 #endif
 
+#ifdef FC_OS_WIN32
+#include <windows.h>
+#endif
+
+std::optional<std::string> getenvUTF8(const char* name) {
+#ifdef FC_OS_WIN32
+    int wideLength = MultiByteToWideChar(CP_UTF8, 0, name, -1, nullptr, 0);
+    std::wstring wideName(wideLength ? wideLength - 1 : 0, L'\0');
+    if (wideLength) {
+        MultiByteToWideChar(CP_UTF8, 0, name, -1, wideName.data(), wideLength);
+    }
+
+    DWORD needed = GetEnvironmentVariableW(wideName.c_str(), nullptr, 0);
+    if (needed == 0) {
+        return std::nullopt;
+    }
+
+    std::wstring wideValue(needed, L'\0');
+    DWORD written = GetEnvironmentVariableW(wideName.c_str(), wideValue.data(), needed);
+    if (written == 0) {
+        return std::nullopt;
+    }
+    wideValue.resize(written);
+    return Base::Tools::wstringToString(wideValue);
+#else
+    if (const char* v = std::getenv(name)) {
+        return std::string(v);
+    }
+    return std::nullopt;
+#endif
+}
+
 FC_LOG_LEVEL_INIT("App", true, true)
 
 using namespace App;
@@ -2744,9 +2779,9 @@ void Application::initConfig(int argc, char ** argv)
 
 void Application::SaveEnv(const char* s)
 {
-    const char *c = getenv(s);
-    if (c)
-        mConfig[s] = c;
+    if (auto c = getenvUTF8(s)) {
+        mConfig[s] = c.value();
+    }
 }
 
 void Application::initApplication()

src/Base/FileInfo.cpp

@@ -40,6 +40,7 @@
 #include "FileInfo.h"
 #include "Exception.h"
 #include "TimeInfo.h"
+#include "Tools.h"
 
 using namespace Base;
 namespace fs = std::filesystem;
@@ -50,13 +51,7 @@ namespace fs = std::filesystem;
 #ifdef FC_OS_WIN32
 std::string ConvertFromWideString(const std::wstring& string)
 {
-    int neededSize = WideCharToMultiByte(CP_UTF8, 0, string.c_str(), -1, 0, 0, 0, 0);
-    char* CharString = new char[static_cast<size_t>(neededSize)];
-    WideCharToMultiByte(CP_UTF8, 0, string.c_str(), -1, CharString, neededSize, 0, 0);
-    std::string String(CharString);
-    delete[] CharString;
-    CharString = NULL;
-    return String;
+    return Tools::wstringToString(string);
 }
 
 std::wstring ConvertToWideString(const std::string& string)

src/Base/Tools.cpp

@@ -34,6 +34,11 @@
 #include "Interpreter.h"
 #include "Tools.h"
 
+#ifdef FC_OS_WIN32
+# include <windows.h>
+# include <stdexcept>
+#endif
+
 namespace
 {
 constexpr auto underscore = static_cast<UChar32>(U'_');
@@ -127,6 +132,35 @@ std::string Base::Tools::narrow(const std::wstring& str)
     return stm.str();
 }
 
+#ifdef FC_OS_WIN32
+std::string Base::Tools::wstringToString(const std::wstring& str)
+{
+    if (str.empty()) {
+        return {};
+    }
+    // Use a two-pass WideCharToMultiByte approach, which is the official recommendation as of this
+    // writing (2026).
+    int neededSize
+        = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, str.data(), -1, nullptr, 0, nullptr, nullptr);
+    char* CharString = new char[static_cast<size_t>(neededSize)];
+    WideCharToMultiByte(
+        CP_UTF8,
+        WC_ERR_INVALID_CHARS,
+        str.data(),
+        -1,
+        CharString,
+        neededSize,
+        nullptr,
+        nullptr
+    );
+    std::string String(CharString);
+    delete[] CharString;
+    CharString = NULL;
+    return String;
+}
+#endif
+
+
 std::string Base::Tools::escapedUnicodeFromUtf8(const char* s)
 {
     Base::PyGILStateLocker lock;

src/Base/Tools.h

@@ -343,7 +343,23 @@ struct BaseExport Tools
      */
     static std::string getIdentifier(const std::string& name);
     static std::wstring widen(const std::string& str);
+
+    /**
+     * Locale-dependent, per-character "narrowing" of a std::wstring into a std::string using the
+     * C++ locale facet std::ctype<char>. Characters outside the locale's representable set get
+     * replaced with 0, producing embedded NULs (and corrupt the string). Use with caution! Most
+     * code should prefer wstringToString().
+     */
     static std::string narrow(const std::wstring& str);
+
+#ifdef FC_OS_WIN32
+    /**
+     * True UTF-16 to UTF-8 conversion. Handles full Unicode range, including surrogate pairs. Only
+     * needed on Windows, and internally uses a Win32 API call to do its work.
+     */
+    static std::string wstringToString(const std::wstring& str);
+#endif
+
     static std::string escapedUnicodeFromUtf8(const char* s);
     static std::string escapedUnicodeToUtf8(const std::string& s);
     static std::string escapeQuotesFromString(const std::string& s);