From 0c66d18c214cbc862ebfed6d24b7408d19d444e4 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Fri, 27 Feb 2026 22:02:30 -0600 Subject: [PATCH] Wide character support for env vars on windows (#27626) * Base: Move wstring conversion into Base::Tools * App: Switch Windows calls from getenv to GetEnvironmentVariableW * Apply suggestion from @kadet1090 --------- Co-authored-by: Kacper Donat --- src/App/Application.cpp | 41 ++++++++++++++++++++++++++++++++++++++--- src/Base/FileInfo.cpp | 9 ++------- src/Base/Tools.cpp | 34 ++++++++++++++++++++++++++++++++++ src/Base/Tools.h | 16 ++++++++++++++++ 4 files changed, 90 insertions(+), 10 deletions(-) diff --git a/src/App/Application.cpp b/src/App/Application.cpp index 0f86fbf69e..2a357c793b 100644 --- a/src/App/Application.cpp +++ b/src/App/Application.cpp @@ -34,15 +34,18 @@ # define WINVER 0x502 // needed for SetDllDirectory # include # endif + # include # include # include # include # include +# include # include # include # include # include +# include # include # include # include @@ -173,6 +176,38 @@ # include #endif +#ifdef FC_OS_WIN32 +#include +#endif + +std::optional getenvUTF8(const char* name) { +#ifdef FC_OS_WIN32 + int wideLength = MultiByteToWideChar(CP_UTF8, 0, name, -1, nullptr, 0); + std::wstring wideName(wideLength ? wideLength - 1 : 0, L'\0'); + if (wideLength) { + MultiByteToWideChar(CP_UTF8, 0, name, -1, wideName.data(), wideLength); + } + + DWORD needed = GetEnvironmentVariableW(wideName.c_str(), nullptr, 0); + if (needed == 0) { + return std::nullopt; + } + + std::wstring wideValue(needed, L'\0'); + DWORD written = GetEnvironmentVariableW(wideName.c_str(), wideValue.data(), needed); + if (written == 0) { + return std::nullopt; + } + wideValue.resize(written); + return Base::Tools::wstringToString(wideValue); +#else + if (const char* v = std::getenv(name)) { + return std::string(v); + } + return std::nullopt; +#endif +} + FC_LOG_LEVEL_INIT("App", true, true) using namespace App; @@ -2744,9 +2779,9 @@ void Application::initConfig(int argc, char ** argv) void Application::SaveEnv(const char* s) { - const char *c = getenv(s); - if (c) - mConfig[s] = c; + if (auto c = getenvUTF8(s)) { + mConfig[s] = c.value(); + } } void Application::initApplication() diff --git a/src/Base/FileInfo.cpp b/src/Base/FileInfo.cpp index fddedc1c0e..0c7226d928 100644 --- a/src/Base/FileInfo.cpp +++ b/src/Base/FileInfo.cpp @@ -40,6 +40,7 @@ #include "FileInfo.h" #include "Exception.h" #include "TimeInfo.h" +#include "Tools.h" using namespace Base; namespace fs = std::filesystem; @@ -50,13 +51,7 @@ namespace fs = std::filesystem; #ifdef FC_OS_WIN32 std::string ConvertFromWideString(const std::wstring& string) { - int neededSize = WideCharToMultiByte(CP_UTF8, 0, string.c_str(), -1, 0, 0, 0, 0); - char* CharString = new char[static_cast(neededSize)]; - WideCharToMultiByte(CP_UTF8, 0, string.c_str(), -1, CharString, neededSize, 0, 0); - std::string String(CharString); - delete[] CharString; - CharString = NULL; - return String; + return Tools::wstringToString(string); } std::wstring ConvertToWideString(const std::string& string) diff --git a/src/Base/Tools.cpp b/src/Base/Tools.cpp index ba2e715033..9cf0c7193f 100644 --- a/src/Base/Tools.cpp +++ b/src/Base/Tools.cpp @@ -34,6 +34,11 @@ #include "Interpreter.h" #include "Tools.h" +#ifdef FC_OS_WIN32 +# include +# include +#endif + namespace { constexpr auto underscore = static_cast(U'_'); @@ -127,6 +132,35 @@ std::string Base::Tools::narrow(const std::wstring& str) return stm.str(); } +#ifdef FC_OS_WIN32 +std::string Base::Tools::wstringToString(const std::wstring& str) +{ + if (str.empty()) { + return {}; + } + // Use a two-pass WideCharToMultiByte approach, which is the official recommendation as of this + // writing (2026). + int neededSize + = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, str.data(), -1, nullptr, 0, nullptr, nullptr); + char* CharString = new char[static_cast(neededSize)]; + WideCharToMultiByte( + CP_UTF8, + WC_ERR_INVALID_CHARS, + str.data(), + -1, + CharString, + neededSize, + nullptr, + nullptr + ); + std::string String(CharString); + delete[] CharString; + CharString = NULL; + return String; +} +#endif + + std::string Base::Tools::escapedUnicodeFromUtf8(const char* s) { Base::PyGILStateLocker lock; diff --git a/src/Base/Tools.h b/src/Base/Tools.h index 69715f8798..9a5d755eb6 100644 --- a/src/Base/Tools.h +++ b/src/Base/Tools.h @@ -343,7 +343,23 @@ struct BaseExport Tools */ static std::string getIdentifier(const std::string& name); static std::wstring widen(const std::string& str); + + /** + * Locale-dependent, per-character "narrowing" of a std::wstring into a std::string using the + * C++ locale facet std::ctype. Characters outside the locale's representable set get + * replaced with 0, producing embedded NULs (and corrupt the string). Use with caution! Most + * code should prefer wstringToString(). + */ static std::string narrow(const std::wstring& str); + +#ifdef FC_OS_WIN32 + /** + * True UTF-16 to UTF-8 conversion. Handles full Unicode range, including surrogate pairs. Only + * needed on Windows, and internally uses a Win32 API call to do its work. + */ + static std::string wstringToString(const std::wstring& str); +#endif + static std::string escapedUnicodeFromUtf8(const char* s); static std::string escapedUnicodeToUtf8(const std::string& s); static std::string escapeQuotesFromString(const std::string& s);