GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix StringUtils::ConvertANSI2UTF16 to properly handle multi-byte characters Length

Browse Source 
#1225
This commit is contained in:
Wojtek Figat
2023-07-03 12:10:06 +02:00
parent 9a1fd12a85
commit 9d640656e6
9 changed files with 26 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Source/Engine/Core/Types/String.cpp
View File

@@ -73,7 +73,7 @@ void String::Set(const char* chars, int32 length)
_length = length;
}
if (chars)
StringUtils::ConvertANSI2UTF16(chars, _data, length);
StringUtils::ConvertANSI2UTF16(chars, _data, length, _length);
}
void String::SetUTF8(const char* chars, int32 length)
@@ -112,7 +112,7 @@ void String::Append(const char* chars, int32 count)
_data = (Char*)Platform::Allocate((_length + 1) * sizeof(Char), 16);
Platform::MemoryCopy(_data, oldData, oldLength * sizeof(Char));
StringUtils::ConvertANSI2UTF16(chars, _data + oldLength, count * sizeof(Char));
StringUtils::ConvertANSI2UTF16(chars, _data + oldLength, count, _length);
_data[_length] = 0;
Platform::Free(oldData);

3
Source/Engine/Core/Types/StringBuilder.h
View File

@@ -125,7 +125,8 @@ public:
const int32 length = str && *str ? StringUtils::Length(str) : 0;
const int32 prevCnt = _data.Count();
_data.AddDefault(length);
StringUtils::ConvertANSI2UTF16(str, _data.Get() + prevCnt, length);
int32 tmp;
StringUtils::ConvertANSI2UTF16(str, _data.Get() + prevCnt, length, tmp);
return *this;
}

6
Source/Engine/Core/Types/Variant.cpp
View File

@@ -660,7 +660,8 @@ Variant::Variant(const StringAnsiView& v)
const int32 length = v.Length() * sizeof(Char) + 2;
AsBlob.Data = Allocator::Allocate(length);
AsBlob.Length = length;
StringUtils::ConvertANSI2UTF16(v.Get(), (Char*)AsBlob.Data, v.Length());
int32 tmp;
StringUtils::ConvertANSI2UTF16(v.Get(), (Char*)AsBlob.Data, v.Length(), tmp);
((Char*)AsBlob.Data)[v.Length()] = 0;
}
else
@@ -2578,7 +2579,8 @@ void Variant::SetString(const StringAnsiView& str)
AsBlob.Data = Allocator::Allocate(length);
AsBlob.Length = length;
}
StringUtils::ConvertANSI2UTF16(str.Get(), (Char*)AsBlob.Data, str.Length());
int32 tmp;
StringUtils::ConvertANSI2UTF16(str.Get(), (Char*)AsBlob.Data, str.Length(), tmp);
((Char*)AsBlob.Data)[str.Length()] = 0;
}

2
Source/Engine/Graphics/Shaders/GPUShader.cpp
View File

@@ -145,7 +145,7 @@ bool GPUShader::Create(MemoryReadStream& stream)
// Create CB
#if GPU_ENABLE_RESOURCE_NAMING
String name = ToString() + TEXT(".CB") + i;
String name = String::Format(TEXT("{}.CB{}"), ToString(), i);
#else
String name;
#endif

2
Source/Engine/Platform/StringUtils.h
View File

@@ -179,7 +179,7 @@ public:
public:
// Converts characters from ANSI to UTF-16
static void ConvertANSI2UTF16(const char* from, Char* to, int32 len);
static void ConvertANSI2UTF16(const char* from, Char* to, int32 fromLength, int32& toLength);
// Converts characters from UTF-16 to ANSI
static void ConvertUTF162ANSI(const Char* from, char* to, int32 len);

7
Source/Engine/Platform/Unix/UnixStringUtils.cpp
View File

@@ -311,14 +311,14 @@ static inline uint32 Utf8ToUtf32Codepoint(const char* src, int32 length)
}
}
void StringUtils::ConvertANSI2UTF16(const char* from, Char* to, int32 len)
void StringUtils::ConvertANSI2UTF16(const char* from, Char* to, int32 fromLength, int32& toLength)
{
const char* const u8end = from + len;
const char* const u8end = from + fromLength;
const char* u8cur = from;
char16_t* u16cur = to;
while (u8cur < u8end)
{
len = Utf8CodepointLength(*u8cur);
int32 len = Utf8CodepointLength(*u8cur);
uint32 codepoint = Utf8ToUtf32Codepoint(u8cur, len);
// Convert the UTF32 codepoint to one or more UTF16 codepoints
@@ -336,6 +336,7 @@ void StringUtils::ConvertANSI2UTF16(const char* from, Char* to, int32 len)
}
u8cur += len;
}
toLength = (int32)(u16cur - to);
}
static const char32_t kByteMask = 0x000000BF;

8
Source/Engine/Platform/Win32/Win32StringUtils.cpp
View File

@@ -179,10 +179,12 @@ const char* StringUtils::Find(const char* str, const char* toFind)
return strstr(str, toFind);
}
void StringUtils::ConvertANSI2UTF16(const char* from, Char* to, int32 len)
void StringUtils::ConvertANSI2UTF16(const char* from, Char* to, int32 fromLength, int32& toLength)
{
if (len)
mbstowcs(to, from, len);
if (fromLength)
toLength = mbstowcs(to, from, fromLength);
else
toLength = 0;
}
void StringUtils::ConvertUTF162ANSI(const Char* from, char* to, int32 len)

3
Source/Engine/Platform/Windows/WindowsPlatform.cpp
View File

@@ -992,7 +992,8 @@ void ReadPipe(HANDLE pipe, Array<char>& rawData, Array<Char>& logData, LogType l
// Log contents
logData.Clear();
logData.Resize(rawData.Count() + 1);
StringUtils::ConvertANSI2UTF16(rawData.Get(), logData.Get(), rawData.Count());
int32 tmp;
StringUtils::ConvertANSI2UTF16(rawData.Get(), logData.Get(), rawData.Count(), tmp);
logData.Last() = '\0';
if (settings.LogOutput)
Log::Logger::Write(logType, StringView(logData.Get(), rawData.Count()));

10
Source/Engine/Utilities/StringConverter.h
View File

@@ -49,7 +49,7 @@ public:
{
}
StringAsANSI(const Char* text, const int32 length)
StringAsANSI(const Char* text, int32 length)
{
if (length + 1 < InlinedSize)
{
@@ -83,7 +83,7 @@ public:
{
}
StringAsUTF8(const Char* text, const int32 length)
StringAsUTF8(const Char* text, int32 length)
{
int32 lengthUtf8;
if (length + 1 < InlinedSize)
@@ -112,17 +112,17 @@ public:
{
}
StringAsUTF16(const char* text, const int32 length)
StringAsUTF16(const char* text, int32 length)
{
if (length + 1 < InlinedSize)
{
StringUtils::ConvertANSI2UTF16(text, this->_inlined, length);
StringUtils::ConvertANSI2UTF16(text, this->_inlined, length, length);
this->_inlined[length] = 0;
}
else
{
this->_dynamic = (CharType*)Allocator::Allocate((length + 1) * sizeof(CharType));
StringUtils::ConvertANSI2UTF16(text, this->_dynamic, length);
StringUtils::ConvertANSI2UTF16(text, this->_dynamic, length, length);
this->_dynamic[length] = 0;
}
}