GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix UTF-8 and UTF-16 encoding support usage in Json resources

Browse Source 
Closes #310
Closes #27
This commit is contained in:
Wojtek Figat
2021-03-07 21:30:01 +01:00
parent 400cc97100
commit 77b21de534
16 changed files with 186 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Source/Editor/Managed/ManagedEditor.Internal.cpp
View File

@@ -578,15 +578,11 @@ public:
MUtils::ToString(outputPathObj, outputPath);
FileSystem::NormalizePath(outputPath);
DataContainer<char> data;
const auto dataObjLength = mono_string_length(dataObj);
const auto dataObjPtr = mono_string_to_utf8(dataObj);
data.Link(dataObjPtr, dataObjLength);
StringAnsiView data(dataObjPtr);
DataContainer<char> dataTypeName;
const auto dataTypeNameObjLength = mono_string_length(dataTypeNameObj);
const auto dataTypeNameObjPtr = mono_string_to_utf8(dataTypeNameObj);
dataTypeName.Link(dataTypeNameObjPtr, dataTypeNameObjLength);
StringAnsiView dataTypeName(dataTypeNameObjPtr);
const bool result = CreateJson::Create(outputPath, data, dataTypeName);

12
Source/Engine/Content/Storage/JsonStorageProxy.cpp
View File

@@ -3,10 +3,10 @@
#include "JsonStorageProxy.h"
#include "Engine/Platform/File.h"
#include "Engine/Core/Log.h"
#include "Engine/Level/Types.h"
#include "Engine/Core/Utilities.h"
#include "Engine/Serialization/JsonTools.h"
#include "Engine/Serialization/JsonWriters.h"
#include "Engine/Level/Types.h"
#include "Engine/Debug/Exceptions/JsonParseException.h"
#include <ThirdParty/rapidjson/document.h>
@@ -17,10 +17,6 @@ bool JsonStorageProxy::IsValidExtension(const StringView& extension)
bool JsonStorageProxy::GetAssetInfo(const StringView& path, Guid& resultId, String& resultDataTypeName)
{
// TODO:
#if USE_EDITOR || true
// TODO: we could just open file and start reading until we find 'ID:..' without parsing whole file - could be much more faster
// Load file
@@ -51,12 +47,6 @@ bool JsonStorageProxy::GetAssetInfo(const StringView& path, Guid& resultId, Stri
}
return false;
#else
todo_loading_json_resources_in_builded_version
#endif
}
#if USE_EDITOR

10
Source/Engine/ContentImporters/CreateJson.cpp
View File

@@ -20,16 +20,12 @@ bool CreateJson::Create(const StringView& path, rapidjson_flax::StringBuffer& da
bool CreateJson::Create(const StringView& path, rapidjson_flax::StringBuffer& data, const char* dataTypename)
{
DataContainer<char> data1;
DataContainer<char> data2;
data1.Link((char*)data.GetString(), (int32)data.GetSize());
data2.Link((char*)dataTypename, StringUtils::Length(dataTypename));
StringAnsiView data1((char*)data.GetString(), (int32)data.GetSize());
StringAnsiView data2((char*)dataTypename, StringUtils::Length(dataTypename));
return Create(path, data1, data2);
}
bool CreateJson::Create(const StringView& path, DataContainer<char>& data, DataContainer<char>& dataTypename)
bool CreateJson::Create(const StringView& path, StringAnsiView& data, StringAnsiView& dataTypename)
{
Guid id = Guid::New();

4
Source/Engine/ContentImporters/CreateJson.h
View File

@@ -9,7 +9,7 @@
#include "Engine/Serialization/Json.h"
/// <summary>
/// Json resources factory.
/// Json resources factory. Ensure to keep data encoded in UTF-8.
/// </summary>
class CreateJson
{
@@ -17,7 +17,7 @@ public:
static bool Create(const StringView& path, rapidjson_flax::StringBuffer& data, const String& dataTypename);
static bool Create(const StringView& path, rapidjson_flax::StringBuffer& data, const char* dataTypename);
static bool Create(const StringView& path, DataContainer<char>& data, DataContainer<char>& dataTypename);
static bool Create(const StringView& path, StringAnsiView& data, StringAnsiView& dataTypename);
};
#endif

100
Source/Engine/Core/Types/String.cpp
View File

@@ -63,6 +63,12 @@ void String::Set(const char* chars, int32 length)
StringUtils::ConvertANSI2UTF16(chars, _data, length);
}
void String::SetUTF8(const char* chars, int32 length)
{
Platform::Free(_data);
_data = StringUtils::ConvertUTF82UTF16(chars, length, _length);
}
void String::Append(const Char* chars, int32 count)
{
if (count == 0)
@@ -338,6 +344,84 @@ StringAnsi::StringAnsi(const StringAnsiView& str)
Set(str.Get(), str.Length());
}
void StringAnsi::Set(const char* chars, int32 length)
{
if (length != _length)
{
ASSERT(length >= 0);
Platform::Free(_data);
if (length != 0)
{
_data = (char*)Platform::Allocate((length + 1) * sizeof(char), 16);
_data[length] = 0;
}
else
{
_data = nullptr;
}
_length = length;
}
Platform::MemoryCopy(_data, chars, length * sizeof(char));
}
void StringAnsi::Set(const Char* chars, int32 length)
{
if (length != _length)
{
Platform::Free(_data);
if (length != 0)
{
_data = (char*)Platform::Allocate((length + 1) * sizeof(char), 16);
_data[length] = 0;
}
else
{
_data = nullptr;
}
_length = length;
}
if (_data)
StringUtils::ConvertUTF162ANSI(chars, _data, length);
}
void StringAnsi::Append(const char* chars, int32 count)
{
if (count == 0)
return;
const auto oldData = _data;
const auto oldLength = _length;
_length = oldLength + count;
_data = (char*)Platform::Allocate((_length + 1) * sizeof(char), 16);
Platform::MemoryCopy(_data, oldData, oldLength * sizeof(char));
Platform::MemoryCopy(_data + oldLength, chars, count * sizeof(char));
_data[_length] = 0;
Platform::Free(oldData);
}
void StringAnsi::Append(const Char* chars, int32 count)
{
if (count == 0)
return;
const auto oldData = _data;
const auto oldLength = _length;
_length = oldLength + count;
_data = (char*)Platform::Allocate((_length + 1) * sizeof(char), 16);
Platform::MemoryCopy(_data, oldData, oldLength * sizeof(char));
StringUtils::ConvertUTF162ANSI(chars, _data + oldLength, count * sizeof(char));
_data[_length] = 0;
Platform::Free(oldData);
}
StringAnsi& StringAnsi::operator+=(const StringAnsiView& str)
{
Append(str.Get(), str.Length());
@@ -368,6 +452,22 @@ bool StringAnsi::EndsWith(const StringAnsiView& suffix, StringSearchCase searchC
return !StringUtils::Compare(&(*this)[Length() - suffix.Length()], *suffix);
}
StringAnsi StringAnsi::ToLower() const
{
StringAnsi result(*this);
for (int32 i = 0; i < result.Length(); i++)
result[i] = StringUtils::ToLower(result[i]);
return result;
}
StringAnsi StringAnsi::ToUpper() const
{
StringAnsi result(*this);
for (int32 i = 0; i < result.Length(); i++)
result[i] = StringUtils::ToUpper(result[i]);
return result;
}
void StringAnsi::Insert(int32 startIndex, const StringAnsi& other)
{
ASSERT(other._data != _data);

105
Source/Engine/Core/Types/String.h
View File

@@ -628,6 +628,13 @@ public:
/// <param name="length">The number of characters to assign.</param>
void Set(const char* chars, int32 length);
/// <summary>
/// Sets an array of characters to the string.
/// </summary>
/// <param name="chars">The pointer to the start of an array of characters to set (UTF-8). This array need not be null-terminated, and null characters are not treated specially.</param>
/// <param name="length">The number of characters to assign.</param>
void SetUTF8(const char* chars, int32 length);
/// <summary>
/// Appends an array of characters to the string.
/// </summary>
@@ -1306,100 +1313,30 @@ public:
/// <summary>
/// Sets an array of characters to the string.
/// </summary>
/// <param name="chars">The pointer to the start of an array of characters to set. This array need not be null-terminated, and null characters are not treated specially.</param>
/// <param name="chars">The pointer to the start of an array of characters to set (ANSI). This array need not be null-terminated, and null characters are not treated specially.</param>
/// <param name="length">The number of characters to assign.</param>
void Set(const char* chars, int32 length)
{
if (length != _length)
{
ASSERT(length >= 0);
Platform::Free(_data);
if (length != 0)
{
_data = (char*)Platform::Allocate((length + 1) * sizeof(char), 16);
_data[length] = 0;
}
else
{
_data = nullptr;
}
_length = length;
}
Platform::MemoryCopy(_data, chars, length * sizeof(char));
}
void Set(const char* chars, int32 length);
/// <summary>
/// Sets an array of characters to the string.
/// </summary>
/// <param name="chars">The pointer to the start of an array of characters to set. This array need not be null-terminated, and null characters are not treated specially.</param>
/// <param name="chars">The pointer to the start of an array of characters to set (UTF-16). This array need not be null-terminated, and null characters are not treated specially.</param>
/// <param name="length">The number of characters to assign.</param>
void Set(const Char* chars, int32 length)
{
if (length != _length)
{
Platform::Free(_data);
if (length != 0)
{
_data = (char*)Platform::Allocate((length + 1) * sizeof(char), 16);
_data[length] = 0;
}
else
{
_data = nullptr;
}
_length = length;
}
if (_data)
StringUtils::ConvertUTF162ANSI(chars, _data, length);
}
void Set(const Char* chars, int32 length);
/// <summary>
/// Appends an array of characters to the string.
/// </summary>
/// <param name="chars">The array of characters to append. It does not need be null-terminated, and null characters are not treated specially.</param>
/// <param name="count">The number of characters to append.</param>
void Append(const char* chars, int32 count)
{
if (count == 0)
return;
const auto oldData = _data;
const auto oldLength = _length;
_length = oldLength + count;
_data = (char*)Platform::Allocate((_length + 1) * sizeof(char), 16);
Platform::MemoryCopy(_data, oldData, oldLength * sizeof(char));
Platform::MemoryCopy(_data + oldLength, chars, count * sizeof(char));
_data[_length] = 0;
Platform::Free(oldData);
}
void Append(const char* chars, int32 count);
/// <summary>
/// Appends an array of characters to the string.
/// </summary>
/// <param name="chars">The array of characters to append. It does not need be null-terminated, and null characters are not treated specially.</param>
/// <param name="count">The number of characters to append.</param>
void Append(const Char* chars, int32 count)
{
if (count == 0)
return;
const auto oldData = _data;
const auto oldLength = _length;
_length = oldLength + count;
_data = (char*)Platform::Allocate((_length + 1) * sizeof(char), 16);
Platform::MemoryCopy(_data, oldData, oldLength * sizeof(char));
StringUtils::ConvertUTF162ANSI(chars, _data + oldLength, count * sizeof(char));
_data[_length] = 0;
Platform::Free(oldData);
}
void Append(const Char* chars, int32 count);
/// <summary>
/// Appends the specified text to this string.
@@ -1663,25 +1600,13 @@ public:
/// Converts all uppercase characters to lowercase.
/// </summary>
/// <returns>The lowercase string.</returns>
StringAnsi ToLower() const
{
StringAnsi result(*this);
for (int32 i = 0; i < result.Length(); i++)
result[i] = StringUtils::ToLower(result[i]);
return result;
}
StringAnsi ToLower() const;
/// <summary>
/// Converts all lowercase characters to uppercase.
/// </summary>
/// <returns>The uppercase string.</returns>
StringAnsi ToUpper() const
{
StringAnsi result(*this);
for (int32 i = 0; i < result.Length(); i++)
result[i] = StringUtils::ToUpper(result[i]);
return result;
}
StringAnsi ToUpper() const;
/// <summary>
/// Gets the left most given number of characters.

5
Source/Engine/Level/Actor.cpp
View File

@@ -1710,7 +1710,10 @@ String Actor::ToJson()
rapidjson_flax::StringBuffer buffer;
CompactJsonWriter writer(buffer);
writer.SceneObject(this);
return String(buffer.GetString());
String result;
const char* c = buffer.GetString();
result.SetUTF8(c, (int32)buffer.GetSize());
return result;
}
void Actor::FromJson(const StringAnsiView& json)

7
Source/Engine/Platform/Base/FileBase.cpp
View File

@@ -113,10 +113,9 @@ bool FileBase::ReadAllText(const StringView& path, String& data)
}
// Convert to UTF-16
auto utf16Data = (Char*)Allocator::Allocate(count * sizeof(Char));
uint32 utf16Length;
StringUtils::ConvertUTF82UTF16(reinterpret_cast<char*>(bytes.Get()), utf16Data, count, &utf16Length);
data = utf16Data;
int32 utf16Length;
Char* utf16Data = StringUtils::ConvertUTF82UTF16(reinterpret_cast<char*>(bytes.Get()), count, utf16Length);
data.Set(utf16Data, utf16Length);
Allocator::Free(utf16Data);
}
break;

4
Source/Engine/Platform/Win32/Win32Network.cpp
View File

@@ -442,13 +442,13 @@ bool Win32Network::CreateEndPoint(String* address, String* port, NetworkIPVersio
// consider using NUMERICHOST/NUMERICSERV if address is a valid Ipv4 or IPv6 so we can skip some look up ( potentially slow when resolving host names )
if ((status = GetAddrInfoW(address == nullptr ? nullptr : address->Get(), port->Get(), &hints, &info)) != 0)
{
LOG(Error, "Unable to query info for address : {0} Error : {1}", address ? address->Get() : String("ANY"), gai_strerror(status));
LOG(Error, "Unable to query info for address : {0} Error : {1}", address ? address->Get() : TEXT("ANY"), gai_strerror(status));
return true;
}
if (info == nullptr)
{
LOG(Error, "Unable to resolve address! Address : {0}", address ? address->Get() : String("ANY"));
LOG(Error, "Unable to resolve address! Address : {0}", address ? address->Get() : TEXT("ANY"));
return true;
}

6
Source/Engine/Scripting/ManagedSerialization.cpp
View File

@@ -41,9 +41,8 @@ void ManagedSerialization::Serialize(ISerializable::SerializeStream& stream, Mon
}
// Write result data
const auto length = mono_string_length(invokeResultStr);
const auto invokeResultChars = mono_string_to_utf8(invokeResultStr);
stream.RawValue(invokeResultChars, length);
stream.RawValue(invokeResultChars);
mono_free(invokeResultChars);
}
@@ -80,9 +79,8 @@ void ManagedSerialization::SerializeDiff(ISerializable::SerializeStream& stream,
}
// Write result data
auto length = mono_string_length(invokeResultStr);
auto invokeResultChars = mono_string_to_utf8(invokeResultStr);
stream.RawValue(invokeResultChars, length);
stream.RawValue(invokeResultChars);
mono_free(invokeResultChars);
}

4
Source/Engine/Serialization/JsonTools.h
View File

@@ -358,9 +358,9 @@ public:
FORCE_INLINE static void GetString(String& result, const Value& node, const char* name)
{
const auto member = node.FindMember(name);
if (member != node.MemberEnd() && member->value.IsString())
if (member != node.MemberEnd())
{
result.Set(member->value.GetString(), member->value.GetStringLength());
result = member->value.GetText();
}
}

13
Source/Engine/Serialization/JsonWriter.h
View File

@@ -73,21 +73,20 @@ public:
void String(const Char* str)
{
const int32 length = StringUtils::Length(str);
const StringAsANSI<256> buf(str, length);
String(buf.Get(), length);
const StringAsUTF8<256> buf(str);
String(buf.Get());
}
void String(const Char* str, const int32 length)
{
const StringAsANSI<256> buf(str, length);
String(buf.Get(), length);
const StringAsUTF8<256> buf(str, length);
String(buf.Get());
}
void String(const ::String& value)
{
const StringAsANSI<256> buf(*value, value.Length());
String(buf.Get(), value.Length());
const StringAsUTF8<256> buf(*value, value.Length());
String(buf.Get());
}
FORCE_INLINE void RawValue(const StringAnsi& str)

3
Source/Engine/UI/UICanvas.cpp
View File

@@ -81,9 +81,8 @@ void UICanvas::Serialize(SerializeStream& stream, const void* otherObj)
else
{
// Write result data
auto length = mono_string_length(invokeResultStr);
auto invokeResultChars = mono_string_to_utf8(invokeResultStr);
stream.RawValue(invokeResultChars, length);
stream.RawValue(invokeResultChars);
mono_free(invokeResultChars);
}
}

5
Source/Engine/UI/UIControl.cpp
View File

@@ -99,14 +99,13 @@ void UIControl::Serialize(SerializeStream& stream, const void* otherObj)
{
stream.JKEY("Control");
const auto controlTypeChars = mono_string_to_utf8(controlType);
stream.String(controlTypeChars, controlTypeLength);
stream.String(controlTypeChars);
mono_free(controlTypeChars);
}
stream.JKEY("Data");
const auto invokeResultLength = mono_string_length(invokeResultStr);
const auto invokeResultChars = mono_string_to_utf8(invokeResultStr);
stream.RawValue(invokeResultChars, invokeResultLength);
stream.RawValue(invokeResultChars);
mono_free(invokeResultChars);
}

63
Source/Engine/Utilities/StringConverter.h
View File

@@ -50,19 +50,8 @@ public:
}
StringAsANSI(const Char* text)
: StringAsANSI(text, StringUtils::Length(text))
{
const int32 length = StringUtils::Length(text);
if (length + 1 < InlinedSize)
{
StringUtils::ConvertUTF162ANSI(text, this->_inlined, length);
this->_inlined[length] = 0;
}
else
{
this->_dynamic = (CharType*)Allocator::Allocate((length + 1) * sizeof(CharType));
StringUtils::ConvertUTF162ANSI(text, this->_dynamic, length);
this->_dynamic[length] = 0;
}
}
StringAsANSI(const Char* text, const int32 length)
@@ -81,6 +70,43 @@ public:
}
};
template<int InlinedSize = 128>
class StringAsUTF8 : public StringAsBase<char>
{
public:
typedef char CharType;
typedef StringAsBase<CharType, InlinedSize> Base;
public:
StringAsUTF8(const char* text)
{
this->_static = text;
}
StringAsUTF8(const Char* text)
: StringAsUTF8(text, StringUtils::Length(text))
{
}
StringAsUTF8(const Char* text, const int32 length)
{
if (length + 1 < InlinedSize)
{
int32 lengthUtf8;
StringUtils::ConvertUTF162UTF8(text, this->_inlined, length, lengthUtf8);
this->_inlined[lengthUtf8] = 0;
}
else
{
int32 lengthUtf8;
this->_dynamic = StringUtils::ConvertUTF162UTF8(text, length, lengthUtf8);
this->_dynamic[lengthUtf8] = 0;
}
}
};
template<int InlinedSize = 128>
class StringAsUTF16 : public StringAsBase<Char>
{
@@ -92,19 +118,8 @@ public:
public:
StringAsUTF16(const char* text)
: StringAsUTF16(text, StringUtils::Length(text))
{
const int32 length = StringUtils::Length(text);
if (length + 1 < InlinedSize)
{
StringUtils::ConvertANSI2UTF16(text, this->_inlined, length);
this->_inlined[length] = 0;
}
else
{
this->_dynamic = (CharType*)Allocator::Allocate((length + 1) * sizeof(CharType));
StringUtils::ConvertANSI2UTF16(text, this->_dynamic, length);
this->_dynamic[length] = 0;
}
}
StringAsUTF16(const char* text, const int32 length)

4
Source/ThirdParty/rapidjson/document.h vendored
View File

@@ -1685,9 +1685,9 @@ public:
if (IsString())
{
if (data_.f.flags & kInlineStrFlag)
result.Set(data_.ss.str, data_.ss.GetLength());
result.SetUTF8(data_.ss.str, data_.ss.GetLength());
else
result.Set(GetStringPointer(), data_.s.length);
result.SetUTF8(GetStringPointer(), data_.s.length);
}
return result;
}