From a2be4bb74a25e69b3715d4876d625ad3f38a3e2f Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Fri, 12 Mar 2021 13:49:45 +0100 Subject: [PATCH] Fix deserialization of UTF-8 string for C# object properties Closes #320 --- Source/Engine/Serialization/JsonSerializer.cs | 15 +- .../Serialization/UnmanagedMemoryStream.cs | 157 ++++++++++++++++++ Source/Engine/UI/GUI/Common/TextBox.cs | 2 - 3 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 Source/Engine/Serialization/UnmanagedMemoryStream.cs diff --git a/Source/Engine/Serialization/JsonSerializer.cs b/Source/Engine/Serialization/JsonSerializer.cs index d3300dd23..127fa4d87 100644 --- a/Source/Engine/Serialization/JsonSerializer.cs +++ b/Source/Engine/Serialization/JsonSerializer.cs @@ -165,17 +165,19 @@ namespace FlaxEngine.Json public StringWriter StringWriter; public JsonTextWriter JsonWriter; public JsonSerializerInternalWriter SerializerWriter; - public UnmanagedStringReader StringReader; + public UnmanagedMemoryStream MemoryStream; + public StreamReader Reader; public bool IsDuringSerialization; - public SerializerCache(JsonSerializerSettings settings) + public unsafe SerializerCache(JsonSerializerSettings settings) { JsonSerializer = Newtonsoft.Json.JsonSerializer.CreateDefault(settings); JsonSerializer.Formatting = Formatting.Indented; StringBuilder = new StringBuilder(256); StringWriter = new StringWriter(StringBuilder, CultureInfo.InvariantCulture); SerializerWriter = new JsonSerializerInternalWriter(JsonSerializer); - StringReader = new UnmanagedStringReader(); + MemoryStream = new UnmanagedMemoryStream((byte*)0, 0); + Reader = new StreamReader(MemoryStream, Encoding.UTF8, false); JsonWriter = new JsonTextWriter(StringWriter) { IndentChar = '\t', @@ -404,14 +406,15 @@ namespace FlaxEngine.Json /// The object. /// The input json data buffer (raw, fixed memory buffer). /// The input json data buffer length (characters count). - public static unsafe void Deserialize(object input, void* jsonBuffer, int jsonLength) + public static unsafe void Deserialize(object input, byte* jsonBuffer, int jsonLength) { var cache = Cache.Value; cache.IsDuringSerialization = false; Current.Value = cache; - cache.StringReader.Initialize(jsonBuffer, jsonLength); - var jsonReader = new JsonTextReader(cache.StringReader); + cache.MemoryStream.Initialize(jsonBuffer, jsonLength); + cache.Reader.DiscardBufferedData(); + var jsonReader = new JsonTextReader(cache.Reader); cache.JsonSerializer.Populate(jsonReader, input); if (!cache.JsonSerializer.CheckAdditionalContent) diff --git a/Source/Engine/Serialization/UnmanagedMemoryStream.cs b/Source/Engine/Serialization/UnmanagedMemoryStream.cs new file mode 100644 index 000000000..12b7c4634 --- /dev/null +++ b/Source/Engine/Serialization/UnmanagedMemoryStream.cs @@ -0,0 +1,157 @@ +// Copyright (c) 2012-2021 Wojciech Figat. All rights reserved. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace FlaxEngine.Json +{ + /// + /// Implements a that reads from unmanaged buffer (provided as raw pointer and length). + /// + internal class UnmanagedMemoryStream : Stream + { + private unsafe byte* _ptr; + private int _length; + private int _pos; + private FileAccess _access; + internal bool _isOpen; + private Task _lastReadTask; + + internal UnmanagedMemoryStream() + { + } + + internal unsafe UnmanagedMemoryStream(byte* pointer, int length, FileAccess access = FileAccess.Read) => Initialize(pointer, length, access); + + internal unsafe void Initialize(byte* pointer, int length, FileAccess access = FileAccess.Read) + { + _ptr = pointer; + _length = length; + _pos = 0; + _access = access; + _isOpen = true; + _lastReadTask = null; + } + + public override bool CanRead => _isOpen && (uint)(_access & FileAccess.Read) > 0U; + + public override bool CanSeek => _isOpen; + + public override bool CanWrite => _isOpen && (uint)(_access & FileAccess.Write) > 0U; + + protected override unsafe void Dispose(bool disposing) + { + _isOpen = false; + _ptr = null; + base.Dispose(disposing); + } + + public override void Flush() + { + } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + Flush(); + return Task.FromResult(0); + } + + public unsafe byte* Pointer => _ptr; + + public override long Length => _length; + + public long Capacity => _length; + + public override long Position + { + get => _pos; + set => _pos = (int)value; + } + + public unsafe byte* PositionPointer + { + get => _ptr + _pos; + set => _pos = (int)(value - _ptr); + } + + public override unsafe int Read(byte[] buffer, int offset, int count) + { + int toRead = _length - _pos; + if (toRead > count) + toRead = count; + if (toRead <= 0) + return 0; + fixed (byte* bufferPtr = buffer) + Utils.MemoryCopy(new IntPtr(_ptr + _pos), new IntPtr(bufferPtr), toRead); + _pos += toRead; + return toRead; + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int result = Read(buffer, offset, count); + return _lastReadTask == null || _lastReadTask.Result != result ? (_lastReadTask = Task.FromResult(result)) : _lastReadTask; + } + + public override unsafe int ReadByte() + { + int index = _pos; + if (index >= _length) + return -1; + _pos = index + 1; + return _ptr[index]; + } + + public override long Seek(long offset, SeekOrigin loc) + { + switch (loc) + { + case SeekOrigin.Begin: + _pos = (int)offset; + break; + case SeekOrigin.Current: + _pos += (int)offset; + break; + case SeekOrigin.End: + _pos = _length + (int)offset; + break; + default: throw new ArgumentOutOfRangeException(); + } + return _pos; + } + + public override void SetLength(long value) + { + _length = (int)value; + if (_pos > value) + _pos = _length; + } + + public override unsafe void Write(byte[] buffer, int offset, int count) + { + int newPos = _pos + count; + if (newPos > _length) + _length = newPos; + fixed (byte* bufferPtr = buffer) + Utils.MemoryCopy(new IntPtr(_pos + _pos), new IntPtr(bufferPtr), count); + _pos = newPos; + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + Write(buffer, offset, count); + return Task.FromResult(0); + } + + public override unsafe void WriteByte(byte value) + { + long newPos = _pos + 1; + if (_pos >= _length) + _length = (int)newPos; + _ptr[_pos] = value; + _pos = (int)newPos; + } + } +} diff --git a/Source/Engine/UI/GUI/Common/TextBox.cs b/Source/Engine/UI/GUI/Common/TextBox.cs index a23ae9c93..b1b873f34 100644 --- a/Source/Engine/UI/GUI/Common/TextBox.cs +++ b/Source/Engine/UI/GUI/Common/TextBox.cs @@ -1,7 +1,5 @@ // Copyright (c) 2012-2021 Wojciech Figat. All rights reserved. -using FlaxEngine.Assertions; - namespace FlaxEngine.GUI { ///