From 35aaacd61b5eab42cf93473e95e70b708ba04e1f Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Fri, 16 Apr 2021 15:47:59 +0200 Subject: [PATCH] Add support for importing `.po` files with strings localization --- .../AssetsImportingManager.cpp | 4 + Source/Engine/ContentImporters/CreateJson.cpp | 175 ++++++++++++++++++ Source/Engine/ContentImporters/CreateJson.h | 1 + 3 files changed, 180 insertions(+) diff --git a/Source/Engine/ContentImporters/AssetsImportingManager.cpp b/Source/Engine/ContentImporters/AssetsImportingManager.cpp index c08f9863f..75aa4788d 100644 --- a/Source/Engine/ContentImporters/AssetsImportingManager.cpp +++ b/Source/Engine/ContentImporters/AssetsImportingManager.cpp @@ -31,6 +31,7 @@ #include "CreateParticleEmitterFunction.h" #include "CreateAnimationGraphFunction.h" #include "CreateVisualScript.h" +#include "CreateJson.h" // Tags used to detect asset creation mode const String AssetsImportingManager::CreateTextureTag(TEXT("Texture")); @@ -413,6 +414,9 @@ bool AssetsImportingManagerService::Init() { TEXT("gltf"), ASSET_FILES_EXTENSION, ImportModelFile::Import }, { TEXT("glb"), ASSET_FILES_EXTENSION, ImportModelFile::Import }, + // gettext PO files + { TEXT("po"), TEXT("json"), CreateJson::ImportPo }, + // Models (untested formats - may fail :/) { TEXT("blend"), ASSET_FILES_EXTENSION, ImportModelFile::Import }, { TEXT("bvh"), ASSET_FILES_EXTENSION, ImportModelFile::Import }, diff --git a/Source/Engine/ContentImporters/CreateJson.cpp b/Source/Engine/ContentImporters/CreateJson.cpp index 6141cb03e..85c5af42e 100644 --- a/Source/Engine/ContentImporters/CreateJson.cpp +++ b/Source/Engine/ContentImporters/CreateJson.cpp @@ -9,7 +9,10 @@ #include "Engine/Platform/FileSystem.h" #include "Engine/Content/Content.h" #include "Engine/Content/Storage/JsonStorageProxy.h" +#include "Engine/Content/AssetReference.h" #include "Engine/Serialization/JsonWriters.h" +#include "Engine/Localization/LocalizedStringTable.h" +#include "Engine/Utilities/TextProcessing.h" #include "FlaxEngine.Gen.h" bool CreateJson::Create(const StringView& path, rapidjson_flax::StringBuffer& data, const String& dataTypename) @@ -80,4 +83,176 @@ bool CreateJson::Create(const StringView& path, StringAnsiView& data, StringAnsi return false; } +void FormatPoValue(String& value) +{ + value.Replace(TEXT("\\n"), TEXT("\n")); + value.Replace(TEXT("%s"), TEXT("{}")); + value.Replace(TEXT("%d"), TEXT("{}")); +} + +CreateAssetResult CreateJson::ImportPo(CreateAssetContext& context) +{ + // Base + IMPORT_SETUP(LocalizedStringTable, 1); + + // Load file (UTF-16) + String inputData; + if (File::ReadAllText(context.InputPath, inputData)) + { + return CreateAssetResult::InvalidPath; + } + + // Use virtual asset for data storage and serialization + AssetReference asset = Content::CreateVirtualAsset(); + if (!asset) + return CreateAssetResult::Error; + + // Parse PO format + int32 pos = 0; + int32 pluralCount = 0; + int32 lineNumber = 0; + bool fuzzy = false, hasNewContext = false; + StringView msgctxt, msgid; + String idTmp; + while (pos < inputData.Length()) + { + // Read line + const int32 startPos = pos; + while (pos < inputData.Length() && inputData[pos] != '\n') + pos++; + const StringView line(&inputData[startPos], pos - startPos); + lineNumber++; + pos++; + const int32 valueStart = line.Find('\"') + 1; + const int32 valueEnd = line.FindLast('\"'); + const StringView value(line.Get() + valueStart, Math::Max(valueEnd - valueStart, 0)); + + if (line.StartsWith(StringView(TEXT("msgid_plural")))) + { + // Plural form + } + else if (line.StartsWith(StringView(TEXT("msgid")))) + { + // Id + msgid = value; + + // Reset context if already used + if (!hasNewContext) + msgctxt = StringView(); + hasNewContext = false; + } + else if (line.StartsWith(StringView(TEXT("msgstr")))) + { + // String + if (msgid.HasChars()) + { + // Format message + String msgstr(value); + FormatPoValue(msgstr); + + // Get message id + StringView id = msgid; + if (msgctxt.HasChars()) + { + idTmp = String(msgctxt) + TEXT(".") + String(msgid); + id = idTmp; + } + + int32 indexStart = line.Find('['); + if (indexStart != -1 && indexStart < valueStart) + { + indexStart++; + while (indexStart < line.Length() && StringUtils::IsWhitespace(line[indexStart])) + indexStart++; + int32 indexEnd = line.Find(']'); + while (indexEnd > indexStart && StringUtils::IsWhitespace(line[indexEnd - 1])) + indexEnd--; + int32 index = -1; + StringUtils::Parse(line.Get() + indexStart, (uint32)(indexEnd - indexStart), &index); + if (pluralCount <= 0) + { + LOG(Error, "Missing 'nplurals'. Cannot use plural message at line {0}", lineNumber); + return CreateAssetResult::Error; + } + if (index < 0 || index > pluralCount) + { + LOG(Error, "Invalid plural message index at line {0}", lineNumber); + return CreateAssetResult::Error; + } + + // Plural message + asset->AddPluralString(id, msgstr, index); + } + else + { + // Message + asset->AddString(id, msgstr); + } + } + } + else if (line.StartsWith(StringView(TEXT("msgctxt")))) + { + // Context + msgctxt = value; + hasNewContext = true; + } + else if (line.StartsWith('\"')) + { + // Config + const Char* pluralForms = StringUtils::Find(line.Get(), TEXT("Plural-Forms")); + if (pluralForms != nullptr && pluralForms < line.Get() + line.Length() - 1) + { + // Process plural forms rule + const Char* nplurals = StringUtils::Find(pluralForms, TEXT("nplurals")); + if (nplurals && nplurals < line.Get() + line.Length()) + { + while (*nplurals && *nplurals != '=') + nplurals++; + while (*nplurals && (StringUtils::IsWhitespace(*nplurals) || *nplurals == '=')) + nplurals++; + const Char* npluralsStart = nplurals; + while (*nplurals && !StringUtils::IsWhitespace(*nplurals) && *nplurals != ';') + nplurals++; + StringUtils::Parse(npluralsStart, (uint32)(nplurals - npluralsStart), &pluralCount); + if (pluralCount < 0 || pluralCount > 100) + { + LOG(Error, "Invalid 'nplurals' at line {0}", lineNumber); + return CreateAssetResult::Error; + } + } + // TODO: parse plural forms rule + } + const Char* language = StringUtils::Find(line.Get(), TEXT("Language")); + if (language != nullptr && language < line.Get() + line.Length() - 1) + { + // Process language locale + while (*language && *language != ':') + language++; + language++; + while (*language && StringUtils::IsWhitespace(*language)) + language++; + const Char* languageStart = language; + while (*language && !StringUtils::IsWhitespace(*language) && *language != '\\' && *language != '\"') + language++; + asset->Locale.Set(languageStart, (int32)(language - languageStart)); + if (asset->Locale == TEXT("English")) + asset->Locale = TEXT("en"); + if (asset->Locale.Length() > 5) + LOG(Warning, "Imported .po file uses invalid locale '{0}'", asset->Locale); + } + } + else if (line.StartsWith('#') || line.IsEmpty()) + { + // Comment + const Char* fuzzyPos = StringUtils::Find(line.Get(), TEXT("fuzzy")); + fuzzy |= fuzzyPos != nullptr && fuzzyPos < line.Get() + line.Length() - 1; + } + } + if (asset->Locale.IsEmpty()) + LOG(Warning, "Imported .po file has missing locale"); + + // Save asset + return asset->Save(context.TargetAssetPath) ? CreateAssetResult::CannotSaveFile : CreateAssetResult::Ok; +} + #endif diff --git a/Source/Engine/ContentImporters/CreateJson.h b/Source/Engine/ContentImporters/CreateJson.h index ccc15a792..38ca4b187 100644 --- a/Source/Engine/ContentImporters/CreateJson.h +++ b/Source/Engine/ContentImporters/CreateJson.h @@ -18,6 +18,7 @@ public: static bool Create(const StringView& path, rapidjson_flax::StringBuffer& data, const String& dataTypename); static bool Create(const StringView& path, rapidjson_flax::StringBuffer& data, const char* dataTypename); static bool Create(const StringView& path, StringAnsiView& data, StringAnsiView& dataTypename); + static CreateAssetResult ImportPo(CreateAssetContext& context); }; #endif