From b2550e9522a59ae6cf6628f8c5965df2354e1904 Mon Sep 17 00:00:00 2001 From: Gymnasiast Date: Sat, 21 Jul 2018 22:41:46 +0200 Subject: [PATCH 1/7] Fix RTL rendering (WIP) --- src/openrct2/drawing/Drawing.String.cpp | 29 ++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/openrct2/drawing/Drawing.String.cpp b/src/openrct2/drawing/Drawing.String.cpp index 219f25d453..633be58b9c 100644 --- a/src/openrct2/drawing/Drawing.String.cpp +++ b/src/openrct2/drawing/Drawing.String.cpp @@ -19,6 +19,12 @@ #include "TTF.h" #include +#include +#include +#include +#include +#include +#include enum : uint32_t { @@ -813,7 +819,28 @@ static const utf8* ttf_process_glyph_run(rct_drawpixelinfo* dpi, const utf8* tex static void ttf_process_string(rct_drawpixelinfo* dpi, const utf8* text, text_draw_info* info) { - const utf8* ch = text; + UErrorCode err = (UErrorCode)0; + + UnicodeString ustr = UnicodeString::fromUTF8(StringPiece(text)); + + int32_t length = ustr.length(); + UnicodeString reordered; + UnicodeString shaped; + UBiDi* bidi = ubidi_openSized(length, 0, &err); + ubidi_setPara(bidi, ustr.getBuffer(), length, UBIDI_DEFAULT_LTR, nullptr, &err); + ubidi_writeReordered(bidi, reordered.getBuffer(length), length, UBIDI_DO_MIRRORING, &err); + ubidi_close(bidi); + reordered.releaseBuffer(length); + u_shapeArabic( + reordered.getBuffer(), length, shaped.getBuffer(length), length, + U_SHAPE_LETTERS_SHAPE | U_SHAPE_LENGTH_FIXED_SPACES_NEAR | U_SHAPE_TEXT_DIRECTION_VISUAL_LTR, &err); + shaped.releaseBuffer(length); + + std::string cppstring; + shaped.toUTF8String(cppstring); + + const utf8* utf8c = cppstring.c_str(); + const utf8* ch = utf8c; const utf8* nextCh; int32_t codepoint; From 87edac1e8575b44537e1b8e6234e7f4d17bfe137 Mon Sep 17 00:00:00 2001 From: Gymnasiast Date: Sun, 22 Jul 2018 00:12:16 +0200 Subject: [PATCH 2/7] Fix word order in mixed strings --- src/openrct2/drawing/Drawing.String.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/openrct2/drawing/Drawing.String.cpp b/src/openrct2/drawing/Drawing.String.cpp index 633be58b9c..8420ee5766 100644 --- a/src/openrct2/drawing/Drawing.String.cpp +++ b/src/openrct2/drawing/Drawing.String.cpp @@ -820,15 +820,17 @@ static const utf8* ttf_process_glyph_run(rct_drawpixelinfo* dpi, const utf8* tex static void ttf_process_string(rct_drawpixelinfo* dpi, const utf8* text, text_draw_info* info) { UErrorCode err = (UErrorCode)0; + // Force a hard left-to-right at the beginning (will mess up mixed strings' word order otherwise) + std::string text2 = std::string(u8"\xE2\x80\xAA") + text; - UnicodeString ustr = UnicodeString::fromUTF8(StringPiece(text)); + UnicodeString ustr = UnicodeString::fromUTF8(StringPiece(text2)); int32_t length = ustr.length(); UnicodeString reordered; UnicodeString shaped; UBiDi* bidi = ubidi_openSized(length, 0, &err); ubidi_setPara(bidi, ustr.getBuffer(), length, UBIDI_DEFAULT_LTR, nullptr, &err); - ubidi_writeReordered(bidi, reordered.getBuffer(length), length, UBIDI_DO_MIRRORING, &err); + ubidi_writeReordered(bidi, reordered.getBuffer(length), length, UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS, &err); ubidi_close(bidi); reordered.releaseBuffer(length); u_shapeArabic( From 11ed9418fcf8903ca2fa9de9c0054f3bc4bc9156 Mon Sep 17 00:00:00 2001 From: Michael Steenbeek Date: Tue, 24 Jul 2018 09:50:56 +0200 Subject: [PATCH 3/7] Use ICU namespace (fixes compilation on macOS) --- src/openrct2/drawing/Drawing.String.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openrct2/drawing/Drawing.String.cpp b/src/openrct2/drawing/Drawing.String.cpp index 8420ee5766..bfd78c2880 100644 --- a/src/openrct2/drawing/Drawing.String.cpp +++ b/src/openrct2/drawing/Drawing.String.cpp @@ -823,11 +823,11 @@ static void ttf_process_string(rct_drawpixelinfo* dpi, const utf8* text, text_dr // Force a hard left-to-right at the beginning (will mess up mixed strings' word order otherwise) std::string text2 = std::string(u8"\xE2\x80\xAA") + text; - UnicodeString ustr = UnicodeString::fromUTF8(StringPiece(text2)); + icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(text2)); int32_t length = ustr.length(); - UnicodeString reordered; - UnicodeString shaped; + icu::UnicodeString reordered; + icu::UnicodeString shaped; UBiDi* bidi = ubidi_openSized(length, 0, &err); ubidi_setPara(bidi, ustr.getBuffer(), length, UBIDI_DEFAULT_LTR, nullptr, &err); ubidi_writeReordered(bidi, reordered.getBuffer(length), length, UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS, &err); From 0ad2c74f0a337814d06b425b48499d9e57f2867f Mon Sep 17 00:00:00 2001 From: Michael Steenbeek Date: Thu, 26 Jul 2018 13:48:45 +0200 Subject: [PATCH 4/7] Do RTL fixes when loading language pack into memory --- src/openrct2/drawing/Drawing.String.cpp | 31 +------------ src/openrct2/localisation/Language.cpp | 48 ++++++++++---------- src/openrct2/localisation/Language.h | 2 +- src/openrct2/localisation/LanguagePack.cpp | 52 +++++++++++++++++++++- 4 files changed, 77 insertions(+), 56 deletions(-) diff --git a/src/openrct2/drawing/Drawing.String.cpp b/src/openrct2/drawing/Drawing.String.cpp index bfd78c2880..219f25d453 100644 --- a/src/openrct2/drawing/Drawing.String.cpp +++ b/src/openrct2/drawing/Drawing.String.cpp @@ -19,12 +19,6 @@ #include "TTF.h" #include -#include -#include -#include -#include -#include -#include enum : uint32_t { @@ -819,30 +813,7 @@ static const utf8* ttf_process_glyph_run(rct_drawpixelinfo* dpi, const utf8* tex static void ttf_process_string(rct_drawpixelinfo* dpi, const utf8* text, text_draw_info* info) { - UErrorCode err = (UErrorCode)0; - // Force a hard left-to-right at the beginning (will mess up mixed strings' word order otherwise) - std::string text2 = std::string(u8"\xE2\x80\xAA") + text; - - icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(text2)); - - int32_t length = ustr.length(); - icu::UnicodeString reordered; - icu::UnicodeString shaped; - UBiDi* bidi = ubidi_openSized(length, 0, &err); - ubidi_setPara(bidi, ustr.getBuffer(), length, UBIDI_DEFAULT_LTR, nullptr, &err); - ubidi_writeReordered(bidi, reordered.getBuffer(length), length, UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS, &err); - ubidi_close(bidi); - reordered.releaseBuffer(length); - u_shapeArabic( - reordered.getBuffer(), length, shaped.getBuffer(length), length, - U_SHAPE_LETTERS_SHAPE | U_SHAPE_LENGTH_FIXED_SPACES_NEAR | U_SHAPE_TEXT_DIRECTION_VISUAL_LTR, &err); - shaped.releaseBuffer(length); - - std::string cppstring; - shaped.toUTF8String(cppstring); - - const utf8* utf8c = cppstring.c_str(); - const utf8* ch = utf8c; + const utf8* ch = text; const utf8* nextCh; int32_t codepoint; diff --git a/src/openrct2/localisation/Language.cpp b/src/openrct2/localisation/Language.cpp index 86240b2b50..c895ff88ba 100644 --- a/src/openrct2/localisation/Language.cpp +++ b/src/openrct2/localisation/Language.cpp @@ -23,30 +23,30 @@ // clang-format off const language_descriptor LanguagesDescriptors[LANGUAGE_COUNT] = { - { "", "", "", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_UNDEFINED - { "ar-EG", "Arabic (experimental)", "Arabic (experimental)", FAMILY(&TTFFamilySansSerif), RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_ARABIC - { "ca-ES", "Catalan", u8"Català", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_SPANISH }, // LANGUAGE_CATALAN - { "zh-CN", "Chinese (Simplified)", "Chinese (Simplified)", FAMILY(&TTFFamilyChineseSimplified), RCT2_LANGUAGE_ID_CHINESE_SIMPLIFIED }, // LANGUAGE_CHINESE_SIMPLIFIED - { "zh-TW", "Chinese (Traditional)", "Chinese (Traditional)", FAMILY(&TTFFamilyChineseTraditional), RCT2_LANGUAGE_ID_CHINESE_TRADITIONAL }, // LANGUAGE_CHINESE_TRADITIONAL - { "cs-CZ", "Czech", "Czech", FAMILY(&TTFFamilySansSerif), RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_CZECH - { "da-DK", "Danish", "Dansk", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_DANISH - { "de-DE", "German", "Deutsch", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_GERMAN }, // LANGUAGE_GERMAN - { "en-GB", "English (UK)", "English (UK)", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_ENGLISH_UK - { "en-US", "English (US)", "English (US)", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_US }, // LANGUAGE_ENGLISH_US - { "es-ES", "Spanish", u8"Español", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_SPANISH }, // LANGUAGE_SPANISH - { "fr-FR", "French", u8"Français", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_FRENCH }, // LANGUAGE_FRENCH - { "it-IT", "Italian", "Italiano", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ITALIAN }, // LANGUAGE_ITALIAN - { "ja-JP", "Japanese", "Japanese", FAMILY(&TTFFamilyJapanese), RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_JAPANESE - { "ko-KR", "Korean", "Korean", FAMILY(&TTFFamilyKorean), RCT2_LANGUAGE_ID_KOREAN }, // LANGUAGE_KOREAN - { "hu-HU", "Hungarian", "Magyar", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_HUNGARIAN - { "nl-NL", "Dutch", "Nederlands", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_DUTCH }, // LANGUAGE_DUTCH - { "nb-NO", "Norwegian", "Norsk", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_NORWEGIAN - { "pl-PL", "Polish", "Polski", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_POLISH - { "pt-BR", "Portuguese (BR)", u8"Português (BR)", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_PORTUGUESE }, // LANGUAGE_PORTUGUESE_BR - { "ru-RU", "Russian", u8"Русский", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_RUSSIAN - { "fi-FI", "Finnish", "Suomi", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_FINNISH - { "sv-SE", "Swedish", "Svenska", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_SWEDISH }, // LANGUAGE_SWEDISH - { "tr-TR", "Turkish", "Türkçe", FAMILY_OPENRCT2_SPRITE, RCT2_LANGUAGE_ID_ENGLISH_UK }, // LANGUAGE_TURKISH + { "", "", "", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_UNDEFINED + { "ar-EG", "Arabic (experimental)", "Arabic (experimental)", FAMILY(&TTFFamilySansSerif), true }, // LANGUAGE_ARABIC + { "ca-ES", "Catalan", u8"Català", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_CATALAN + { "zh-CN", "Chinese (Simplified)", "Chinese (Simplified)", FAMILY(&TTFFamilyChineseSimplified), false }, // LANGUAGE_CHINESE_SIMPLIFIED + { "zh-TW", "Chinese (Traditional)", "Chinese (Traditional)", FAMILY(&TTFFamilyChineseTraditional), false }, // LANGUAGE_CHINESE_TRADITIONAL + { "cs-CZ", "Czech", "Czech", FAMILY(&TTFFamilySansSerif), false }, // LANGUAGE_CZECH + { "da-DK", "Danish", "Dansk", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_DANISH + { "de-DE", "German", "Deutsch", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_GERMAN + { "en-GB", "English (UK)", "English (UK)", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_ENGLISH_UK + { "en-US", "English (US)", "English (US)", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_ENGLISH_US + { "es-ES", "Spanish", u8"Español", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_SPANISH + { "fr-FR", "French", u8"Français", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_FRENCH + { "it-IT", "Italian", "Italiano", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_ITALIAN + { "ja-JP", "Japanese", "Japanese", FAMILY(&TTFFamilyJapanese), false }, // LANGUAGE_JAPANESE + { "ko-KR", "Korean", "Korean", FAMILY(&TTFFamilyKorean), false }, // LANGUAGE_KOREAN + { "hu-HU", "Hungarian", "Magyar", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_HUNGARIAN + { "nl-NL", "Dutch", "Nederlands", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_DUTCH + { "nb-NO", "Norwegian", "Norsk", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_NORWEGIAN + { "pl-PL", "Polish", "Polski", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_POLISH + { "pt-BR", "Portuguese (BR)", u8"Português (BR)", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_PORTUGUESE_BR + { "ru-RU", "Russian", u8"Русский", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_RUSSIAN + { "fi-FI", "Finnish", "Suomi", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_FINNISH + { "sv-SE", "Swedish", "Svenska", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_SWEDISH + { "tr-TR", "Turkish", "Türkçe", FAMILY_OPENRCT2_SPRITE, false }, // LANGUAGE_TURKISH }; // clang-format on diff --git a/src/openrct2/localisation/Language.h b/src/openrct2/localisation/Language.h index 52c4d7edab..5791a14229 100644 --- a/src/openrct2/localisation/Language.h +++ b/src/openrct2/localisation/Language.h @@ -79,7 +79,7 @@ struct language_descriptor #else void* font_family; #endif - RCT2LanguageId rct2_original_id; + bool isRtl; }; extern const language_descriptor LanguagesDescriptors[LANGUAGE_COUNT]; diff --git a/src/openrct2/localisation/LanguagePack.cpp b/src/openrct2/localisation/LanguagePack.cpp index 4829c09f15..76a5b2fdd3 100644 --- a/src/openrct2/localisation/LanguagePack.cpp +++ b/src/openrct2/localisation/LanguagePack.cpp @@ -20,6 +20,14 @@ #include #include #include +#ifndef _WIN32 +#include +#include +#include +#include +#include +#include +#endif // Don't try to load more than language files that exceed 64 MiB constexpr uint64_t MAX_LANGUAGE_SIZE = 64 * 1024 * 1024; @@ -564,7 +572,18 @@ private: } } - auto s = std::string(sb.GetBuffer(), sb.GetLength()); + std::string s; + if (LanguagesDescriptors[_id].isRtl) + { + auto ts = std::string(sb.GetBuffer(), sb.GetLength()); + s = FixRTL(ts); + + } + else + { + s = std::string(sb.GetBuffer(), sb.GetLength()); + } + if (_currentGroup.empty()) { // Make sure the list is big enough to contain this string id @@ -627,6 +646,37 @@ private: return true; } + + std::string FixRTL(std::string& input) + { +#ifdef _WIN32 + return input; +#else + UErrorCode err = (UErrorCode)0; + // Force a hard left-to-right at the beginning (will mess up mixed strings' word order otherwise) + std::string text2 = std::string(u8"\xE2\x80\xAA") + input; + + icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(text2)); + + int32_t length = ustr.length(); + icu::UnicodeString reordered; + icu::UnicodeString shaped; + UBiDi* bidi = ubidi_openSized(length, 0, &err); + // UBIDI_DEFAULT_LTR preserves formatting codes. + ubidi_setPara(bidi, ustr.getBuffer(), length, UBIDI_DEFAULT_LTR, nullptr, &err); + ubidi_writeReordered(bidi, reordered.getBuffer(length), length, UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS, &err); + ubidi_close(bidi); + reordered.releaseBuffer(length); + u_shapeArabic( + reordered.getBuffer(), length, shaped.getBuffer(length), length, + U_SHAPE_LETTERS_SHAPE | U_SHAPE_LENGTH_FIXED_SPACES_NEAR | U_SHAPE_TEXT_DIRECTION_VISUAL_LTR, &err); + shaped.releaseBuffer(length); + + std::string cppstring; + shaped.toUTF8String(cppstring); + return cppstring; +#endif + } }; namespace LanguagePackFactory From 3d74c5195997507197be161637321676603c9815 Mon Sep 17 00:00:00 2001 From: Michael Steenbeek Date: Thu, 26 Jul 2018 14:26:43 +0200 Subject: [PATCH 5/7] Add missing include --- src/openrct2/localisation/LanguagePack.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/openrct2/localisation/LanguagePack.cpp b/src/openrct2/localisation/LanguagePack.cpp index 76a5b2fdd3..0ed62edd8a 100644 --- a/src/openrct2/localisation/LanguagePack.cpp +++ b/src/openrct2/localisation/LanguagePack.cpp @@ -15,6 +15,7 @@ #include "../core/String.hpp" #include "../core/StringBuilder.hpp" #include "../core/StringReader.hpp" +#include "Language.h" #include "Localisation.h" #include From ef6f09cf5b1d3f28e78919a74341fd78cedc231b Mon Sep 17 00:00:00 2001 From: Michael Steenbeek Date: Thu, 26 Jul 2018 15:02:55 +0200 Subject: [PATCH 6/7] Fix formatting [ci skip] --- src/openrct2/localisation/LanguagePack.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/openrct2/localisation/LanguagePack.cpp b/src/openrct2/localisation/LanguagePack.cpp index 0ed62edd8a..79a1da0556 100644 --- a/src/openrct2/localisation/LanguagePack.cpp +++ b/src/openrct2/localisation/LanguagePack.cpp @@ -22,12 +22,12 @@ #include #include #ifndef _WIN32 -#include -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include #endif // Don't try to load more than language files that exceed 64 MiB @@ -578,7 +578,6 @@ private: { auto ts = std::string(sb.GetBuffer(), sb.GetLength()); s = FixRTL(ts); - } else { @@ -669,8 +668,8 @@ private: ubidi_close(bidi); reordered.releaseBuffer(length); u_shapeArabic( - reordered.getBuffer(), length, shaped.getBuffer(length), length, - U_SHAPE_LETTERS_SHAPE | U_SHAPE_LENGTH_FIXED_SPACES_NEAR | U_SHAPE_TEXT_DIRECTION_VISUAL_LTR, &err); + reordered.getBuffer(), length, shaped.getBuffer(length), length, + U_SHAPE_LETTERS_SHAPE | U_SHAPE_LENGTH_FIXED_SPACES_NEAR | U_SHAPE_TEXT_DIRECTION_VISUAL_LTR, &err); shaped.releaseBuffer(length); std::string cppstring; From eae09ea16d01879592c19fbe7c8e36fbd90deb1d Mon Sep 17 00:00:00 2001 From: Gymnasiast Date: Sat, 29 Sep 2018 23:06:26 +0200 Subject: [PATCH 7/7] Attempt at fixing language pack test --- test/tests/LanguagePackTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/tests/LanguagePackTest.cpp b/test/tests/LanguagePackTest.cpp index ad59ae10de..73cd0aee0c 100644 --- a/test/tests/LanguagePackTest.cpp +++ b/test/tests/LanguagePackTest.cpp @@ -9,10 +9,15 @@ #include "openrct2/localisation/LanguagePack.h" +#include "openrct2/localisation/Language.h" #include "openrct2/localisation/StringIds.h" #include +#ifndef _WIN32 +const language_descriptor LanguagesDescriptors[] = {}; +#endif + class LanguagePackTest : public testing::Test { protected: