Use ICU for converting strings to UTF-8 instead of our own tables.

Co-authored-by: Fusxfaranto <fusxfaranto@gmail.com>

src/openrct2/CMakeLists.txt

@@ -1,5 +1,5 @@
 # CMAKE project for libopenrct2 (core OpenRCT2 component)
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.7)
 if (CMAKE_BINARY_DIR STREQUAL CMAKE_SOURCE_DIR)
     message(FATAL_ERROR "Building in-source is not supported! Create a build dir and remove ${CMAKE_SOURCE_DIR}/CMakeCache.txt")
 endif ()
@@ -58,6 +58,9 @@ if (NOT DISABLE_TTF)
     endif ()
 endif ()
 
+# For unicode code page conversion.
+find_package(ICU 59.0 REQUIRED COMPONENTS uc)
+
 # Sources
 file(GLOB_RECURSE OPENRCT2_CORE_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp"
                                         "${CMAKE_CURRENT_LIST_DIR}/*.h"
@@ -78,12 +81,14 @@ if (STATIC)
     target_link_libraries(${PROJECT} ${JANSSON_STATIC_LIBRARIES}
                                      ${PNG_STATIC_LIBRARIES}
                                      ${ZLIB_STATIC_LIBRARIES}
-                                     ${LIBZIP_STATIC_LIBRARIES})
+                                     ${LIBZIP_STATIC_LIBRARIES}
+                                     ${ICU_STATIC_LIBRARIES})
 else ()
     target_link_libraries(${PROJECT} ${JANSSON_LIBRARIES}
                                      ${PNG_LIBRARIES}
                                      ${ZLIB_LIBRARIES}
-                                     ${LIBZIP_LIBRARIES})
+                                     ${LIBZIP_LIBRARIES}
+                                     ${ICU_LIBRARIES})
 endif ()
 
 if (UNIX AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "BSD")

src/openrct2/core/String.cpp

@@ -17,6 +17,9 @@
 #include <cwctype>
 #include <stdexcept>
 #include <vector>
+#ifndef _WIN32
+#include <unicode/unistr.h>
+#endif
 
 #ifdef _WIN32
 #ifndef NOMINMAX
@@ -26,6 +29,8 @@
 #include <windows.h>
 #endif
 
+#include "../common.h"
+#include "../localisation/ConversionTables.h"
 #include "../localisation/Language.h"
 #include "../util/Util.h"
 
@@ -567,8 +572,52 @@ namespace String
 
         return dst;
 #else
+        const char* codepage;
+        switch (srcCodePage)
+        {
+        case CODE_PAGE::CP_932:
+            codepage = "windows-932";
+            break;
+
+        case CODE_PAGE::CP_936:
+            codepage = "GB2312";
+            break;
+
+        case CODE_PAGE::CP_949:
+            codepage = "windows-949";
+            break;
+
+        case CODE_PAGE::CP_950:
+            codepage = "big5";
+            break;
+
+        default:
+            throw std::runtime_error("Unsupported code page: " + std::to_string(srcCodePage));
+        }
+
+        icu::UnicodeString convertString(src.data(), codepage);
+        std::string result;
+
+        if (dstCodePage == CODE_PAGE::CP_UTF8)
+            convertString.toUTF8String(result);
+        return result;
+#endif
+    }
+
+    std::string ToUpper(const utf8 * src)
+    {
+#ifdef _WIN32
+        // TODO: LCMapStringEx on Windows.
         STUB();
-        return std::string(src);
+        return String::ToStd(src);
+#else
+        icu::UnicodeString str = icu::UnicodeString::fromUTF8(src);
+        str.toUpper();
+
+        std::string res;
+        str.toUTF8String(res);
+
+        return res;
 #endif
     }
 } // namespace String

src/openrct2/localisation/ConversionTables.h

@@ -0,0 +1,32 @@
+#pragma region Copyright (c) 2014-2018 OpenRCT2 Developers
+/*****************************************************************************
+ * OpenRCT2, an open source clone of Roller Coaster Tycoon 2.
+ *
+ * OpenRCT2 is the work of many authors, a full list can be found in contributors.md
+ * For more information, visit https://github.com/OpenRCT2/OpenRCT2
+ *
+ * OpenRCT2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * A full copy of the GNU General Public License can be found in licence.txt
+ *****************************************************************************/
+#pragma endregion
+
+#pragma once
+
+#include "../common.h"
+
+struct encoding_convert_entry
+{
+    uint16 code;
+    uint32 unicode;
+};
+
+extern const encoding_convert_entry RCT2ToUnicodeTable[256];
+
+static sint32 encoding_search_compare(const void *pKey, const void *pEntry);
+static wchar_t encoding_convert_x_to_unicode(wchar_t code, const encoding_convert_entry *table, size_t count);
+wchar_t encoding_convert_rct2_to_unicode(wchar_t rct2str);
+uint32 encoding_convert_unicode_to_rct2(uint32 unicode);

src/openrct2/localisation/Convert.cpp

@@ -151,39 +151,16 @@ static std::string DecodeConvertWithTable(const std::string_view& src, TConvertF
 std::string rct2_to_utf8(const std::string_view& src, RCT2LanguageId languageId)
 {
     auto codePage = GetCodePageForRCT2Language(languageId);
-
-    std::string result;
-    switch (codePage)
+    if (codePage == CODE_PAGE::CP_1252)
     {
-        case CODE_PAGE::CP_1252:
-            // The code page used by RCT2 was not quite 1252 as some codes were used for Polish characters.
-            result = DecodeConvertWithTable(src, encoding_convert_rct2_to_unicode);
-            break;
-
-#ifdef _WIN32
-        default:
-            auto decoded = DecodeToMultiByte(src);
-            result = String::Convert(decoded, codePage, CODE_PAGE::CP_UTF8);
-#else
-        // TODO Change this to use a library such as libicu
-        case CODE_PAGE::CP_932:
-            result = DecodeConvertWithTable(src, encoding_convert_cp932_to_unicode);
-            break;
-        case CODE_PAGE::CP_936:
-            result = DecodeConvertWithTable(src, encoding_convert_gb2312_to_unicode);
-            break;
-        case CODE_PAGE::CP_949:
-            result = DecodeConvertWithTable(src, encoding_convert_cp949_to_unicode);
-            break;
-        case CODE_PAGE::CP_950:
-            result = DecodeConvertWithTable(src, encoding_convert_big5_to_unicode);
-            break;
-        default:
-            throw std::runtime_error("Unsupported code page: " + std::to_string(codePage));
-            break;
-#endif
+        // The code page used by RCT2 was not quite 1252 as some codes were used for Polish characters.
+        return DecodeConvertWithTable(src, encoding_convert_rct2_to_unicode);
+    }
+    else
+    {
+        auto decoded = DecodeToMultiByte(src);
+        return String::Convert(decoded, codePage, CODE_PAGE::CP_UTF8);
     }
-    return result;
 }
 
 std::string utf8_to_rct2(const std::string_view& src)

src/openrct2/localisation/Language.cpp

@@ -18,7 +18,6 @@
 #include "../Context.h"
 #include "../core/Path.hpp"
 #include "../core/String.hpp"
-#include "../core/StringBuilder.hpp"
 #include "../interface/Fonts.h"
 #include "../interface/FontFamilies.h"
 #include "../object/ObjectManager.h"

src/openrct2/localisation/Localisation.h

@@ -56,15 +56,6 @@ sint32 win1252_to_utf8(utf8string dst, const char *src, size_t srcLength, size_t
 wchar_t encoding_convert_rct2_to_unicode(wchar_t rct2str);
 uint32 encoding_convert_unicode_to_rct2(uint32 unicode);
 
-#ifndef _WIN32
-
-wchar_t encoding_convert_gb2312_to_unicode(wchar_t gb2312);
-wchar_t encoding_convert_big5_to_unicode(wchar_t big5);
-wchar_t encoding_convert_cp932_to_unicode(wchar_t cp932);
-wchar_t encoding_convert_cp949_to_unicode(wchar_t cp949);
-
-#endif
-
 #define MAX_USER_STRINGS 1024
 #define USER_STRING_MAX_LENGTH 32
 

test/tests/CMakeLists.txt

@@ -91,6 +91,10 @@ add_library(test-common STATIC ${COMMON_TEST_SOURCES})
 execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink "${CMAKE_CURRENT_LIST_DIR}/testdata" "${CMAKE_CURRENT_BINARY_DIR}/testdata")
 install(CODE "execute_process(COMMAND \"${CMAKE_COMMAND}\" -E create_symlink \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_DATADIR}/openrct2\" \"${CMAKE_CURRENT_BINARY_DIR}/data\")")
 
+# For unicode code page conversion (required for ini and string tests)
+find_package(ICU 59.0 REQUIRED COMPONENTS uc)
+target_link_libraries(test-common ${ICU_LIBRARIES})
+
 # Start of our tests
 
 # sawyercoding test