Gamestate snapshots (#8819)

* Add initial interface. * Implement move operator in MemoryStream * Add pod array serialisation traits. * Add push_back with move semantics to CircularBuffer * Initial implementation of GameStateSnapshots * Add GameStateSnapshots to Context. * Add mp_desync console command. * Compare sprite data and fill change list. * Minor changes. * Proof of concept. * Calculate offset instead of using offsetof * Implement game state difference detection * Update mp_desync console command. * Fix identification of sprite remove/add. * Fix crash when only one peep in park when using mp_desync * Output state differences into user directory desync folder. * Add desync debugging as an option. * Add information to network status when a desync report was created. * Cast to proper type for %llu. * Update xcode project * Add more information to the diffed data. * Remove client-only relevant fields. * Cleanup. * Add better name output for misc sprites * Add srand0 and tick information to the output * Bump up network version * Cleanup * Set desync_debugging to false as default * Apply suggestions
2026-01-21 14:02:59 +01:00 · 2019-05-11 21:31:34 +02:00
parent 1f6c7c9942
commit c8f822ea70
22 changed files with 1152 additions and 19 deletions
--- a/src/openrct2/network/Network.cpp
+++ b/src/openrct2/network/Network.cpp
@@ -11,6 +11,7 @@

 #include "../Context.h"
 #include "../Game.h"
+#include "../GameStateSnapshots.h"
 #include "../OpenRCT2.h"
 #include "../PlatformEnvironment.h"
 #include "../actions/LoadOrQuitAction.hpp"
@@ -31,12 +32,16 @@
 // This string specifies which version of network stream current build uses.
 // It is used for making sure only compatible builds get connected, even within
 // single OpenRCT2 version.
-#define NETWORK_STREAM_VERSION "24"
+#define NETWORK_STREAM_VERSION "25"
 #define NETWORK_STREAM_ID OPENRCT2_VERSION "-" NETWORK_STREAM_VERSION

 static Peep* _pickup_peep = nullptr;
 static int32_t _pickup_peep_old_x = LOCATION_NULL;

+// General chunk size is 63 KiB, this can not be any larger because the packet size is encoded
+// with uint16_t and needs some spare room for other data in the packet.
+static constexpr uint32_t CHUNK_SIZE = 1024 * 63;
+
 #ifndef DISABLE_NETWORK

 #    include "../Cheats.h"
@@ -137,7 +142,9 @@ public:
    void SendPacketToClients(NetworkPacket& packet, bool front = false, bool gameCmd = false);
    bool CheckSRAND(uint32_t tick, uint32_t srand0);
    bool IsDesynchronised();
-    void CheckDesynchronizaton();
+    bool CheckDesynchronizaton();
+    void RequestStateSnapshot();
+    NetworkServerState_t GetServerState() const;
    void KickPlayer(int32_t playerId);
    void SetPassword(const char* password);
    void ShutdownClient();
@@ -160,6 +167,8 @@ public:
    void AppendServerLog(const std::string& s);
    void CloseServerLog();

+    void Client_Send_RequestGameState(uint32_t tick);
+
    void Client_Send_TOKEN();
    void Client_Send_AUTH(
        const std::string& name, const std::string& password, const std::string& pubkey, const std::vector<uint8_t>& signature);
@@ -309,7 +318,6 @@ private:
    uint16_t listening_port = 0;
    SOCKET_STATUS _lastConnectStatus = SOCKET_STATUS_CLOSED;
    uint32_t last_ping_sent_time = 0;
-    uint32_t server_tick = 0;
    uint8_t player_id = 0;
    std::list<std::unique_ptr<NetworkConnection>> client_connection_list;
    std::multiset<GameCommand> game_command_queue;
@@ -317,7 +325,8 @@ private:
    std::string _host;
    uint16_t _port = 0;
    std::string _password;
-    bool _desynchronised = false;
+    NetworkServerState_t _serverState;
+    MemoryStream _serverGameState;
    uint32_t server_connect_time = 0;
    uint8_t default_group = 0;
    uint32_t _commandId;
@@ -336,6 +345,7 @@ private:
 private:
    std::vector<void (Network::*)(NetworkConnection& connection, NetworkPacket& packet)> client_command_handlers;
    std::vector<void (Network::*)(NetworkConnection& connection, NetworkPacket& packet)> server_command_handlers;
+    void Server_Handle_REQUEST_GAMESTATE(NetworkConnection& connection, NetworkPacket& packet);
    void Client_Handle_AUTH(NetworkConnection& connection, NetworkPacket& packet);
    void Server_Handle_AUTH(NetworkConnection& connection, NetworkPacket& packet);
    void Server_Client_Joined(const char* name, const std::string& keyhash, NetworkConnection& connection);
@@ -361,6 +371,7 @@ private:
    void Client_Handle_TOKEN(NetworkConnection& connection, NetworkPacket& packet);
    void Server_Handle_TOKEN(NetworkConnection& connection, NetworkPacket& packet);
    void Client_Handle_OBJECTS(NetworkConnection& connection, NetworkPacket& packet);
+    void Client_Handle_GAMESTATE(NetworkConnection& connection, NetworkPacket& packet);
    void Server_Handle_OBJECTS(NetworkConnection& connection, NetworkPacket& packet);

    uint8_t* save_for_network(size_t& out_size, const std::vector<const ObjectRepositoryItem*>& objects) const;
@@ -397,6 +408,7 @@ Network::Network()
    client_command_handlers[NETWORK_COMMAND_GAMEINFO] = &Network::Client_Handle_GAMEINFO;
    client_command_handlers[NETWORK_COMMAND_TOKEN] = &Network::Client_Handle_TOKEN;
    client_command_handlers[NETWORK_COMMAND_OBJECTS] = &Network::Client_Handle_OBJECTS;
+    client_command_handlers[NETWORK_COMMAND_GAMESTATE] = &Network::Client_Handle_GAMESTATE;
    server_command_handlers.resize(NETWORK_COMMAND_MAX, nullptr);
    server_command_handlers[NETWORK_COMMAND_AUTH] = &Network::Server_Handle_AUTH;
    server_command_handlers[NETWORK_COMMAND_CHAT] = &Network::Server_Handle_CHAT;
@@ -406,6 +418,7 @@ Network::Network()
    server_command_handlers[NETWORK_COMMAND_GAMEINFO] = &Network::Server_Handle_GAMEINFO;
    server_command_handlers[NETWORK_COMMAND_TOKEN] = &Network::Server_Handle_TOKEN;
    server_command_handlers[NETWORK_COMMAND_OBJECTS] = &Network::Server_Handle_OBJECTS;
+    server_command_handlers[NETWORK_COMMAND_REQUEST_GAMESTATE] = &Network::Server_Handle_REQUEST_GAMESTATE;

    _chat_log_fs << std::unitbuf;
    _server_log_fs << std::unitbuf;
@@ -532,6 +545,8 @@ bool Network::BeginClient(const std::string& host, uint16_t port)
    _serverConnection = std::make_unique<NetworkConnection>();
    _serverConnection->Socket = CreateTcpSocket();
    _serverConnection->Socket->ConnectAsync(host, port);
+    _serverState.gamestateSnapshotsEnabled = false;
+
    status = NETWORK_STATUS_CONNECTING;
    _lastConnectStatus = SOCKET_STATUS_CLOSED;
    _clientMapLoaded = false;
@@ -664,6 +679,8 @@ bool Network::BeginServer(uint16_t port, const std::string& address)

    status = NETWORK_STATUS_CONNECTED;
    listening_port = port;
+    _serverState.gamestateSnapshotsEnabled = gConfigNetwork.desync_debugging;
+
    if (gConfigNetwork.advertise)
    {
        _advertiser = CreateServerAdvertiser(listening_port);
@@ -703,7 +720,7 @@ int32_t Network::GetAuthStatus()

 uint32_t Network::GetServerTick()
 {
-    return server_tick;
+    return _serverState.tick;
 }

 uint8_t Network::GetPlayerID()
@@ -997,7 +1014,10 @@ bool Network::CheckSRAND(uint32_t tick, uint32_t srand0)
    _serverTickData.erase(itTickData);

    if (storedTick.srand0 != srand0)
+    {
+        log_info("Srand0 mismatch, client = %08X, server = %08X", srand0, storedTick.srand0);
        return false;
+    }

    if (!storedTick.spriteHash.empty())
    {
@@ -1005,6 +1025,7 @@ bool Network::CheckSRAND(uint32_t tick, uint32_t srand0)
        std::string clientSpriteHash = checksum.ToString();
        if (clientSpriteHash != storedTick.spriteHash)
        {
+            log_info("Sprite hash mismatch, client = %s, server = %s", clientSpriteHash.c_str(), storedTick.spriteHash.c_str());
            return false;
        }
    }
@@ -1014,15 +1035,17 @@ bool Network::CheckSRAND(uint32_t tick, uint32_t srand0)

 bool Network::IsDesynchronised()
 {
-    return gNetwork._desynchronised;
+    return _serverState.state == NETWORK_SERVER_STATE_DESYNCED;
 }

-void Network::CheckDesynchronizaton()
+bool Network::CheckDesynchronizaton()
 {
    // Check synchronisation
-    if (GetMode() == NETWORK_MODE_CLIENT && !_desynchronised && !CheckSRAND(gCurrentTicks, scenario_rand_state().s0))
+    if (GetMode() == NETWORK_MODE_CLIENT && _serverState.state != NETWORK_SERVER_STATE_DESYNCED
+        && !CheckSRAND(gCurrentTicks, scenario_rand_state().s0))
    {
-        _desynchronised = true;
+        _serverState.state = NETWORK_SERVER_STATE_DESYNCED;
+        _serverState.desyncTick = gCurrentTicks;

        char str_desync[256];
        format_string(str_desync, 256, STR_MULTIPLAYER_DESYNC, nullptr);
@@ -1035,7 +1058,23 @@ void Network::CheckDesynchronizaton()
        {
            Close();
        }
+
+        return true;
    }
+
+    return false;
+}
+
+void Network::RequestStateSnapshot()
+{
+    log_info("Requesting game state for tick %u", _serverState.desyncTick);
+
+    Client_Send_RequestGameState(_serverState.desyncTick);
+}
+
+NetworkServerState_t Network::GetServerState() const
+{
+    return _serverState;
 }

 void Network::KickPlayer(int32_t playerId)
@@ -1403,6 +1442,20 @@ void Network::CloseServerLog()
    _server_log_fs.close();
 }

+void Network::Client_Send_RequestGameState(uint32_t tick)
+{
+    if (_serverState.gamestateSnapshotsEnabled == false)
+    {
+        log_verbose("Server does not store a gamestate history");
+        return;
+    }
+
+    log_verbose("Requesting gamestate from server for tick %u", tick);
+    std::unique_ptr<NetworkPacket> packet(NetworkPacket::Allocate());
+    *packet << (uint32_t)NETWORK_COMMAND_REQUEST_GAMESTATE << tick;
+    _serverConnection->QueuePacket(std::move(packet));
+}
+
 void Network::Client_Send_TOKEN()
 {
    log_verbose("requesting token");
@@ -1531,7 +1584,7 @@ void Network::Server_Send_MAP(NetworkConnection* connection)
        }
        return;
    }
-    size_t chunksize = 65000;
+    size_t chunksize = CHUNK_SIZE;
    for (size_t i = 0; i < out_size; i += chunksize)
    {
        size_t datasize = std::min(chunksize, out_size - i);
@@ -1784,6 +1837,8 @@ void Network::Server_Send_GAMEINFO(NetworkConnection& connection)
    json_object_set_new(obj, "provider", jsonProvider);

    packet->WriteString(json_dumps(obj, 0));
+    *packet << _serverState.gamestateSnapshotsEnabled;
+
    json_decref(obj);
 #    endif
    connection.QueuePacket(std::move(packet));
@@ -2313,6 +2368,48 @@ void Network::Client_Handle_TOKEN(NetworkConnection& connection, NetworkPacket&
    Client_Send_AUTH(gConfigNetwork.player_name.c_str(), password, pubkey.c_str(), signature);
 }

+void Network::Server_Handle_REQUEST_GAMESTATE(NetworkConnection& connection, NetworkPacket& packet)
+{
+    uint32_t tick;
+    packet >> tick;
+
+    if (_serverState.gamestateSnapshotsEnabled == false)
+    {
+        // Ignore this if this is off.
+        return;
+    }
+
+    IGameStateSnapshots* snapshots = GetContext()->GetGameStateSnapshots();
+
+    const GameStateSnapshot_t* snapshot = snapshots->GetLinkedSnapshot(tick);
+    if (snapshot)
+    {
+        MemoryStream snapshotMemory;
+        DataSerialiser ds(true, snapshotMemory);
+
+        snapshots->SerialiseSnapshot(const_cast<GameStateSnapshot_t&>(*snapshot), ds);
+
+        uint32_t bytesSent = 0;
+        uint32_t length = (uint32_t)snapshotMemory.GetLength();
+        while (bytesSent < length)
+        {
+            uint32_t dataSize = CHUNK_SIZE;
+            if (bytesSent + dataSize > snapshotMemory.GetLength())
+            {
+                dataSize = snapshotMemory.GetLength() - bytesSent;
+            }
+
+            std::unique_ptr<NetworkPacket> gameStateChunk(NetworkPacket::Allocate());
+            *gameStateChunk << (uint32_t)NETWORK_COMMAND_GAMESTATE << tick << length << bytesSent << dataSize;
+            gameStateChunk->Write((const uint8_t*)snapshotMemory.GetData() + bytesSent, dataSize);
+
+            connection.QueuePacket(std::move(gameStateChunk));
+
+            bytesSent += dataSize;
+        }
+    }
+}
+
 void Network::Client_Handle_AUTH(NetworkConnection& connection, NetworkPacket& packet)
 {
    uint32_t auth_status;
@@ -2436,6 +2533,73 @@ void Network::Client_Handle_OBJECTS(NetworkConnection& connection, NetworkPacket
    Client_Send_OBJECTS(requested_objects);
 }

+void Network::Client_Handle_GAMESTATE(NetworkConnection& connection, NetworkPacket& packet)
+{
+    uint32_t tick;
+    uint32_t totalSize;
+    uint32_t offset;
+    uint32_t dataSize;
+
+    packet >> tick >> totalSize >> offset >> dataSize;
+
+    if (offset == 0)
+    {
+        // Reset
+        _serverGameState = MemoryStream();
+    }
+
+    _serverGameState.SetPosition(offset);
+
+    const uint8_t* data = packet.Read(dataSize);
+    _serverGameState.Write(data, dataSize);
+
+    log_verbose("Received Game State %.02f%%", ((float)_serverGameState.GetLength() / (float)totalSize) * 100.0f);
+
+    if (_serverGameState.GetLength() == totalSize)
+    {
+        _serverGameState.SetPosition(0);
+        DataSerialiser ds(false, _serverGameState);
+
+        IGameStateSnapshots* snapshots = GetContext()->GetGameStateSnapshots();
+
+        GameStateSnapshot_t& serverSnapshot = snapshots->CreateSnapshot();
+        snapshots->SerialiseSnapshot(serverSnapshot, ds);
+
+        const GameStateSnapshot_t* desyncSnapshot = snapshots->GetLinkedSnapshot(tick);
+        if (desyncSnapshot)
+        {
+            GameStateCompareData_t cmpData = snapshots->Compare(serverSnapshot, *desyncSnapshot);
+
+            std::string outputPath = GetContext()->GetPlatformEnvironment()->GetDirectoryPath(
+                DIRBASE::USER, DIRID::LOG_DESYNCS);
+
+            platform_ensure_directory_exists(outputPath.c_str());
+
+            char uniqueFileName[128] = {};
+            snprintf(
+                uniqueFileName, sizeof(uniqueFileName), "desync_%llu_%u.txt",
+                (long long unsigned)platform_get_datetime_now_utc(), tick);
+
+            std::string outputFile = Path::Combine(outputPath, uniqueFileName);
+
+            if (snapshots->LogCompareDataToFile(outputFile, cmpData))
+            {
+                log_info("Wrote desync report to '%s'", outputFile.c_str());
+
+                uint8_t args[32]{};
+                set_format_arg_on(args, 0, char*, uniqueFileName);
+
+                char str_desync[1024];
+                format_string(str_desync, sizeof(str_desync), STR_DESYNC_REPORT, args);
+
+                auto intent = Intent(WC_NETWORK_STATUS);
+                intent.putExtra(INTENT_EXTRA_MESSAGE, std::string{ str_desync });
+                context_open_intent(&intent);
+            }
+        }
+    }
+}
+
 void Network::Server_Handle_OBJECTS(NetworkConnection& connection, NetworkPacket& packet)
 {
    uint32_t size;
@@ -2644,9 +2808,10 @@ void Network::Client_Handle_MAP([[maybe_unused]] NetworkConnection& connection,
            game_load_init();
            game_command_queue.clear();
            _serverTickData.clear();
-            server_tick = gCurrentTicks;
+            _serverState.tick = gCurrentTicks;
+            _serverTickData.clear();
            // window_network_status_open("Loaded new map from network");
-            _desynchronised = false;
+            _serverState.state = NETWORK_SERVER_STATE_OK;
            _clientMapLoaded = true;
            gFirstTimeSaving = true;

@@ -2977,11 +3142,13 @@ void Network::Client_Handle_TICK([[maybe_unused]] NetworkConnection& connection,
 {
    uint32_t srand0;
    uint32_t flags;
-    packet >> server_tick >> srand0 >> flags;
+    uint32_t serverTick;
+
+    packet >> serverTick >> srand0 >> flags;

    ServerTickData_t tickData;
    tickData.srand0 = srand0;
-    tickData.tick = server_tick;
+    tickData.tick = serverTick;

    if (flags & NETWORK_TICK_FLAG_CHECKSUMS)
    {
@@ -2998,7 +3165,8 @@ void Network::Client_Handle_TICK([[maybe_unused]] NetworkConnection& connection,
        _serverTickData.erase(_serverTickData.begin());
    }

-    _serverTickData.emplace(server_tick, tickData);
+    _serverState.tick = serverTick;
+    _serverTickData.emplace(serverTick, tickData);
 }

 void Network::Client_Handle_PLAYERINFO([[maybe_unused]] NetworkConnection& connection, NetworkPacket& packet)
@@ -3154,6 +3322,7 @@ static std::string json_stdstring_value(const json_t* string)
 void Network::Client_Handle_GAMEINFO([[maybe_unused]] NetworkConnection& connection, NetworkPacket& packet)
 {
    const char* jsonString = packet.ReadString();
+    packet >> _serverState.gamestateSnapshotsEnabled;

    json_error_t error;
    json_t* root = json_loads(jsonString, 0, &error);
@@ -3234,11 +3403,16 @@ bool network_is_desynchronised()
    return gNetwork.IsDesynchronised();
 }

-void network_check_desynchronization()
+bool network_check_desynchronisation()
 {
    return gNetwork.CheckDesynchronizaton();
 }

+void network_request_gamestate_snapshot()
+{
+    return gNetwork.RequestStateSnapshot();
+}
+
 void network_send_tick()
 {
    gNetwork.Server_Send_TICK();
@@ -3995,6 +4169,16 @@ NetworkStats_t network_get_stats()
    return gNetwork.GetStats();
 }

+NetworkServerState_t network_get_server_state()
+{
+    return gNetwork.GetServerState();
+}
+
+bool network_gamestate_snapshots_enabled()
+{
+    return network_get_server_state().gamestateSnapshotsEnabled;
+}
+
 #else
 int32_t network_get_mode()
 {
@@ -4022,7 +4206,15 @@ bool network_is_desynchronised()
 {
    return false;
 }
-void network_check_desynchronization()
+bool network_gamestate_snapshots_enabled()
+{
+    return false;
+}
+bool network_check_desynchronisation()
+{
+    return false;
+}
+void network_request_gamestate_snapshot()
 {
 }
 void network_enqueue_game_action(const GameAction* action)
@@ -4240,4 +4432,8 @@ NetworkStats_t network_get_stats()
 {
    return NetworkStats_t{};
 }
+NetworkServerState_t network_get_server_state()
+{
+    return NetworkServerState_t{};
+}
 #endif /* DISABLE_NETWORK */