Refactor Compression and Streams, and Add IStream Direct Interface

src/openrct2/ReplayManager.cpp

@@ -25,7 +25,10 @@
 #include "actions/TileModifyAction.h"
 #include "actions/TrackPlaceAction.h"
 #include "config/Config.h"
+#include "core/Compression.h"
 #include "core/DataSerialiser.h"
+#include "core/FileStream.h"
+#include "core/FileSystem.hpp"
 #include "core/Path.hpp"
 #include "entity/EntityRegistry.h"
 #include "entity/EntityTweener.h"
@@ -35,7 +38,6 @@
 #include "object/ObjectRepository.h"
 #include "park/ParkFile.h"
 #include "world/Park.h"
-#include "zlib.h"
 
 #include <chrono>
 #include <memory>
@@ -103,7 +105,7 @@ namespace OpenRCT2
     {
         static constexpr uint16_t kReplayVersion = 10;
         static constexpr uint32_t kReplayMagic = 0x5243524F; // ORCR.
-        static constexpr int kReplayCompressionLevel = 9;
+        static constexpr int kReplayCompressionLevel = Compression::kZlibMaxCompressionLevel;
         static constexpr int kNormalRecordingChecksumTicks = 1;
         static constexpr int kSilentRecordingChecksumTicks = 40; // Same as network server
 
@@ -307,44 +309,24 @@ namespace OpenRCT2
             // Serialise Body.
             DataSerialiser recSerialiser(true);
             Serialise(recSerialiser, *_currentRecording);
+            auto& stream = recSerialiser.GetStream();
 
-            const auto& stream = recSerialiser.GetStream();
+            MemoryStream compressed;
-            unsigned long streamLength = static_cast<unsigned long>(stream.GetLength());
+            bool compressStatus = Compression::zlibCompress(
-            unsigned long compressLength = compressBound(streamLength);
+                stream, static_cast<size_t>(stream.GetLength()), compressed, Compression::ZlibHeaderType::zlib,
-
-            MemoryStream data(compressLength);
-
-            ReplayRecordFile file{ _currentRecording->magic, _currentRecording->version, streamLength, data };
-
-            auto compressBuf = std::make_unique<unsigned char[]>(compressLength);
-            compress2(
-                compressBuf.get(), &compressLength, static_cast<const unsigned char*>(stream.GetData()), stream.GetLength(),
                 kReplayCompressionLevel);
-            file.data.Write(compressBuf.get(), compressLength);
+            if (!compressStatus)
+                throw IOException("Compression Error");
 
-            DataSerialiser fileSerialiser(true);
-            fileSerialiser << file.magic;
-            fileSerialiser << file.version;
-            fileSerialiser << file.uncompressedSize;
-            fileSerialiser << file.data;
-
-            bool result = false;
-
-            const std::string& outFile = _currentRecording->filePath;
-
-            FILE* fp = fopen(outFile.c_str(), "wb");
-            if (fp != nullptr)
             {
-                const auto& fileStream = fileSerialiser.GetStream();
+                ReplayRecordFile file{ _currentRecording->magic, _currentRecording->version, stream.GetLength(), compressed };
-                fwrite(fileStream.GetData(), 1, fileStream.GetLength(), fp);
-                fclose(fp);
 
-                result = true;
+                FileStream filestream(_currentRecording->filePath, FileMode::write);
-            }
+                DataSerialiser fileSerialiser(true, filestream);
-            else
+                fileSerialiser << file.magic;
-            {
+                fileSerialiser << file.version;
-                LOG_ERROR("Unable to write to file '%s'", outFile.c_str());
+                fileSerialiser << file.uncompressedSize;
-                result = false;
+                fileSerialiser << file.data;
             }
 
             // When normalizing the output we don't touch the mode.
@@ -356,7 +338,7 @@ namespace OpenRCT2
             News::Item* news = News::AddItemToQueue(News::ItemType::blank, "Replay recording stopped", 0);
             news->setFlags(News::ItemFlags::hasButton); // Has no subject.
 
-            return result;
+            return true;
         }
 
         virtual bool GetCurrentReplayInfo(ReplayRecordInfo& info) const override
@@ -561,35 +543,16 @@ namespace OpenRCT2
             return true;
         }
 
-        bool ReadReplayFromFile(const std::string& file, MemoryStream& stream)
-        {
-            FILE* fp = fopen(file.c_str(), "rb");
-            if (fp == nullptr)
-                return false;
-
-            char buffer[128];
-            while (feof(fp) == 0)
-            {
-                size_t numBytesRead = fread(buffer, 1, 128, fp);
-                if (numBytesRead == 0)
-                    break;
-                stream.Write(buffer, numBytesRead);
-            }
-
-            fclose(fp);
-            return true;
-        }
-
         /**
          * Returns true if decompression was not needed or succeeded
          * @param stream
          * @return
          */
-        bool TryDecompress(MemoryStream& stream)
+        MemoryStream DecompressFile(FileStream& fileStream)
         {
             ReplayRecordFile recFile;
-            stream.SetPosition(0);
+            fileStream.SetPosition(0);
-            DataSerialiser fileSerializer(false, stream);
+            DataSerialiser fileSerializer(false, fileStream);
             fileSerializer << recFile.magic;
             fileSerializer << recFile.version;
 
@@ -598,52 +561,49 @@ namespace OpenRCT2
                 fileSerializer << recFile.uncompressedSize;
                 fileSerializer << recFile.data;
 
-                auto buff = std::make_unique<unsigned char[]>(recFile.uncompressedSize);
+                MemoryStream decompressed;
-                unsigned long outSize = recFile.uncompressedSize;
+                bool decompressStatus = true;
-                uncompress(
+
-                    static_cast<unsigned char*>(buff.get()), &outSize,
+                recFile.data.SetPosition(0);
-                    static_cast<const unsigned char*>(recFile.data.GetData()), recFile.data.GetLength());
+                decompressStatus = Compression::zlibDecompress(
-                if (outSize != recFile.uncompressedSize)
+                    recFile.data, static_cast<size_t>(recFile.data.GetLength()), decompressed,
-                {
+                    static_cast<size_t>(recFile.uncompressedSize), Compression::ZlibHeaderType::zlib);
-                    return false;
+
-                }
+                if (!decompressStatus)
-                stream.SetPosition(0);
+                    throw IOException("Decompression Error");
-                stream.Write(buff.get(), outSize);
+
+                recFile.data = std::move(decompressed);
+            }
+            else
+            {
+                // Read whole file into memory
+                fileStream.SetPosition(0);
+                recFile.data.CopyFromStream(fileStream, fileStream.GetLength());
             }
 
-            return true;
+            return recFile.data;
         }
 
         bool ReadReplayData(const std::string& file, ReplayRecordData& data)
         {
-            MemoryStream stream;
+            fs::path filePath = file;
+            if (filePath.extension() != ".parkrep")
+                filePath += ".parkrep";
 
-            std::string fileName = file;
+            if (filePath.is_relative())
-            if (fileName.size() < 5 || fileName.substr(fileName.size() - 5) != ".parkrep")
             {
-                fileName += ".parkrep";
+                fs::path replayPath = GetContext()->GetPlatformEnvironment().GetDirectoryPath(
+                                          DirBase::user, DirId::replayRecordings)
+                    / filePath;
+                if (fs::is_regular_file(replayPath))
+                    filePath = replayPath;
             }
 
-            std::string outPath = GetContext()->GetPlatformEnvironment().GetDirectoryPath(
+            if (!fs::is_regular_file(filePath))
-                DirBase::user, DirId::replayRecordings);
-            std::string outFile = Path::Combine(outPath, fileName);
-
-            bool loaded = false;
-            if (ReadReplayFromFile(outFile, stream))
-            {
-                data.filePath = outFile;
-                loaded = true;
-            }
-            else if (ReadReplayFromFile(file, stream))
-            {
-                data.filePath = file;
-                loaded = true;
-            }
-            if (!loaded)
                 return false;
 
-            if (!TryDecompress(stream))
+            FileStream fileStream(filePath, FileMode::open);
-                return false;
+            MemoryStream stream = DecompressFile(fileStream);
 
             stream.SetPosition(0);
             DataSerialiser serialiser(false, stream);

src/openrct2/core/ChecksumStream.cpp

@@ -11,8 +11,6 @@
 
 #include "Endianness.h"
 
-#include <cstddef>
-
 namespace OpenRCT2
 {
 #ifndef DISABLE_NETWORK
@@ -25,23 +23,24 @@ namespace OpenRCT2
 
     void ChecksumStream::Write(const void* buffer, uint64_t length)
     {
-        uint64_t* hash = reinterpret_cast<uint64_t*>(_checksum.data());
         for (size_t i = 0; i < length; i += sizeof(uint64_t))
         {
             const auto maxLen = std::min<size_t>(sizeof(uint64_t), length - i);
 
-            uint64_t temp{};
+            uint64_t value{};
-            std::memcpy(&temp, reinterpret_cast<const std::byte*>(buffer) + i, maxLen);
+            std::memcpy(&value, reinterpret_cast<const std::byte*>(buffer) + i, maxLen);
 
-            // Always use value as little endian, most common systems are little.
+            Step(value);
-    #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-            temp = ByteSwapBE(temp);
-    #endif
-
-            *hash ^= temp;
-            *hash *= kPrime;
         }
     }
 
+    void ChecksumStream::Step(uint64_t value)
+    {
+        uint64_t* hash = reinterpret_cast<uint64_t*>(_checksum.data());
+
+        *hash ^= value;
+        *hash *= kPrime;
+    }
+
 #endif
 } // namespace OpenRCT2

src/openrct2/core/ChecksumStream.h

@@ -11,7 +11,9 @@
 
 #include "IStream.hpp"
 
+#include <algorithm>
 #include <array>
+#include <cstring>
 
 namespace OpenRCT2
 {
@@ -34,7 +36,7 @@ namespace OpenRCT2
         const void* GetData() const override
         {
             return _checksum.data();
-        };
+        }
 
         ///////////////////////////////////////////////////////////////////////////
         // ISteam methods
@@ -72,41 +74,61 @@ namespace OpenRCT2
 
         void Write(const void* buffer, uint64_t length) override;
 
-        void Write1(const void* buffer) override
-        {
-            Write<1>(buffer);
-        }
-
-        void Write2(const void* buffer) override
-        {
-            Write<2>(buffer);
-        }
-
-        void Write4(const void* buffer) override
-        {
-            Write<4>(buffer);
-        }
-
-        void Write8(const void* buffer) override
-        {
-            Write<8>(buffer);
-        }
-
-        void Write16(const void* buffer) override
-        {
-            Write<16>(buffer);
-        }
-
         template<size_t N>
         void Write(const void* buffer)
         {
-            Write(buffer, N);
+            for (size_t i = 0; i < N; i += sizeof(uint64_t))
+            {
+                const auto maxLen = std::min<size_t>(sizeof(uint64_t), N - i);
+
+                uint64_t value{};
+                std::memcpy(&value, reinterpret_cast<const std::byte*>(buffer) + i, maxLen);
+
+                Step(value);
+            }
         }
 
         uint64_t TryRead(void* buffer, uint64_t length) override
         {
             return 0;
         }
+
+    private:
+        void Write1(const void* buffer) override
+        {
+            Step(*static_cast<const uint8_t*>(buffer));
+        }
+
+        void Write2(const void* buffer) override
+        {
+            WriteUnaligned<uint16_t>(buffer);
+        }
+
+        void Write4(const void* buffer) override
+        {
+            WriteUnaligned<uint32_t>(buffer);
+        }
+
+        void Write8(const void* buffer) override
+        {
+            WriteUnaligned<uint64_t>(buffer);
+        }
+
+        void Write16(const void* buffer) override
+        {
+            WriteUnaligned<uint64_t>(static_cast<const std::byte*>(buffer) + 0);
+            WriteUnaligned<uint64_t>(static_cast<const std::byte*>(buffer) + 8);
+        }
+
+        template<typename T>
+        void WriteUnaligned(const void* buffer)
+        {
+            T value;
+            std::memcpy(&value, buffer, sizeof(T));
+            Step(value);
+        }
+
+        void Step(uint64_t value);
     };
 
 } // namespace OpenRCT2

src/openrct2/core/Compression.cpp

@@ -10,167 +10,132 @@
 #include "Compression.h"
 
 #include "../Diagnostic.h"
-#include "zlib.h"
+#include "Guard.hpp"
+#include "StreamBuffer.hpp"
 
-#include <cassert>
+#ifndef ZLIB_CONST
-#include <stdexcept>
+    #define ZLIB_CONST
-#include <string>
+#endif
+
+#include <limits>
+#include <zlib.h>
 
 namespace OpenRCT2::Compression
 {
-    constexpr size_t kChunkSize = 128 * 1024;
+    constexpr size_t kZlibChunkSize = 128 * 1024;
+    constexpr size_t kZlibMaxChunkSize = static_cast<size_t>(std::numeric_limits<uInt>::max());
+    constexpr int kZlibWindowBits[] = { -15, 15, 15 + 16 };
 
-    // Compress the source to gzip-compatible stream, write to dest.
+    /*
-    // Mainly used for compressing the crashdumps
+     * Modified copy of compressBound() from zlib 1.3.1, with the argument type changed from ULong
-    bool gzipCompress(FILE* source, FILE* dest)
+     * (which may be only 32 bits) to uint64_t, and adds space for the default gzip header,
+     */
+    static uint64_t zlibCompressBound(uint64_t length)
     {
-        if (source == nullptr || dest == nullptr)
+        return length + (length >> 12) + (length >> 14) + (length >> 25) + 13 + (18 - 6);
-        {
+    }
-            return false;
+
-        }
+    bool zlibCompress(IStream& source, uint64_t sourceLength, IStream& dest, ZlibHeaderType header, int16_t level)
-        int ret, flush;
+    {
-        size_t have;
+        Guard::Assert(sourceLength <= source.GetLength() - source.GetPosition());
+
+        int ret;
+        StreamReadBuffer sourceBuf(source, sourceLength, kZlibChunkSize);
+        StreamWriteBuffer destBuf(dest, zlibCompressBound(sourceLength), kZlibChunkSize);
+
         z_stream strm{};
-        strm.zalloc = Z_NULL;
+        ret = deflateInit2(&strm, level, Z_DEFLATED, kZlibWindowBits[static_cast<int>(header)], 8, Z_DEFAULT_STRATEGY);
-        strm.zfree = Z_NULL;
-        strm.opaque = Z_NULL;
-        unsigned char in[kChunkSize];
-        unsigned char out[kChunkSize];
-        int windowBits = 15;
-        int GZIP_ENCODING = 16;
-        ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, windowBits | GZIP_ENCODING, 8, Z_DEFAULT_STRATEGY);
         if (ret != Z_OK)
         {
             LOG_ERROR("Failed to initialise stream");
             return false;
         }
+
         do
         {
-            strm.avail_in = uInt(fread(in, 1, kChunkSize, source));
+            auto readBlock = sourceBuf.ReadBlock(source, kZlibMaxChunkSize);
-            if (ferror(source))
+            strm.next_in = static_cast<const Bytef*>(readBlock.first);
-            {
+            strm.avail_in = static_cast<uInt>(readBlock.second);
-                deflateEnd(&strm);
+
-                LOG_ERROR("Failed to read data from source");
-                return false;
-            }
-            flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
-            strm.next_in = in;
             do
             {
-                strm.avail_out = kChunkSize;
+                Guard::Assert(destBuf, "Compression Overruns Ouput Size");
-                strm.next_out = out;
+
-                ret = deflate(&strm, flush);
+                auto writeBlock = destBuf.WriteBlockStart(kZlibMaxChunkSize);
+                strm.next_out = static_cast<Bytef*>(writeBlock.first);
+                strm.avail_out = static_cast<uInt>(writeBlock.second);
+
+                ret = deflate(&strm, sourceBuf ? Z_NO_FLUSH : Z_FINISH);
                 if (ret == Z_STREAM_ERROR)
                 {
                     LOG_ERROR("Failed to compress data");
-                    return false;
-                }
-                have = kChunkSize - strm.avail_out;
-                if (fwrite(out, 1, have, dest) != have || ferror(dest))
-                {
                     deflateEnd(&strm);
-                    LOG_ERROR("Failed to write data to destination");
                     return false;
                 }
-            } while (strm.avail_out == 0);
+
-        } while (flush != Z_FINISH);
+                destBuf.WriteBlockCommit(dest, writeBlock.second - strm.avail_out);
+            } while (strm.avail_in > 0);
+        } while (sourceBuf);
+
         deflateEnd(&strm);
         return true;
     }
 
-    std::vector<uint8_t> gzip(const void* data, const size_t dataLen)
+    bool zlibDecompress(IStream& source, uint64_t sourceLength, IStream& dest, uint64_t decompressLength, ZlibHeaderType header)
     {
-        assert(data != nullptr);
+        Guard::Assert(sourceLength <= source.GetLength() - source.GetPosition());
+
+        int ret;
+        StreamReadBuffer sourceBuf(source, sourceLength, kZlibChunkSize);
+        StreamWriteBuffer destBuf(dest, decompressLength, kZlibChunkSize);
 
-        std::vector<uint8_t> output;
         z_stream strm{};
-        strm.zalloc = Z_NULL;
+        ret = inflateInit2(&strm, kZlibWindowBits[static_cast<int>(header)]);
-        strm.zfree = Z_NULL;
+        if (ret != Z_OK)
-        strm.opaque = Z_NULL;
-
         {
-            const auto ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY);
+            LOG_ERROR("Failed to initialise stream");
-            if (ret != Z_OK)
+            return false;
-            {
-                throw std::runtime_error("deflateInit2 failed with error " + std::to_string(ret));
-            }
         }
 
-        int flush = 0;
-        const auto* src = static_cast<const Bytef*>(data);
-        size_t srcRemaining = dataLen;
         do
         {
-            const auto nextBlockSize = std::min(srcRemaining, kChunkSize);
+            auto readBlock = sourceBuf.ReadBlock(source, kZlibMaxChunkSize);
-            srcRemaining -= nextBlockSize;
+            strm.next_in = static_cast<const Bytef*>(readBlock.first);
+            strm.avail_in = static_cast<uInt>(readBlock.second);
 
-            flush = srcRemaining == 0 ? Z_FINISH : Z_NO_FLUSH;
-            strm.avail_in = static_cast<uInt>(nextBlockSize);
-            strm.next_in = const_cast<Bytef*>(src);
             do
             {
-                output.resize(output.size() + nextBlockSize);
+                if (!destBuf)
-                strm.avail_out = static_cast<uInt>(nextBlockSize);
+                {
-                strm.next_out = &output[output.size() - nextBlockSize];
+                    LOG_ERROR("Decompressed data larger than expected");
-                const auto ret = deflate(&strm, flush);
+                    inflateEnd(&strm);
+                    return false;
+                }
+
+                auto writeBlock = destBuf.WriteBlockStart(kZlibMaxChunkSize);
+                strm.next_out = static_cast<Bytef*>(writeBlock.first);
+                strm.avail_out = static_cast<uInt>(writeBlock.second);
+
+                ret = inflate(&strm, sourceBuf ? Z_NO_FLUSH : Z_FINISH);
                 if (ret == Z_STREAM_ERROR)
                 {
-                    throw std::runtime_error("deflate failed with error " + std::to_string(ret));
+                    LOG_ERROR("Failed to decompress data");
+                    inflateEnd(&strm);
+                    return false;
                 }
-                output.resize(output.size() - strm.avail_out);
-            } while (strm.avail_out == 0);
 
-            src += nextBlockSize;
+                destBuf.WriteBlockCommit(dest, writeBlock.second - strm.avail_out);
-        } while (flush != Z_FINISH);
+            } while (strm.avail_in > 0);
-        deflateEnd(&strm);
+        } while (sourceBuf);
-        return output;
-    }
-
-    std::vector<uint8_t> ungzip(const void* data, const size_t dataLen)
-    {
-        assert(data != nullptr);
-
-        std::vector<uint8_t> output;
-        z_stream strm{};
-        strm.zalloc = Z_NULL;
-        strm.zfree = Z_NULL;
-        strm.opaque = Z_NULL;
 
+        if (destBuf)
         {
-            const auto ret = inflateInit2(&strm, 15 | 16);
+            LOG_ERROR("Decompressed data smaller than expected");
-            if (ret != Z_OK)
+            inflateEnd(&strm);
-            {
+            return false;
-                throw std::runtime_error("inflateInit2 failed with error " + std::to_string(ret));
-            }
         }
 
-        int flush = 0;
-        const auto* src = static_cast<const Bytef*>(data);
-        size_t srcRemaining = dataLen;
-        do
-        {
-            const auto nextBlockSize = std::min(srcRemaining, kChunkSize);
-            srcRemaining -= nextBlockSize;
-
-            flush = srcRemaining == 0 ? Z_FINISH : Z_NO_FLUSH;
-            strm.avail_in = static_cast<uInt>(nextBlockSize);
-            strm.next_in = const_cast<Bytef*>(src);
-            do
-            {
-                output.resize(output.size() + nextBlockSize);
-                strm.avail_out = static_cast<uInt>(nextBlockSize);
-                strm.next_out = &output[output.size() - nextBlockSize];
-                const auto ret = inflate(&strm, flush);
-                if (ret == Z_STREAM_ERROR)
-                {
-                    throw std::runtime_error("deflate failed with error " + std::to_string(ret));
-                }
-                output.resize(output.size() - strm.avail_out);
-            } while (strm.avail_out == 0);
-
-            src += nextBlockSize;
-        } while (flush != Z_FINISH);
         inflateEnd(&strm);
-        return output;
+        return true;
     }
 } // namespace OpenRCT2::Compression

src/openrct2/core/Compression.h

@@ -9,13 +9,31 @@
 
 #pragma once
 
+#include "IStream.hpp"
+
 #include <cstdint>
 #include <cstdio>
 #include <vector>
 
 namespace OpenRCT2::Compression
 {
-    bool gzipCompress(FILE* source, FILE* dest);
+    // zlib doesn't use 0 as a real compression level, so use it to mean no compression
-    std::vector<uint8_t> gzip(const void* data, const size_t dataLen);
+    constexpr int16_t kNoCompressionLevel = 0;
-    std::vector<uint8_t> ungzip(const void* data, const size_t dataLen);
+
+    // Zlib methods, using the DEFLATE compression algorithm
+    constexpr int16_t kZlibDefaultCompressionLevel = -1; // zlib value for "default level"
+    constexpr int16_t kZlibMaxCompressionLevel = 9;
+
+    enum class ZlibHeaderType
+    {
+        none = 0,
+        zlib = 1,
+        gzip = 2,
+    };
+
+    bool zlibCompress(
+        IStream& source, uint64_t sourceLength, IStream& dest, ZlibHeaderType header,
+        int16_t level = kZlibDefaultCompressionLevel);
+    bool zlibDecompress(
+        IStream& source, uint64_t sourceLength, IStream& dest, uint64_t decompressLength, ZlibHeaderType header);
 } // namespace OpenRCT2::Compression

src/openrct2/core/DataSerialiserTraits.h

@@ -45,14 +45,11 @@ struct DataSerializerTraitsEnum
 
     static void encode(OpenRCT2::IStream* stream, const T& val)
     {
-        TUnderlying temp = ByteSwapBE(static_cast<TUnderlying>(val));
+        stream->WriteValue(ByteSwapBE(static_cast<TUnderlying>(val)));
-        stream->Write(&temp);
     }
     static void decode(OpenRCT2::IStream* stream, T& val)
     {
-        TUnderlying temp;
+        val = static_cast<T>(ByteSwapBE(stream->ReadValue<TUnderlying>()));
-        stream->Read(&temp);
-        val = static_cast<T>(ByteSwapBE(temp));
     }
     static void log(OpenRCT2::IStream* stream, const T& val)
     {
@@ -72,14 +69,11 @@ struct DataSerializerTraitsIntegral
 {
     static void encode(OpenRCT2::IStream* stream, const T& val)
     {
-        T temp = ByteSwapBE(val);
+        stream->WriteValue(ByteSwapBE(val));
-        stream->Write(&temp);
     }
     static void decode(OpenRCT2::IStream* stream, T& val)
     {
-        T temp;
+        val = ByteSwapBE(stream->ReadValue<T>());
-        stream->Read(&temp);
-        val = ByteSwapBE(temp);
     }
     static void log(OpenRCT2::IStream* stream, const T& val)
     {
@@ -96,11 +90,11 @@ struct DataSerializerTraitsT<bool>
 {
     static void encode(OpenRCT2::IStream* stream, const bool& val)
     {
-        stream->Write(&val);
+        stream->WriteValue(val);
     }
     static void decode(OpenRCT2::IStream* stream, bool& val)
     {
-        stream->Read(&val);
+        stream->ReadValue(val);
     }
     static void log(OpenRCT2::IStream* stream, const bool& val)
     {
@@ -162,19 +156,16 @@ struct DataSerializerTraitsT<std::string>
     static void encode(OpenRCT2::IStream* stream, const std::string& str)
     {
         uint16_t len = static_cast<uint16_t>(str.size());
-        uint16_t swapped = ByteSwapBE(len);
+        stream->WriteValue(ByteSwapBE(len));
-        stream->Write(&swapped);
         if (len == 0)
         {
             return;
         }
-        stream->WriteArray(str.c_str(), len);
+        stream->Write(str.c_str(), len);
     }
     static void decode(OpenRCT2::IStream* stream, std::string& res)
     {
-        uint16_t len;
+        uint16_t len = ByteSwapBE(stream->ReadValue<uint16_t>());
-        stream->Read(&len);
-        len = ByteSwapBE(len);
         if (len == 0)
         {
             res.clear();
@@ -199,15 +190,11 @@ struct DataSerializerTraitsT<NetworkPlayerId_t>
 {
     static void encode(OpenRCT2::IStream* stream, const NetworkPlayerId_t& val)
     {
-        uint32_t temp = static_cast<uint32_t>(val.id);
+        stream->WriteValue(ByteSwapBE(val.id));
-        temp = ByteSwapBE(temp);
-        stream->Write(&temp);
     }
     static void decode(OpenRCT2::IStream* stream, NetworkPlayerId_t& val)
     {
-        uint32_t temp;
+        val.id = ByteSwapBE(stream->ReadValue<int32_t>());
-        stream->Read(&temp);
-        val.id = static_cast<decltype(val.id)>(ByteSwapBE(temp));
     }
     static void log(OpenRCT2::IStream* stream, const NetworkPlayerId_t& val)
     {
@@ -288,9 +275,7 @@ struct DataSerializerTraitsPODArray
 {
     static void encode(OpenRCT2::IStream* stream, const _Ty (&val)[_Size])
     {
-        uint16_t len = static_cast<uint16_t>(_Size);
+        stream->WriteValue(ByteSwapBE(static_cast<uint16_t>(_Size)));
-        uint16_t swapped = ByteSwapBE(len);
-        stream->Write(&swapped);
 
         DataSerializerTraits<_Ty> s;
         for (auto&& sub : val)
@@ -300,9 +285,7 @@ struct DataSerializerTraitsPODArray
     }
     static void decode(OpenRCT2::IStream* stream, _Ty (&val)[_Size])
     {
-        uint16_t len;
+        uint16_t len = ByteSwapBE(stream->ReadValue<uint16_t>());
-        stream->Read(&len);
-        len = ByteSwapBE(len);
 
         if (len != _Size)
             throw std::runtime_error("Invalid size, can't decode");
@@ -362,9 +345,7 @@ struct DataSerializerTraitsT<std::array<_Ty, _Size>>
 {
     static void encode(OpenRCT2::IStream* stream, const std::array<_Ty, _Size>& val)
     {
-        uint16_t len = static_cast<uint16_t>(_Size);
+        stream->WriteValue(ByteSwapBE(static_cast<uint16_t>(_Size)));
-        uint16_t swapped = ByteSwapBE(len);
-        stream->Write(&swapped);
 
         DataSerializerTraits<_Ty> s;
         for (auto&& sub : val)
@@ -374,9 +355,7 @@ struct DataSerializerTraitsT<std::array<_Ty, _Size>>
     }
     static void decode(OpenRCT2::IStream* stream, std::array<_Ty, _Size>& val)
     {
-        uint16_t len;
+        uint16_t len = ByteSwapBE(stream->ReadValue<uint16_t>());
-        stream->Read(&len);
-        len = ByteSwapBE(len);
 
         if (len != _Size)
             throw std::runtime_error("Invalid size, can't decode");
@@ -405,9 +384,7 @@ struct DataSerializerTraitsT<std::vector<_Ty>>
 {
     static void encode(OpenRCT2::IStream* stream, const std::vector<_Ty>& val)
     {
-        uint16_t len = static_cast<uint16_t>(val.size());
+        stream->WriteValue(ByteSwapBE(static_cast<uint16_t>(val.size())));
-        uint16_t swapped = ByteSwapBE(len);
-        stream->Write(&swapped);
 
         DataSerializerTraits<_Ty> s;
         for (auto&& sub : val)
@@ -417,9 +394,7 @@ struct DataSerializerTraitsT<std::vector<_Ty>>
     }
     static void decode(OpenRCT2::IStream* stream, std::vector<_Ty>& val)
     {
-        uint16_t len;
+        uint16_t len = ByteSwapBE(stream->ReadValue<uint16_t>());
-        stream->Read(&len);
-        len = ByteSwapBE(len);
 
         DataSerializerTraits<_Ty> s;
         for (auto i = 0; i < len; ++i)
@@ -620,14 +595,11 @@ struct DataSerializerTraitsT<NetworkCheatType_t>
 {
     static void encode(OpenRCT2::IStream* stream, const NetworkCheatType_t& val)
     {
-        uint32_t temp = ByteSwapBE(val.id);
+        stream->WriteValue(ByteSwapBE(val.id));
-        stream->Write(&temp);
     }
     static void decode(OpenRCT2::IStream* stream, NetworkCheatType_t& val)
     {
-        uint32_t temp;
+        val.id = ByteSwapBE(stream->ReadValue<int32_t>());
-        stream->Read(&temp);
-        val.id = ByteSwapBE(temp);
     }
     static void log(OpenRCT2::IStream* stream, const NetworkCheatType_t& val)
     {
@@ -641,15 +613,12 @@ struct DataSerializerTraitsT<RCTObjectEntry>
 {
     static void encode(OpenRCT2::IStream* stream, const RCTObjectEntry& val)
     {
-        uint32_t temp = ByteSwapBE(val.flags);
+        stream->WriteValue(ByteSwapBE(val.flags));
-        stream->Write(&temp);
         stream->WriteArray(val.nameWOC, 12);
     }
     static void decode(OpenRCT2::IStream* stream, RCTObjectEntry& val)
     {
-        uint32_t temp;
+        val.flags = ByteSwapBE(stream->ReadValue<uint32_t>());
-        stream->Read(&temp);
-        val.flags = ByteSwapBE(temp);
         auto str = stream->ReadArray<char>(12);
         memcpy(val.nameWOC, str.get(), 12);
     }
@@ -690,7 +659,7 @@ struct DataSerializerTraitsT<ObjectEntryDescriptor>
         else
         {
             auto type = static_cast<ObjectType>(stream->ReadValue<uint8_t>());
-            auto identifier = stream->ReadStdString();
+            auto identifier = stream->ReadString();
             val = ObjectEntryDescriptor(type, identifier);
         }
     }
@@ -709,21 +678,21 @@ struct DataSerializerTraitsT<TrackDesignTrackElement>
 {
     static void encode(OpenRCT2::IStream* stream, const TrackDesignTrackElement& val)
     {
-        stream->Write(&val.type);
+        stream->WriteValue(val.type);
-        stream->Write(&val.flags);
+        stream->WriteValue(val.flags);
-        stream->Write(&val.colourScheme);
+        stream->WriteValue(val.colourScheme);
-        stream->Write(&val.stationIndex);
+        stream->WriteValue(val.stationIndex);
-        stream->Write(&val.brakeBoosterSpeed);
+        stream->WriteValue(val.brakeBoosterSpeed);
-        stream->Write(&val.seatRotation);
+        stream->WriteValue(val.seatRotation);
     }
     static void decode(OpenRCT2::IStream* stream, TrackDesignTrackElement& val)
     {
-        stream->Read(&val.type);
+        stream->ReadValue(val.type);
-        stream->Read(&val.flags);
+        stream->ReadValue(val.flags);
-        stream->Read(&val.colourScheme);
+        stream->ReadValue(val.colourScheme);
-        stream->Read(&val.stationIndex);
+        stream->ReadValue(val.stationIndex);
-        stream->Read(&val.brakeBoosterSpeed);
+        stream->ReadValue(val.brakeBoosterSpeed);
-        stream->Read(&val.seatRotation);
+        stream->ReadValue(val.seatRotation);
     }
     static void log(OpenRCT2::IStream* stream, const TrackDesignTrackElement& val)
     {
@@ -738,13 +707,13 @@ struct DataSerializerTraitsT<TrackDesignMazeElement>
 {
     static void encode(OpenRCT2::IStream* stream, const TrackDesignMazeElement& val)
     {
-        stream->Write(&val.location);
+        stream->WriteValue(val.location);
-        stream->Write(&val.mazeEntry);
+        stream->WriteValue(val.mazeEntry);
     }
     static void decode(OpenRCT2::IStream* stream, TrackDesignMazeElement& val)
     {
-        stream->Read(&val.location);
+        stream->ReadValue(val.location);
-        stream->Read(&val.mazeEntry);
+        stream->ReadValue(val.mazeEntry);
     }
     static void log(OpenRCT2::IStream* stream, const TrackDesignMazeElement& val)
     {
@@ -761,13 +730,13 @@ struct DataSerializerTraitsT<TrackDesignEntranceElement>
 {
     static void encode(OpenRCT2::IStream* stream, const TrackDesignEntranceElement& val)
     {
-        stream->Write(&val.location);
+        stream->WriteValue(val.location);
-        stream->Write(&val.isExit);
+        stream->WriteValue(val.isExit);
     }
     static void decode(OpenRCT2::IStream* stream, TrackDesignEntranceElement& val)
     {
-        stream->Read(&val.location);
+        stream->ReadValue(val.location);
-        stream->Read(&val.isExit);
+        stream->ReadValue(val.isExit);
     }
     static void log(OpenRCT2::IStream* stream, const TrackDesignEntranceElement& val)
     {
@@ -784,21 +753,21 @@ struct DataSerializerTraitsT<TrackDesignSceneryElement>
 {
     static void encode(OpenRCT2::IStream* stream, const TrackDesignSceneryElement& val)
     {
-        stream->Write(&val.loc);
+        stream->WriteValue(val.loc);
-        stream->Write(&val.flags);
+        stream->WriteValue(val.flags);
-        stream->Write(&val.primaryColour);
+        stream->WriteValue(val.primaryColour);
-        stream->Write(&val.secondaryColour);
+        stream->WriteValue(val.secondaryColour);
-        stream->Write(&val.tertiaryColour);
+        stream->WriteValue(val.tertiaryColour);
         DataSerializerTraits<ObjectEntryDescriptor> s;
         s.encode(stream, val.sceneryObject);
     }
     static void decode(OpenRCT2::IStream* stream, TrackDesignSceneryElement& val)
     {
-        stream->Read(&val.loc);
+        stream->ReadValue(val.loc);
-        stream->Read(&val.flags);
+        stream->ReadValue(val.flags);
-        stream->Read(&val.primaryColour);
+        stream->ReadValue(val.primaryColour);
-        stream->Read(&val.secondaryColour);
+        stream->ReadValue(val.secondaryColour);
-        stream->Read(&val.tertiaryColour);
+        stream->ReadValue(val.tertiaryColour);
         DataSerializerTraits<ObjectEntryDescriptor> s;
         s.decode(stream, val.sceneryObject);
     }
@@ -821,15 +790,15 @@ struct DataSerializerTraitsT<TrackColour>
 {
     static void encode(OpenRCT2::IStream* stream, const TrackColour& val)
     {
-        stream->Write(&val.main);
+        stream->WriteValue(val.main);
-        stream->Write(&val.additional);
+        stream->WriteValue(val.additional);
-        stream->Write(&val.supports);
+        stream->WriteValue(val.supports);
     }
     static void decode(OpenRCT2::IStream* stream, TrackColour& val)
     {
-        stream->Read(&val.main);
+        stream->ReadValue(val.main);
-        stream->Read(&val.additional);
+        stream->ReadValue(val.additional);
-        stream->Read(&val.supports);
+        stream->ReadValue(val.supports);
     }
     static void log(OpenRCT2::IStream* stream, const TrackColour& val)
     {
@@ -845,15 +814,15 @@ struct DataSerializerTraitsT<VehicleColour>
 {
     static void encode(OpenRCT2::IStream* stream, const VehicleColour& val)
     {
-        stream->Write(&val.Body);
+        stream->WriteValue(val.Body);
-        stream->Write(&val.Trim);
+        stream->WriteValue(val.Trim);
-        stream->Write(&val.Tertiary);
+        stream->WriteValue(val.Tertiary);
     }
     static void decode(OpenRCT2::IStream* stream, VehicleColour& val)
     {
-        stream->Read(&val.Body);
+        stream->ReadValue(val.Body);
-        stream->Read(&val.Trim);
+        stream->ReadValue(val.Trim);
-        stream->Read(&val.Tertiary);
+        stream->ReadValue(val.Tertiary);
     }
     static void log(OpenRCT2::IStream* stream, const VehicleColour& val)
     {
@@ -868,15 +837,15 @@ struct DataSerializerTraitsT<RatingTuple>
 {
     static void encode(OpenRCT2::IStream* stream, const RatingTuple& val)
     {
-        stream->Write(&val.excitement);
+        stream->WriteValue(val.excitement);
-        stream->Write(&val.intensity);
+        stream->WriteValue(val.intensity);
-        stream->Write(&val.nausea);
+        stream->WriteValue(val.nausea);
     }
     static void decode(OpenRCT2::IStream* stream, RatingTuple& val)
     {
-        stream->Read(&val.excitement);
+        stream->ReadValue(val.excitement);
-        stream->Read(&val.intensity);
+        stream->ReadValue(val.intensity);
-        stream->Read(&val.nausea);
+        stream->ReadValue(val.nausea);
     }
     static void log(OpenRCT2::IStream* stream, const RatingTuple& val)
     {
@@ -893,13 +862,11 @@ struct DataSerializerTraitsT<IntensityRange>
 {
     static void encode(OpenRCT2::IStream* stream, const IntensityRange& val)
     {
-        uint8_t temp = uint8_t(val);
+        stream->WriteValue(static_cast<uint8_t>(val));
-        stream->Write(&temp);
     }
     static void decode(OpenRCT2::IStream* stream, IntensityRange& val)
     {
-        auto temp = stream->ReadValue<uint8_t>();
+        val = static_cast<IntensityRange>(stream->ReadValue<uint8_t>());
-        val = IntensityRange(temp);
     }
     static void log(OpenRCT2::IStream* stream, const IntensityRange& val)
     {
@@ -914,17 +881,17 @@ struct DataSerializerTraitsT<PeepThought>
 {
     static void encode(OpenRCT2::IStream* stream, const PeepThought& val)
     {
-        stream->Write(&val.type);
+        stream->WriteValue(val.type);
-        stream->Write(&val.item);
+        stream->WriteValue(val.item);
-        stream->Write(&val.freshness);
+        stream->WriteValue(val.freshness);
-        stream->Write(&val.fresh_timeout);
+        stream->WriteValue(val.fresh_timeout);
     }
     static void decode(OpenRCT2::IStream* stream, PeepThought& val)
     {
-        stream->Read(&val.type);
+        stream->ReadValue(val.type);
-        stream->Read(&val.item);
+        stream->ReadValue(val.item);
-        stream->Read(&val.freshness);
+        stream->ReadValue(val.freshness);
-        stream->Read(&val.fresh_timeout);
+        stream->ReadValue(val.fresh_timeout);
     }
     static void log(OpenRCT2::IStream* stream, const PeepThought& val)
     {
@@ -1007,11 +974,11 @@ struct DataSerializerTraitsT<Banner>
     {
         DataSerializerTraits<BannerIndex>().decode(stream, banner.id);
         DataSerializerTraits<ObjectEntryIndex>().decode(stream, banner.type);
-        stream->Read(&banner.flags);
+        stream->ReadValue(banner.flags);
-        banner.text = stream->ReadStdString();
+        banner.text = stream->ReadString();
-        stream->Read(&banner.colour);
+        stream->ReadValue(banner.colour);
         DataSerializerTraits<RideId>().decode(stream, banner.rideIndex);
-        stream->Read(&banner.textColour);
+        stream->ReadValue(banner.textColour);
         DataSerializerTraits<TileCoordsXY>().decode(stream, banner.position);
     }
 

src/openrct2/core/FileStream.cpp

@@ -195,9 +195,4 @@ namespace OpenRCT2
         return readBytes;
     }
 
-    const void* FileStream::GetData() const
-    {
-        return nullptr;
-    }
-
 } // namespace OpenRCT2

src/openrct2/core/FileStream.h

@@ -52,7 +52,6 @@ namespace OpenRCT2
         void Read(void* buffer, uint64_t length) override;
         void Write(const void* buffer, uint64_t length) override;
         uint64_t TryRead(void* buffer, uint64_t length) override;
-        const void* GetData() const override;
     };
 
 } // namespace OpenRCT2

src/openrct2/core/IStream.cpp

@@ -9,13 +9,26 @@
 
 #include "IStream.hpp"
 
+#include "StreamBuffer.hpp"
 #include "String.hpp"
 
 #include <vector>
 
 namespace OpenRCT2
 {
-    std::string IStream::ReadStdString()
+    constexpr size_t kIStreamCopyBufferLength = 16 * 1024;
+
+    void IStream::CopyFromStream(IStream& stream, uint64_t length)
+    {
+        StreamReadBuffer buffer(stream, length, kIStreamCopyBufferLength);
+        while (buffer)
+        {
+            auto block = buffer.ReadBlock(stream);
+            this->Write(block.first, block.second);
+        }
+    }
+
+    std::string IStream::ReadString()
     {
         std::string result;
         uint8_t ch;
@@ -26,32 +39,11 @@ namespace OpenRCT2
         return result;
     }
 
-    void IStream::WriteString(const utf8* str)
+    void IStream::WriteString(std::string_view str)
     {
-        if (str == nullptr)
+        // if the string contains any null characters, then stop the write at the first one
-        {
+        str = String::toStringView(str.data(), str.size());
-            WriteValue<uint8_t>(0);
+        Write(str.data(), str.size());
-        }
-        else
-        {
-            size_t numBytes = String::sizeOf(str) + 1;
-            Write(str, numBytes);
-        }
-    }
-
-    void IStream::WriteString(const std::string_view str)
-    {
-        for (const auto c : str)
-        {
-            if (c == '\0')
-                break;
-            WriteValue<uint8_t>(c);
-        }
         WriteValue<uint8_t>(0);
     }
-
-    void IStream::WriteString(const std::string& str)
-    {
-        WriteString(str.c_str());
-    }
 } // namespace OpenRCT2

src/openrct2/core/IStream.hpp

@@ -59,11 +59,34 @@ namespace OpenRCT2
 
         virtual uint64_t TryRead(void* buffer, uint64_t length) = 0;
 
-        virtual const void* GetData() const = 0;
+        virtual const void* GetData() const
+        {
+            return nullptr;
+        }
+
+        virtual void CopyFromStream(IStream& stream, uint64_t length);
+
+        ///////////////////////////////////////////////////////////////////////////
+        // Direct Read/Write methods, class can override them if they're memory-backed.
+        ///////////////////////////////////////////////////////////////////////////
+        virtual const void* ReadDirect(size_t length)
+        {
+            return nullptr;
+        }
+
+        virtual void* WriteDirectStart(size_t maxLength)
+        {
+            return nullptr;
+        }
+
+        virtual void WriteDirectCommit(size_t length)
+        {
+        }
 
         ///////////////////////////////////////////////////////////////////////////
         // Fast path methods, class can override them to use specialised copies.
         ///////////////////////////////////////////////////////////////////////////
+    private:
         virtual void Read1(void* buffer)
         {
             Read(buffer, 1);
@@ -109,90 +132,88 @@ namespace OpenRCT2
         ///////////////////////////////////////////////////////////////////////////
         // Helper methods
         ///////////////////////////////////////////////////////////////////////////
+    public:
         /**
-         * Reads the size of the given type from the stream directly into the given address.
+         * Reads the given type from the stream
          */
         template<typename T>
-        void Read(T* value)
+        void ReadValue(T& value)
         {
             // Selects the best path at compile time
             if constexpr (sizeof(T) == 1)
             {
-                Read1(value);
+                Read1(&value);
             }
             else if constexpr (sizeof(T) == 2)
             {
-                Read2(value);
+                Read2(&value);
             }
             else if constexpr (sizeof(T) == 4)
             {
-                Read4(value);
+                Read4(&value);
             }
             else if constexpr (sizeof(T) == 8)
             {
-                Read8(value);
+                Read8(&value);
             }
             else if constexpr (sizeof(T) == 16)
             {
-                Read16(value);
+                Read16(&value);
             }
             else
             {
-                Read(value, sizeof(T));
+                Read(&value, sizeof(T));
             }
         }
 
         /**
-         * Writes the size of the given type to the stream directly from the given address.
+         * Reads the given type from the stream and returns the value directly
-         */
-        template<typename T>
-        void Write(const T* value)
-        {
-            // Selects the best path at compile time
-            if constexpr (sizeof(T) == 1)
-            {
-                Write1(value);
-            }
-            else if constexpr (sizeof(T) == 2)
-            {
-                Write2(value);
-            }
-            else if constexpr (sizeof(T) == 4)
-            {
-                Write4(value);
-            }
-            else if constexpr (sizeof(T) == 8)
-            {
-                Write8(value);
-            }
-            else if constexpr (sizeof(T) == 16)
-            {
-                Write16(value);
-            }
-            else
-            {
-                Write(value, sizeof(T));
-            }
-        }
-
-        /**
-         * Reads the given type from the stream. Use this only for small types (e.g. int8_t, int64_t, double)
          */
         template<typename T>
         T ReadValue()
         {
-            T buffer;
+            T value;
-            Read(&buffer);
+            ReadValue(value);
-            return buffer;
+            return value;
         }
 
         /**
-         * Writes the given type to the stream. Use this only for small types (e.g. int8_t, int64_t, double)
+         * Writes the given type to the stream
          */
         template<typename T>
-        void WriteValue(const T value)
+        void WriteValue(const T& value)
         {
-            Write(&value);
+            // Selects the best path at compile time
+            if constexpr (sizeof(T) == 1)
+            {
+                Write1(&value);
+            }
+            else if constexpr (sizeof(T) == 2)
+            {
+                Write2(&value);
+            }
+            else if constexpr (sizeof(T) == 4)
+            {
+                Write4(&value);
+            }
+            else if constexpr (sizeof(T) == 8)
+            {
+                Write8(&value);
+            }
+            else if constexpr (sizeof(T) == 16)
+            {
+                Write16(&value);
+            }
+            else
+            {
+                Write(&value, sizeof(T));
+            }
+        }
+
+        template<typename T>
+        void ReadArray(T* buffer, size_t count)
+        {
+            Read(buffer, sizeof(T) * count);
         }
 
         template<typename T>
@@ -209,10 +230,8 @@ namespace OpenRCT2
             Write(buffer, sizeof(T) * count);
         }
 
-        std::string ReadStdString();
+        std::string ReadString();
-        void WriteString(const utf8* str);
+        void WriteString(std::string_view string);
-        void WriteString(const std::string_view string);
-        void WriteString(const std::string& string);
     };
 
 } // namespace OpenRCT2

src/openrct2/core/MemoryStream.cpp

@@ -9,6 +9,7 @@
 
 #include "MemoryStream.h"
 
+#include "Guard.hpp"
 #include "Memory.hpp"
 
 #include <cstring>
@@ -23,17 +24,23 @@ namespace OpenRCT2
 
         if (_access & MEMORY_ACCESS::OWNER)
         {
-            _data = Memory::Allocate<void>(_dataCapacity);
+            _data = Memory::Allocate<uint8_t>(_dataCapacity);
             std::memcpy(_data, copy._data, _dataCapacity);
-            _position = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(_data) + copy.GetPosition());
         }
     }
 
     MemoryStream::MemoryStream(size_t capacity)
     {
         _dataCapacity = capacity;
-        _data = Memory::Allocate<void>(capacity);
+        _data = Memory::Allocate<uint8_t>(capacity);
-        _position = _data;
+    }
+
+    MemoryStream::MemoryStream(const std::vector<uint8_t>& v)
+    {
+        _dataCapacity = v.size();
+        _dataSize = v.size();
+        _data = Memory::Allocate<uint8_t>(v.size());
+        std::memcpy(_data, v.data(), v.size());
     }
 
     MemoryStream::MemoryStream(void* data, size_t dataSize, uint8_t access)
@@ -41,23 +48,7 @@ namespace OpenRCT2
         _access = access;
         _dataCapacity = dataSize;
         _dataSize = dataSize;
-        _data = data;
+        _data = static_cast<uint8_t*>(data);
-        _position = _data;
-    }
-
-    MemoryStream::MemoryStream(const void* data, size_t dataSize)
-        : MemoryStream(const_cast<void*>(data), dataSize, MEMORY_ACCESS::READ)
-    {
-    }
-
-    MemoryStream::MemoryStream(std::vector<uint8_t>&& v)
-    {
-        _access = MEMORY_ACCESS::OWNER;
-        _dataCapacity = v.size();
-        _dataSize = v.size();
-        _data = Memory::Allocate<void>(v.size());
-        _position = _data;
-        std::memcpy(_data, v.data(), v.size());
     }
 
     MemoryStream::MemoryStream(MemoryStream&& mv) noexcept
@@ -68,7 +59,7 @@ namespace OpenRCT2
         , _position(mv._position)
     {
         mv._data = nullptr;
-        mv._position = nullptr;
+        mv._position = 0;
         mv._dataCapacity = 0;
         mv._dataSize = 0;
     }
@@ -76,18 +67,21 @@ namespace OpenRCT2
     MemoryStream::~MemoryStream()
     {
         if (_access & MEMORY_ACCESS::OWNER)
-        {
             Memory::Free(_data);
-        }
+
+        _data = nullptr;
+        _position = 0;
         _dataCapacity = 0;
         _dataSize = 0;
-        _data = nullptr;
     }
 
     MemoryStream& MemoryStream::operator=(MemoryStream&& mv) noexcept
     {
         if (this != &mv)
         {
+            if (_access & MEMORY_ACCESS::OWNER)
+                Memory::Free(_data);
+
             _access = mv._access;
             _dataCapacity = mv._dataCapacity;
             _data = mv._data;
@@ -95,7 +89,7 @@ namespace OpenRCT2
             _position = mv._position;
 
             mv._data = nullptr;
-            mv._position = nullptr;
+            mv._position = 0;
             mv._dataCapacity = 0;
             mv._dataSize = 0;
         }
@@ -103,172 +97,111 @@ namespace OpenRCT2
         return *this;
     }
 
-    const void* MemoryStream::GetData() const
-    {
-        return _data;
-    }
-
-    bool MemoryStream::CanRead() const
-    {
-        return (_access & MEMORY_ACCESS::READ) != 0;
-    }
-
-    bool MemoryStream::CanWrite() const
-    {
-        return (_access & MEMORY_ACCESS::WRITE) != 0;
-    }
-
-    uint64_t MemoryStream::GetLength() const
-    {
-        return _dataSize;
-    }
-
-    uint64_t MemoryStream::GetPosition() const
-    {
-        return static_cast<uint64_t>(reinterpret_cast<uintptr_t>(_position) - reinterpret_cast<uintptr_t>(_data));
-    }
-
     void MemoryStream::SetPosition(uint64_t position)
     {
-        Seek(position, STREAM_SEEK_BEGIN);
+        if (position > _dataSize)
+            throw IOException("New position out of bounds.");
+        _position = static_cast<size_t>(position);
     }
 
     void MemoryStream::Seek(int64_t offset, int32_t origin)
     {
-        uint64_t newPosition;
         switch (origin)
         {
             default:
             case STREAM_SEEK_BEGIN:
-                newPosition = offset;
+                SetPosition(offset);
                 break;
             case STREAM_SEEK_CURRENT:
-                newPosition = GetPosition() + offset;
+                SetPosition(_position + offset);
                 break;
             case STREAM_SEEK_END:
-                newPosition = _dataSize + offset;
+                SetPosition(_dataSize + offset);
                 break;
         }
-
-        if (newPosition > _dataSize)
-        {
-            throw IOException("New position out of bounds.");
-        }
-        _position = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(_data) + static_cast<uintptr_t>(newPosition));
     }
 
     void MemoryStream::Read(void* buffer, uint64_t length)
     {
-        uint64_t position = GetPosition();
+        if (_position + length > _dataSize)
-        if (position + length > _dataSize)
-        {
             throw IOException("Attempted to read past end of stream.");
-        }
 
-        std::memcpy(buffer, _position, length);
+        std::memcpy(buffer, _data + _position, length);
-        _position = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(_position) + length);
+        _position += static_cast<size_t>(length);
-    }
-
-    void MemoryStream::Read1(void* buffer)
-    {
-        Read<1>(buffer);
-    }
-
-    void MemoryStream::Read2(void* buffer)
-    {
-        Read<2>(buffer);
-    }
-
-    void MemoryStream::Read4(void* buffer)
-    {
-        Read<4>(buffer);
-    }
-
-    void MemoryStream::Read8(void* buffer)
-    {
-        Read<8>(buffer);
-    }
-
-    void MemoryStream::Read16(void* buffer)
-    {
-        Read<16>(buffer);
     }
 
     uint64_t MemoryStream::TryRead(void* buffer, uint64_t length)
     {
-        uint64_t remainingBytes = GetLength() - GetPosition();
+        uint64_t bytesToRead = std::min(length, static_cast<uint64_t>(_dataSize - _position));
-        uint64_t bytesToRead = std::min(length, remainingBytes);
+        std::memcpy(buffer, _data + _position, static_cast<size_t>(bytesToRead));
-        Read(buffer, bytesToRead);
+        _position += static_cast<size_t>(bytesToRead);
         return bytesToRead;
     }
 
     void MemoryStream::Write(const void* buffer, uint64_t length)
     {
-        uint64_t position = GetPosition();
+        EnsureWriteCapacity(length);
-        uint64_t nextPosition = position + length;
+        std::memcpy(_data + _position, buffer, length);
-        if (nextPosition > _dataCapacity)
+        _position += static_cast<size_t>(length);
+        _dataSize = std::max(_dataSize, _position);
+    }
+
+    void MemoryStream::Clear()
+    {
+        _dataSize = 0;
+        _position = 0;
+    }
+
+    const void* MemoryStream::ReadDirect(size_t length)
+    {
+        if (_position + length > _dataSize)
+            throw IOException("Attempted to read past end of stream.");
+
+        const void* readPosition = _data + _position;
+        _position += length;
+        return readPosition;
+    }
+
+    void* MemoryStream::WriteDirectStart(size_t maxLength)
+    {
+        EnsureWriteCapacity(maxLength);
+        return _data + _position;
+    }
+
+    void MemoryStream::WriteDirectCommit(size_t length)
+    {
+        Guard::Assert(_dataCapacity >= _position + length);
+        _position += length;
+        _dataSize = std::max(_dataSize, _position);
+    }
+
+    void MemoryStream::CopyFromStream(IStream& stream, uint64_t length)
+    {
+        EnsureWriteCapacity(length);
+        // Copy directly into storage to avoid intermediate buffer
+        stream.Read(_data + _position, length);
+        _position += static_cast<size_t>(length);
+        _dataSize = std::max(_dataSize, _position);
+    }
+
+    void MemoryStream::EnsureWriteCapacity(uint64_t length)
+    {
+        if (_position + static_cast<size_t>(length) > _dataCapacity)
         {
             if (_access & MEMORY_ACCESS::OWNER)
             {
-                EnsureCapacity(static_cast<size_t>(nextPosition));
+                // resize to the larger of the requested capacity and double the current capacity, with a minimum of 8.
+                size_t newCapacity = std::max(_position + static_cast<size_t>(length), _dataCapacity * 2);
+                newCapacity = std::max<size_t>(newCapacity, 8);
+
+                _dataCapacity = newCapacity;
+                _data = Memory::Reallocate(_data, _dataCapacity);
             }
             else
             {
                 throw IOException("Attempted to write past end of stream.");
             }
         }
-
-        std::memcpy(_position, buffer, length);
-        _position = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(_position) + length);
-        _dataSize = std::max<size_t>(_dataSize, static_cast<size_t>(nextPosition));
-    }
-
-    void MemoryStream::Write1(const void* buffer)
-    {
-        Write<1>(buffer);
-    }
-
-    void MemoryStream::Write2(const void* buffer)
-    {
-        Write<2>(buffer);
-    }
-
-    void MemoryStream::Write4(const void* buffer)
-    {
-        Write<4>(buffer);
-    }
-
-    void MemoryStream::Write8(const void* buffer)
-    {
-        Write<8>(buffer);
-    }
-
-    void MemoryStream::Write16(const void* buffer)
-    {
-        Write<16>(buffer);
-    }
-
-    void MemoryStream::Clear()
-    {
-        _dataSize = 0;
-        SetPosition(0);
-    }
-
-    void MemoryStream::EnsureCapacity(size_t capacity)
-    {
-        if (_dataCapacity < capacity)
-        {
-            size_t newCapacity = std::max<size_t>(8, _dataCapacity);
-            while (newCapacity < capacity)
-            {
-                newCapacity *= 2;
-            }
-
-            uint64_t position = GetPosition();
-            _dataCapacity = newCapacity;
-            _data = Memory::Reallocate(_data, _dataCapacity);
-            _position = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(_data) + static_cast<uintptr_t>(position));
-        }
     }
 
 } // namespace OpenRCT2

src/openrct2/core/MemoryStream.h

@@ -11,6 +11,7 @@
 
 #include "IStream.hpp"
 
+#include <algorithm>
 #include <cstring>
 #include <vector>
 
@@ -32,89 +33,130 @@ namespace OpenRCT2
         uint8_t _access = MEMORY_ACCESS::READ | MEMORY_ACCESS::WRITE | MEMORY_ACCESS::OWNER;
         size_t _dataCapacity = 0;
         size_t _dataSize = 0;
-        void* _data = nullptr;
+        uint8_t* _data = nullptr;
-        void* _position = nullptr;
+        size_t _position = 0;
 
     public:
         MemoryStream() = default;
         MemoryStream(const MemoryStream& copy);
         MemoryStream(MemoryStream&& mv) noexcept;
         explicit MemoryStream(size_t capacity);
+        MemoryStream(const std::vector<uint8_t>& v);
         MemoryStream(void* data, size_t dataSize, uint8_t access = MEMORY_ACCESS::READ);
-        MemoryStream(const void* data, size_t dataSize);
+        MemoryStream(const void* data, size_t dataSize)
-        MemoryStream(std::vector<uint8_t>&& v);
+            : MemoryStream(const_cast<void*>(data), dataSize, MEMORY_ACCESS::READ)
+        {
+        }
         virtual ~MemoryStream();
 
         MemoryStream& operator=(MemoryStream&& mv) noexcept;
 
-        const void* GetData() const override;
-
         ///////////////////////////////////////////////////////////////////////////
         // ISteam methods
         ///////////////////////////////////////////////////////////////////////////
-        bool CanRead() const override;
-        bool CanWrite() const override;
 
-        uint64_t GetLength() const override;
+        const void* GetData() const override
-        uint64_t GetPosition() const override;
+        {
+            return _data;
+        }
+
+        bool CanRead() const override
+        {
+            return (_access & MEMORY_ACCESS::READ) != 0;
+        }
+        bool CanWrite() const override
+        {
+            return (_access & MEMORY_ACCESS::WRITE) != 0;
+        }
+
+        uint64_t GetLength() const override
+        {
+            return _dataSize;
+        }
+        uint64_t GetPosition() const override
+        {
+            return _position;
+        }
+
         void SetPosition(uint64_t position) override;
         void Seek(int64_t offset, int32_t origin) override;
 
         void Read(void* buffer, uint64_t length) override;
-        void Read1(void* buffer) override;
+        uint64_t TryRead(void* buffer, uint64_t length) override;
-        void Read2(void* buffer) override;
-        void Read4(void* buffer) override;
-        void Read8(void* buffer) override;
-        void Read16(void* buffer) override;
 
         template<size_t N>
         void Read(void* buffer)
         {
-            uint64_t position = GetPosition();
+            if (_position + N > _dataSize)
-            if (position + N > _dataSize)
-            {
                 throw IOException("Attempted to read past end of stream.");
-            }
 
-            std::memcpy(buffer, _position, N);
+            std::memcpy(buffer, _data + _position, N);
-            _position = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(_position) + N);
+            _position += static_cast<size_t>(N);
         }
 
         void Write(const void* buffer, uint64_t length) override;
-        void Write1(const void* buffer) override;
-        void Write2(const void* buffer) override;
-        void Write4(const void* buffer) override;
-        void Write8(const void* buffer) override;
-        void Write16(const void* buffer) override;
 
         template<size_t N>
         void Write(const void* buffer)
         {
-            uint64_t position = GetPosition();
+            EnsureWriteCapacity(N);
-            uint64_t nextPosition = position + N;
+            std::memcpy(_data + _position, buffer, N);
-            if (nextPosition > _dataCapacity)
+            _position += static_cast<size_t>(N);
-            {
+            _dataSize = std::max(_dataSize, _position);
-                if (_access & MEMORY_ACCESS::OWNER)
-                {
-                    EnsureCapacity(static_cast<size_t>(nextPosition));
-                }
-                else
-                {
-                    throw IOException("Attempted to write past end of stream.");
-                }
-            }
-
-            std::memcpy(_position, buffer, N);
-            _position = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(_position) + N);
-            _dataSize = std::max<size_t>(_dataSize, static_cast<size_t>(nextPosition));
         }
 
-        uint64_t TryRead(void* buffer, uint64_t length) override;
+        const void* ReadDirect(size_t length) override;
+        void* WriteDirectStart(size_t maxLength) override;
+        void WriteDirectCommit(size_t length) override;
+
+        void CopyFromStream(IStream& stream, uint64_t length) override;
 
         void Clear();
 
     private:
-        void EnsureCapacity(size_t capacity);
+        void Read1(void* buffer) override
+        {
+            Read<1>(buffer);
+        }
+        void Read2(void* buffer) override
+        {
+            Read<2>(buffer);
+        }
+        void Read4(void* buffer) override
+        {
+            Read<4>(buffer);
+        }
+        void Read8(void* buffer) override
+        {
+            Read<8>(buffer);
+        }
+        void Read16(void* buffer) override
+        {
+            Read<16>(buffer);
+        }
+
+        void Write1(const void* buffer) override
+        {
+            Write<1>(buffer);
+        }
+        void Write2(const void* buffer) override
+        {
+            Write<2>(buffer);
+        }
+        void Write4(const void* buffer) override
+        {
+            Write<4>(buffer);
+        }
+        void Write8(const void* buffer) override
+        {
+            Write<8>(buffer);
+        }
+        void Write16(const void* buffer) override
+        {
+            Write<16>(buffer);
+        }
+
+        void EnsureWriteCapacity(uint64_t length);
     };
 
 } // namespace OpenRCT2

src/openrct2/core/OrcaStream.hpp

@@ -90,35 +90,46 @@ namespace OpenRCT2
                     _chunks.push_back(entry);
                 }
 
-                // Read compressed data into buffer (read in blocks)
-                _buffer = MemoryStream{};
-                uint8_t temp[2048];
-                uint64_t bytesLeft = _header.CompressedSize;
-                do
-                {
-                    auto readLen = std::min(size_t(bytesLeft), sizeof(temp));
-                    _stream->Read(temp, readLen);
-                    _buffer.Write(temp, readLen);
-                    bytesLeft -= readLen;
-                } while (bytesLeft > 0);
-
                 // Uncompress
-                if (_header.Compression == CompressionType::gzip)
+                if (_header.Compression != CompressionType::none)
                 {
-                    auto uncompressedData = Compression::ungzip(_buffer.GetData(), _buffer.GetLength());
+                    size_t compressedSize = static_cast<size_t>(_header.CompressedSize);
-                    if (_header.UncompressedSize != uncompressedData.size())
+                    size_t uncompressedSize = static_cast<size_t>(_header.UncompressedSize);
+                    bool decompressStatus = false;
+
+                    switch (_header.Compression)
                     {
-                        // Warning?
+                        case CompressionType::gzip:
+                            decompressStatus = Compression::zlibDecompress(
+                                *_stream, compressedSize, _buffer, uncompressedSize, Compression::ZlibHeaderType::gzip);
+                            break;
+                        default:
+                            throw IOException("Unknown Park Compression Type");
                     }
-                    _buffer.Clear();
+
-                    _buffer.Write(uncompressedData.data(), uncompressedData.size());
+                    if (!decompressStatus)
+                        throw IOException("Decompression Error");
+                }
+                else
+                {
+                    if (_header.UncompressedSize != _header.CompressedSize)
+                        throw IOException("None Compression Sizes Don't Match");
+                    _buffer.CopyFromStream(*_stream, _header.UncompressedSize);
+                }
+
+                // early in-dev versions used SHA1 instead of FNV1a, so just assume any file
+                // with a verison number of 0 may be one of these, and don't check their hashes.
+                if (_header.TargetVersion > 0)
+                {
+                    auto checksum = Crypt::FNV1a(_buffer.GetData(), _buffer.GetLength());
+                    if (checksum != _header.FNV1a)
+                        throw IOException("Checksum Is Not Valid");
                 }
             }
             else
             {
                 _header = {};
                 _header.Compression = CompressionType::gzip;
-
                 _buffer = MemoryStream{};
             }
         }
@@ -129,26 +140,37 @@ namespace OpenRCT2
         {
             if (_mode == Mode::WRITING)
             {
-                const void* uncompressedData = _buffer.GetData();
-                const uint64_t uncompressedSize = _buffer.GetLength();
-
                 _header.NumChunks = static_cast<uint32_t>(_chunks.size());
-                _header.UncompressedSize = uncompressedSize;
+                _header.UncompressedSize = _buffer.GetLength();
-                _header.CompressedSize = uncompressedSize;
+                _header.CompressedSize = _buffer.GetLength();
-                _header.FNV1a = Crypt::FNV1a(uncompressedData, uncompressedSize);
+                _header.FNV1a = Crypt::FNV1a(_buffer.GetData(), _buffer.GetLength());
 
                 // Compress data
-                std::optional<std::vector<uint8_t>> compressedBytes;
+                if (_header.Compression != CompressionType::none)
-                if (_header.Compression == CompressionType::gzip)
                 {
-                    compressedBytes = Compression::gzip(uncompressedData, uncompressedSize);
+                    MemoryStream compressed;
-                    if (compressedBytes)
+                    size_t bufferLength = static_cast<size_t>(_buffer.GetLength());
+                    bool compressStatus = false;
+
+                    _buffer.SetPosition(0);
+                    switch (_header.Compression)
                     {
-                        _header.CompressedSize = compressedBytes->size();
+                        case CompressionType::gzip:
+                            compressStatus = Compression::zlibCompress(
+                                _buffer, bufferLength, compressed, Compression::ZlibHeaderType::gzip);
+                            break;
+                        default:
+                            break;
+                    }
+
+                    if (compressStatus && compressed.GetLength() < _buffer.GetLength())
+                    {
+                        _buffer = std::move(compressed);
+                        _header.CompressedSize = _buffer.GetLength();
                     }
                     else
                     {
-                        // Compression failed
+                        // Compression increases filesize, so just store uncompressed data
                         _header.Compression = CompressionType::none;
                     }
                 }
@@ -156,19 +178,9 @@ namespace OpenRCT2
                 // Write header and chunk table
                 _stream->WriteValue(_header);
                 for (const auto& chunk : _chunks)
-                {
                     _stream->WriteValue(chunk);
-                }
-
                 // Write chunk data
-                if (compressedBytes)
+                _stream->Write(_buffer.GetData(), _buffer.GetLength());
-                {
-                    _stream->Write(compressedBytes->data(), compressedBytes->size());
-                }
-                else
-                {
-                    _stream->Write(uncompressedData, uncompressedSize);
-                }
             }
         }
 

src/openrct2/core/StreamBuffer.cpp

@@ -0,0 +1,161 @@
+/*****************************************************************************
+ * Copyright (c) 2014-2025 OpenRCT2 developers
+ *
+ * For a complete list of all authors, please refer to contributors.md
+ * Interested in contributing? Visit https://github.com/OpenRCT2/OpenRCT2
+ *
+ * OpenRCT2 is licensed under the GNU General Public License version 3.
+ *****************************************************************************/
+
+#include "StreamBuffer.hpp"
+
+#include "Guard.hpp"
+#include "Memory.hpp"
+
+namespace OpenRCT2
+{
+    StreamReadBuffer::StreamReadBuffer(IStream& stream, uint64_t length, size_t bufferLength)
+    {
+        OpenRCT2::Guard::Assert(bufferLength > 0);
+
+        _remaining = length;
+        if (length <= std::numeric_limits<size_t>::max())
+            _buffer = static_cast<const uint8_t*>(stream.ReadDirect(static_cast<size_t>(length)));
+
+        if (_buffer == nullptr)
+        {
+            if (length > stream.GetLength() - stream.GetPosition())
+                throw IOException("Buffer Will Read Past the End of the Stream");
+
+            _bufferLength = bufferLength;
+            _buffer = OpenRCT2::Memory::Allocate<const uint8_t>(_bufferLength);
+        }
+    }
+
+    StreamReadBuffer::~StreamReadBuffer()
+    {
+        if (_bufferLength > 0)
+            OpenRCT2::Memory::Free(_buffer);
+
+        _buffer = nullptr;
+        _bufferLength = 0;
+        _remaining = 0;
+    }
+
+    StreamReadBuffer::StreamReadBuffer(StreamReadBuffer&& other) noexcept
+    {
+        _buffer = other._buffer;
+        _bufferLength = other._bufferLength;
+        _remaining = other._remaining;
+
+        other._buffer = nullptr;
+        other._bufferLength = 0;
+        other._remaining = 0;
+    }
+
+    StreamReadBuffer& StreamReadBuffer::operator=(StreamReadBuffer&& other) noexcept
+    {
+        if (&other != this)
+        {
+            _buffer = other._buffer;
+            _bufferLength = other._bufferLength;
+            _remaining = other._remaining;
+
+            other._buffer = nullptr;
+            other._bufferLength = 0;
+            other._remaining = 0;
+        }
+        return *this;
+    }
+
+    std::pair<const void*, size_t> StreamReadBuffer::ReadBlock(IStream& stream, size_t maxLength)
+    {
+        const uint8_t* ptr = _buffer;
+        size_t len = static_cast<size_t>(std::min<uint64_t>(_remaining, maxLength));
+        if (_bufferLength == 0)
+        {
+            _buffer += len;
+        }
+        else
+        {
+            len = static_cast<size_t>(std::min(len, _bufferLength));
+            stream.Read(const_cast<uint8_t*>(ptr), len);
+        }
+        _remaining -= len;
+        return { ptr, len };
+    }
+
+    StreamWriteBuffer::StreamWriteBuffer(IStream& stream, uint64_t length, size_t bufferLength)
+    {
+        OpenRCT2::Guard::Assert(bufferLength > 0);
+
+        _remaining = length;
+        if (length <= std::numeric_limits<size_t>::max())
+            _buffer = static_cast<uint8_t*>(stream.WriteDirectStart(static_cast<size_t>(length)));
+
+        if (_buffer == nullptr)
+        {
+            _bufferLength = bufferLength;
+            _buffer = OpenRCT2::Memory::Allocate<uint8_t>(_bufferLength);
+        }
+    }
+
+    StreamWriteBuffer::~StreamWriteBuffer()
+    {
+        if (_bufferLength > 0)
+            OpenRCT2::Memory::Free(_buffer);
+
+        _buffer = nullptr;
+        _bufferLength = 0;
+        _remaining = 0;
+    }
+
+    StreamWriteBuffer::StreamWriteBuffer(StreamWriteBuffer&& other) noexcept
+    {
+        _buffer = other._buffer;
+        _bufferLength = other._bufferLength;
+        _remaining = other._remaining;
+
+        other._buffer = nullptr;
+        other._bufferLength = 0;
+        other._remaining = 0;
+    }
+
+    StreamWriteBuffer& StreamWriteBuffer::operator=(StreamWriteBuffer&& other) noexcept
+    {
+        if (&other != this)
+        {
+            _buffer = other._buffer;
+            _bufferLength = other._bufferLength;
+            _remaining = other._remaining;
+
+            other._buffer = nullptr;
+            other._bufferLength = 0;
+            other._remaining = 0;
+        }
+        return *this;
+    }
+
+    std::pair<void*, size_t> StreamWriteBuffer::WriteBlockStart(size_t maxLength)
+    {
+        size_t len = static_cast<size_t>(std::min<uint64_t>(_remaining, maxLength));
+        if (_bufferLength > 0)
+            len = std::min(len, _bufferLength);
+
+        return { _buffer, len };
+    }
+
+    void StreamWriteBuffer::WriteBlockCommit(IStream& stream, size_t length)
+    {
+        if (_bufferLength == 0)
+        {
+            stream.WriteDirectCommit(length);
+            _buffer += length;
+        }
+        else
+        {
+            stream.Write(_buffer, length);
+        }
+        _remaining -= length;
+    }
+} // namespace OpenRCT2

src/openrct2/core/StreamBuffer.hpp

@@ -0,0 +1,78 @@
+/*****************************************************************************
+ * Copyright (c) 2014-2025 OpenRCT2 developers
+ *
+ * For a complete list of all authors, please refer to contributors.md
+ * Interested in contributing? Visit https://github.com/OpenRCT2/OpenRCT2
+ *
+ * OpenRCT2 is licensed under the GNU General Public License version 3.
+ *****************************************************************************/
+
+#pragma once
+
+#include "IStream.hpp"
+
+#include <cstdint>
+#include <limits>
+#include <utility>
+
+namespace OpenRCT2
+{
+    struct StreamReadBuffer
+    {
+        StreamReadBuffer(IStream& stream, uint64_t length, size_t bufferLength);
+
+        ~StreamReadBuffer();
+        // not copyable
+        StreamReadBuffer(const StreamReadBuffer&) = delete;
+        StreamReadBuffer& operator=(const StreamReadBuffer&) = delete;
+        // moveable
+        StreamReadBuffer(StreamReadBuffer&&) noexcept;
+        StreamReadBuffer& operator=(StreamReadBuffer&&) noexcept;
+
+        std::pair<const void*, size_t> ReadBlock(IStream& stream, size_t maxLength = std::numeric_limits<size_t>::max());
+
+        uint64_t GetRemaining() const
+        {
+            return _remaining;
+        }
+        operator bool() const
+        {
+            return _remaining > 0;
+        }
+
+    private:
+        const uint8_t* _buffer = nullptr;
+        size_t _bufferLength = 0;
+        uint64_t _remaining = 0;
+    };
+
+    struct StreamWriteBuffer
+    {
+        StreamWriteBuffer(IStream& stream, uint64_t length, size_t bufferLength);
+
+        ~StreamWriteBuffer();
+        // not copyable
+        StreamWriteBuffer(const StreamWriteBuffer&) = delete;
+        StreamWriteBuffer& operator=(const StreamWriteBuffer&) = delete;
+        // moveable
+        StreamWriteBuffer(StreamWriteBuffer&&) noexcept;
+        StreamWriteBuffer& operator=(StreamWriteBuffer&&) noexcept;
+
+        std::pair<void*, size_t> WriteBlockStart(size_t maxLength = std::numeric_limits<size_t>::max());
+        void WriteBlockCommit(IStream& stream, size_t length);
+
+        uint64_t GetRemaining() const
+        {
+            return _remaining;
+        }
+        operator bool() const
+        {
+            return _remaining > 0;
+        }
+
+    private:
+        uint8_t* _buffer = nullptr;
+        size_t _bufferLength = 0;
+        uint64_t _remaining = 0;
+    };
+} // namespace OpenRCT2

src/openrct2/core/Zip.cpp

@@ -319,11 +319,6 @@ private:
             return static_cast<uint64_t>(readBytes);
         }
 
-        const void* GetData() const override
-        {
-            return nullptr;
-        }
-
     private:
         void Close()
         {

src/openrct2/core/ZipStream.hpp

@@ -81,6 +81,21 @@ namespace OpenRCT2
         {
             return _base->GetData();
         }
+
+        const void* ReadDirect(size_t length) override
+        {
+            return _base->ReadDirect(length);
+        }
+
+        void* WriteDirectStart(size_t maxLength) override
+        {
+            return _base->WriteDirectStart(maxLength);
+        }
+
+        void WriteDirectCommit(size_t length) override
+        {
+            _base->WriteDirectCommit(length);
+        }
     };
 
 } // namespace OpenRCT2

src/openrct2/libopenrct2.vcxproj

@@ -229,6 +229,7 @@
     <ClInclude Include="core\RTL.h" />
     <ClInclude Include="core\SawyerCoding.h" />
     <ClInclude Include="core\Speed.hpp" />
+    <ClInclude Include="core\StreamBuffer.hpp" />
     <ClInclude Include="core\String.hpp" />
     <ClInclude Include="core\StringBuilder.h" />
     <ClInclude Include="core\StringReader.h" />
@@ -790,6 +791,7 @@
     <ClCompile Include="core\RTL.FriBidi.cpp" />
     <ClCompile Include="core\RTL.ICU.cpp" />
     <ClCompile Include="core\SawyerCoding.cpp" />
+    <ClCompile Include="core\StreamBuffer.cpp" />
     <ClCompile Include="core\String.cpp" />
     <ClCompile Include="core\StringBuilder.cpp" />
     <ClCompile Include="core\StringReader.cpp" />

src/openrct2/network/ServerList.cpp

@@ -198,10 +198,10 @@ std::vector<ServerListEntry> ServerList::ReadFavourites() const
             for (size_t i = 0; i < numEntries; i++)
             {
                 ServerListEntry serverInfo;
-                serverInfo.Address = fs.ReadStdString();
+                serverInfo.Address = fs.ReadString();
-                serverInfo.Name = fs.ReadStdString();
+                serverInfo.Name = fs.ReadString();
                 serverInfo.RequiresPassword = false;
-                serverInfo.Description = fs.ReadStdString();
+                serverInfo.Description = fs.ReadString();
                 serverInfo.Version.clear();
                 serverInfo.Favourite = true;
                 serverInfo.Players = 0;

src/openrct2/object/StringTable.cpp

@@ -60,7 +60,7 @@ void StringTable::Read(IReadObjectContext* context, OpenRCT2::IStream* stream, O
             uint8_t languageId = (EnumValue(rct2LanguageId) <= EnumValue(RCT2LanguageId::Portuguese))
                 ? RCT2ToOpenRCT2LanguageId[EnumValue(rct2LanguageId)]
                 : static_cast<uint8_t>(LANGUAGE_UNDEFINED);
-            std::string stringAsWin1252 = stream->ReadStdString();
+            std::string stringAsWin1252 = stream->ReadString();
             auto stringAsUtf8 = RCT2StringToUTF8(stringAsWin1252, rct2LanguageId);
 
             if (!StringIsBlank(stringAsUtf8.data()))

src/openrct2/platform/Crash.cpp

@@ -33,6 +33,7 @@
     #include "../config/Config.h"
     #include "../core/Compression.h"
     #include "../core/Console.hpp"
+    #include "../core/FileStream.h"
     #include "../core/Guard.hpp"
     #include "../core/Path.hpp"
     #include "../core/SawyerCoding.h"
@@ -134,10 +135,10 @@ static bool OnCrash(
 
     // Compress the dump
     {
-        FILE* input = _wfopen(dumpFilePath, L"rb");
+        FileStream source(dumpFilePath, FileMode::open);
-        FILE* dest = _wfopen(dumpFilePathGZIP, L"wb");
+        FileStream dest(dumpFilePathGZIP, FileMode::write);
 
-        if (Compression::gzipCompress(input, dest))
+        if (Compression::zlibCompress(source, source.GetLength(), dest, Compression::ZlibHeaderType::gzip))
         {
             // TODO: enable upload of gzip-compressed dumps once supported on
             // backtrace.io (uncomment the line below). For now leave compression
@@ -148,8 +149,6 @@ static bool OnCrash(
             _uploadFiles[L"upload_file_minidump"] = dumpFilePathGZIP;
             */
         }
-        fclose(input);
-        fclose(dest);
     }
 
     bool with_record = StopSilentRecord();

src/openrct2/rct12/SawyerChunkReader.cpp

@@ -169,7 +169,7 @@ namespace OpenRCT2
 
                 for (size_t n = 0; n < count; n++)
                 {
-                    buf.Write1(src8 + i);
+                    buf.WriteValue(src8[i]);
                 }
             }
             else
@@ -213,7 +213,7 @@ namespace OpenRCT2
                     throw SawyerChunkException(EXCEPTION_MSG_CORRUPT_RLE);
                 }
                 i++;
-                buf.Write1(src8 + i);
+                buf.WriteValue(src8[i]);
             }
             else
             {
@@ -253,8 +253,7 @@ namespace OpenRCT2
         uint8_t code = 1;
         for (size_t i = 0; i < srcLength; i++)
         {
-            uint8_t temp = Numerics::ror8(src8[i], code);
+            buf.WriteValue(Numerics::ror8(src8[i], code));
-            buf.Write1(&temp);
             code = (code + 2) % 8;
         }
 

src/openrct2/scenario/ScenarioRepository.cpp

@@ -537,8 +537,8 @@ private:
             for (uint32_t i = 0; i < numHighscores; i++)
             {
                 ScenarioHighscoreEntry* highscore = InsertHighscore();
-                highscore->fileName = fs.ReadStdString();
+                highscore->fileName = fs.ReadString();
-                highscore->name = fs.ReadStdString();
+                highscore->name = fs.ReadString();
                 highscore->company_value = fileVersion == 1 ? fs.ReadValue<money32>() : fs.ReadValue<money64>();
                 highscore->timestamp = fs.ReadValue<datetime64>();
             }

test/tests/IniWriterTest.cpp

@@ -47,7 +47,7 @@ TEST_F(IniWriterTest, create_one_section)
     ASSERT_LE(ms.GetPosition(), 13); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(ini.c_str(), "[OpenRCT2]" PLATFORM_NEWLINE);
 }
 
@@ -66,7 +66,7 @@ TEST_F(IniWriterTest, create_multiple_sections)
     ASSERT_LE(ms.GetPosition(), 55); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(
         ini.c_str(),
         "[OpenRCT1]" PLATFORM_NEWLINE PLATFORM_NEWLINE "[OpenRCT2]" PLATFORM_NEWLINE PLATFORM_NEWLINE
@@ -85,7 +85,7 @@ TEST_F(IniWriterTest, create_loose_bool_entry)
     ASSERT_LE(ms.GetPosition(), 17); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(ini.c_str(), "boolval = true" PLATFORM_NEWLINE);
 }
 
@@ -102,7 +102,7 @@ TEST_F(IniWriterTest, create_loose_enum_entry)
     ASSERT_LE(ms.GetPosition(), 37); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(ini.c_str(), "by_string = stringval" PLATFORM_NEWLINE "int32_t = 0" PLATFORM_NEWLINE);
 }
 
@@ -118,7 +118,7 @@ TEST_F(IniWriterTest, create_loose_float_entry)
     ASSERT_LE(ms.GetPosition(), 17); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     // This will be non-fatal due to float.
     EXPECT_STREQ(ini.c_str(), "one = 1.000000" PLATFORM_NEWLINE);
 }
@@ -139,7 +139,7 @@ TEST_F(IniWriterTest, create_loose_int32_t_entry)
     ASSERT_LE(ms.GetPosition(), 78); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(
         ini.c_str(),
         "one = 1" PLATFORM_NEWLINE "zero = 0" PLATFORM_NEWLINE "minusone = -1" PLATFORM_NEWLINE
@@ -158,7 +158,7 @@ TEST_F(IniWriterTest, create_loose_string_entry)
     ASSERT_LE(ms.GetPosition(), 44); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(
         ini.c_str(), "path = \"C:'\\\\some/dir\\\\here/\xE7\xA5\x9E\xE9\xB7\xB9\xE6\x9A\xA2\xE9\x81\x8A\"" PLATFORM_NEWLINE);
 }
@@ -181,7 +181,7 @@ TEST_F(IniWriterTest, create_multiple_section_with_values)
     ASSERT_LE(ms.GetPosition(), 108); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(
         ini.c_str(),
         "[bool]" PLATFORM_NEWLINE "boolval = true" PLATFORM_NEWLINE PLATFORM_NEWLINE "[int]" PLATFORM_NEWLINE
@@ -203,7 +203,7 @@ TEST_F(IniWriterTest, create_duplicate_sections)
     ASSERT_LE(ms.GetPosition(), 43); // Accommodate for varying-sized newline (Windows)
     ASSERT_EQ(ms.GetLength(), ms.GetPosition());
     ms.SetPosition(0);
-    auto ini = ms.ReadStdString();
+    auto ini = ms.ReadString();
     ASSERT_STREQ(
         ini.c_str(),
         "[section]" PLATFORM_NEWLINE PLATFORM_NEWLINE "[section]" PLATFORM_NEWLINE PLATFORM_NEWLINE