Move most StreamBuffer code into base-buffer.hpp

luprex/cpp/core/animqueue.hpp

@@ -93,7 +93,7 @@
 #include "wrap-deque.hpp"
 #include "wrap-unordered-map.hpp"
 
-#include "base-writer.hpp"
+#include "base-buffer.hpp"
 #include "luastack.hpp"
 #include "streambuffer.hpp"
 #include "debugcollector.hpp"

luprex/cpp/core/drivenengine.cpp

@@ -3,6 +3,7 @@
 #include "util.hpp"
 #include "drivenengine.hpp"
 #include "world.hpp"
+#include "base-buffer.hpp"
 
 #include <string_view>
 #include <utility>
@@ -481,7 +482,7 @@ void DrivenEngine::drv_invoke_event_update(double clock) {
 }
 
 void DrivenEngine::drv_set_lua_source(uint32_t srcpklen, const char *srcpk) {
-    StreamBuffer sb(srcpk, srcpklen);
+    StreamBuffer sb(std::string_view(srcpk, srcpklen));
     uint32_t nfiles = sb.read_uint32();
     lua_source_.reset(new util::LuaSourceVec);
     lua_source_->resize(nfiles);

luprex/cpp/core/lpxclient.cpp

@@ -204,10 +204,7 @@ public:
             int64_t nactor = world_->patch_everything(sb, &dbc);
             if (nactor != actor_id_) change_actor_id(nactor);
             dbc.dump(stdostream());
-        } catch (const StreamEof &seof) {
+        } catch (const StreamException &sexcept) {
-            abandon_server();
-            return;
-        } catch (const StreamCorruption &scorr) {
             abandon_server();
             return;
         }
@@ -222,7 +219,8 @@ public:
             sb->unread_to(tr_before);
             return false;
         }
-        StreamBuffer body(sb->read_bytes(message_body_len), message_body_len);
+        const char *message_body = sb->read_bytes(message_body_len);
+        StreamBuffer body(std::string_view(message_body, message_body_len));
         if (message_type == util::MSG_ACK) {
             receive_ack_from_server(&body);
         } else if (message_type == util::MSG_DIFF) {

luprex/cpp/core/lpxserver.cpp

@@ -146,7 +146,7 @@ public:
                 return false;
             }
             inv.deserialize(sb);
-        } catch (const StreamEof &seof) {
+        } catch (const StreamEofOnRead &seof) {
             sb->unread_to(tr_before);
             return false;
         } catch (const StreamCorruption &scorr) {

luprex/cpp/core/serializelua.cpp

@@ -116,7 +116,7 @@ public:
             }
             uint8_t b = sb_->read_uint8();
             deserialize_r(b, val);
-        } catch (const StreamEof &e) {
+        } catch (const StreamException &e) {
             error_ = "EOF reached while deserializing data";
             lua_settop(LS_.state(), top);
             LS_.set(val, LuaNil);

luprex/cpp/core/streambuffer.cpp

@@ -7,320 +7,6 @@
 #include <cassert>
 #include <cstring>
 
-void StreamBuffer::init(bool fixed, bool owned, char *buf, int64_t size) {
-    buf_lo_ = buf;
-    buf_hi_ = buf_lo_ + size;
-    read_cursor_ = buf_lo_;
-    write_cursor_ = buf_lo_;
-    pre_read_count_ = 0;
-    owned_ = owned;
-    fixed_size_ = fixed;
-}
-
-StreamBuffer::StreamBuffer() {
-    init(false, true, 0, 0);
-}
-
-StreamBuffer::StreamBuffer(int64_t size, bool fixed) {
-    assert(size >= 0);
-    init(fixed, true, (char*)eng::malloc(size), size);
-}
-
-StreamBuffer::StreamBuffer(const char *s, int64_t size) {
-    assert(size >= 0);
-    init(true, false, const_cast<char *>(s), size);
-    write_cursor_ = buf_hi_;
-}
-
-StreamBuffer::StreamBuffer(std::string_view data) {
-    init(true, false, const_cast<char *>(data.data()), data.size());
-    write_cursor_ = buf_hi_;
-}
-
-StreamBuffer::~StreamBuffer() {
-    if (owned_ && (buf_lo_ != 0)) eng::free(buf_lo_);
-}
-
-int64_t StreamBuffer::total_reads() const {
-    return (read_cursor_ - buf_lo_) + pre_read_count_;
-}
-
-int64_t StreamBuffer::total_writes() const {
-    return (write_cursor_ - buf_lo_) + pre_read_count_;
-}
-
-int64_t StreamBuffer::fill() const {
-    return write_cursor_ - read_cursor_;
-}
-
-const char *StreamBuffer::data() const {
-    return read_cursor_;
-}
-
-std::string_view StreamBuffer::view() const {
-    return std::string_view(read_cursor_, write_cursor_ - read_cursor_);
-}
-
-bool StreamBuffer::layout_is(int64_t a, int64_t b, int64_t c) {
-    if (read_cursor_ - buf_lo_ != a) return false;
-    if (write_cursor_ - read_cursor_ != b) return false;
-    if (buf_hi_ - write_cursor_ != c) return false;
-    return true;
-}
-
-void StreamBuffer::make_space_slow(int64_t bytes) {
-    assert(owned_ && "We don't own this buffer, can't grow it");
-
-    // Decide whether the current buffer is big enough.
-    int64_t data_size = (write_cursor_ - read_cursor_);
-    int64_t existing_size = (buf_hi_ - buf_lo_);
-    int64_t desired_size = 8192 + ((data_size + bytes) * 2);
-
-    // Update some simple things.
-    pre_read_count_ += (read_cursor_ - buf_lo_);
-
-    // Move the data to the beginning of the buffer, or to
-    // the beginning of a new buffer.
-    if (fixed_size_) {
-        assert((data_size + bytes <= existing_size) && "Not enough space in fixed-size buffer");
-        if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size);
-    } else if (existing_size >= desired_size) {
-        if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size);
-    } else {
-        char *nbuf = (char *)eng::malloc(desired_size);
-        if (data_size > 0) memcpy(nbuf, read_cursor_, data_size);
-        if (buf_lo_ != nullptr) eng::free(buf_lo_);
-        buf_lo_ = nbuf;
-        buf_hi_ = nbuf + desired_size;
-    }
-
-    // Update the pointers to the data region.
-    read_cursor_ = buf_lo_;
-    write_cursor_ = buf_lo_ + data_size;
-}
-
-void StreamBuffer::wrote_space(int64_t bytes) {
-    int64_t available = buf_hi_ - write_cursor_;
-    assert(bytes >= 0);
-    assert(available >= bytes);
-    write_cursor_ += bytes;
-}
-
-char *StreamBuffer::get_overwrite(int64_t size, int64_t write_count_after) {
-    int64_t write_count_before = write_count_after - size;
-    assert(write_count_before >= total_reads());
-    assert(write_count_after <= total_writes());
-    return buf_lo_ + (write_count_before - pre_read_count_);
-}
-
-void StreamBuffer::clear() {
-    assert(owned_);
-    if (!fixed_size_) {
-        if (buf_lo_ != nullptr) eng::free(buf_lo_);
-        buf_lo_ = 0;
-        buf_hi_ = 0;
-    }
-    owned_ = true;
-    read_cursor_ = buf_lo_;
-    write_cursor_ = buf_lo_;
-    pre_read_count_ = 0;
-}
-
-eng::string StreamBuffer::readline() {
-    char *p = read_cursor_;
-    while ((p < write_cursor_) && (*p != '\n')) p++;
-    if (p == write_cursor_) {
-        return "";
-    } else {
-        p++;
-        eng::string result(read_cursor_, p - read_cursor_);
-        read_cursor_ = p;
-        return result;
-    }
-}
-
-void StreamBuffer::write_bytes(const char *s, int64_t len) {
-    make_space(len);
-    memcpy(write_cursor_, s, len);
-    write_cursor_ += len;
-}
-
-void StreamBuffer::write_bytes(std::string_view s) {
-    write_bytes(s.data(), s.size());
-}
-
-
-const char *StreamBuffer::read_bytes(int64_t bytes) {
-    check_available(bytes);
-    char *data = read_cursor_;
-    read_cursor_ += bytes;
-    return data;
-}
-
-void StreamBuffer::read_bytes_into(char *data, int64_t size) {
-    check_available(size);
-    memcpy(data, read_cursor_, size);
-    read_cursor_ += size;
-}
-
-std::string_view StreamBuffer::read_string_view_limit(uint64_t limit) {
-    uint64_t length = read_length();
-    if (length > limit) throw StreamCorruption();
-    check_available(length);
-    std::string_view result(read_cursor_, length);
-    read_cursor_ += length;
-    return result;
-}
-
-void StreamBuffer::write_xyz(const util::XYZ &xyz) {
-    make_space(12);
-    memcpy(write_cursor_, &xyz.x, 4);
-    write_cursor_ += 4;
-    memcpy(write_cursor_, &xyz.y, 4);
-    write_cursor_ += 4;
-    memcpy(write_cursor_, &xyz.z, 4);
-    write_cursor_ += 4;
-}
-
-void StreamBuffer::write_dxyz(const util::DXYZ &xyz) {
-    make_space(24);
-    memcpy(write_cursor_, &xyz.x, 8);
-    write_cursor_ += 8;
-    memcpy(write_cursor_, &xyz.y, 8);
-    write_cursor_ += 8;
-    memcpy(write_cursor_, &xyz.z, 8);
-    write_cursor_ += 8;
-}
-
-util::XYZ StreamBuffer::read_xyz() {
-    check_available(12);
-    util::XYZ result;
-    memcpy(&result.x, read_cursor_, 4);
-    read_cursor_ += 4;
-    memcpy(&result.y, read_cursor_, 4);
-    read_cursor_ += 4;
-    memcpy(&result.z, read_cursor_, 4);
-    read_cursor_ += 4;
-    return result;
-}
-
-util::DXYZ StreamBuffer::read_dxyz() {
-    check_available(24);
-    util::DXYZ result;
-    memcpy(&result.x, read_cursor_, 8);
-    read_cursor_ += 8;
-    memcpy(&result.y, read_cursor_, 8);
-    read_cursor_ += 8;
-    memcpy(&result.z, read_cursor_, 8);
-    read_cursor_ += 8;
-    return result;
-}
-
-void StreamBuffer::write_hashvalue(const util::HashValue &hv) {
-    write_uint64(hv.first);
-    write_uint64(hv.second);
-}
-
-util::HashValue StreamBuffer::read_hashvalue() {
-    uint64_t f = read_uint64();
-    uint64_t s = read_uint64();
-    return util::HashValue(f,s);
-}
-
-void StreamBuffer::overwrite_int8(int64_t write_count_after, int64_t vv) {
-    int8_t v = vv;
-    assert(int64_t(v) == vv);
-    char *target = get_overwrite(1, write_count_after);
-    memcpy(target, &v, 1);
-}
-
-void StreamBuffer::overwrite_int16(int64_t write_count_after, int64_t vv) {
-    int16_t v = vv;
-    assert(int64_t(v) == vv);
-    char *target = get_overwrite(2, write_count_after);
-    memcpy(target, &v, 2);
-}
-
-void StreamBuffer::overwrite_int32(int64_t write_count_after, int64_t vv) {
-    int32_t v = vv;
-    assert(int64_t(v) == vv);
-    char *target = get_overwrite(4, write_count_after);
-    memcpy(target, &v, 4);
-}
-
-void StreamBuffer::overwrite_int64(int64_t write_count_after, int64_t vv) {
-    int64_t v = vv;
-    assert(int64_t(v) == vv);
-    char *target = get_overwrite(8, write_count_after);
-    memcpy(target, &v, 8);
-}
-
-void StreamBuffer::overwrite_uint8(int64_t write_count_after, uint64_t vv) {
-    uint8_t v = vv;
-    assert(uint64_t(v) == vv);
-    char *target = get_overwrite(1, write_count_after);
-    memcpy(target, &v, 1);
-}
-
-void StreamBuffer::overwrite_uint16(int64_t write_count_after, uint64_t vv) {
-    uint16_t v = vv;
-    assert(uint64_t(v) == vv);
-    char *target = get_overwrite(2, write_count_after);
-    memcpy(target, &v, 2);
-}
-
-void StreamBuffer::overwrite_uint32(int64_t write_count_after, uint64_t vv) {
-    uint32_t v = vv;
-    assert(uint64_t(v) == vv);
-    char *target = get_overwrite(4, write_count_after);
-    memcpy(target, &v, 4);
-}
-
-void StreamBuffer::overwrite_uint64(int64_t write_count_after, uint64_t vv) {
-    uint64_t v = vv;
-    assert(uint64_t(v) == vv);
-    char *target = get_overwrite(8, write_count_after);
-    memcpy(target, &v, 8);
-}
-
-bool StreamBuffer::empty() {
-    return (read_cursor_ == write_cursor_);
-}
-
-void StreamBuffer::verify_empty() {
-    if (read_cursor_ != write_cursor_) {
-        throw StreamCorruption();
-    }
-}
-
-void StreamBuffer::unread_to(int64_t rd_count) {
-    assert(rd_count >= pre_read_count_);
-    assert(rd_count <= total_reads());
-    read_cursor_ = buf_lo_ + (rd_count - pre_read_count_);
-}
-
-void StreamBuffer::unwrite_to(int64_t wr_count) {
-    assert(wr_count >= total_reads());
-    assert(wr_count <= total_writes());
-    write_cursor_ = buf_lo_ + (wr_count - pre_read_count_);
-}
-
-void StreamBuffer::copy_into(StreamBuffer *sb) {
-    sb->write_bytes(read_cursor_, write_cursor_ - read_cursor_);
-}
-
-void StreamBuffer::transfer_into(StreamBuffer *sb) {
-    sb->write_bytes(read_cursor_, write_cursor_ - read_cursor_);
-    read_cursor_ = write_cursor_;
-}
-
-bool StreamBuffer::contents_equal(const StreamBuffer *other) const {
-    int64_t len = fill();
-    if (len != other->fill()) {
-        return false;
-    }
-    return memcmp(read_cursor_, other->read_cursor_, len) == 0;
-}
 
 int lua_writer_into_streambuffer(lua_State *L, const void* bytes, size_t sz, void* sbv) {
     StreamBuffer *sb = (StreamBuffer *)sbv;
@@ -334,10 +20,6 @@ static bool streq(const char *str, const char *data) {
     return memcmp(str, data, len) == 0;
 }
 
-static void write_ztbytes(StreamBuffer *sb, const char *bytes) {
-    sb->write_bytes(bytes, strlen(bytes));
-}
-
 LuaDefine(unittests_streambuffer, "", "some unit tests") {
     // An 11-byte fixed-size stream buffer.
     StreamBuffer sb11(11, true);
@@ -346,7 +28,7 @@ LuaDefine(unittests_streambuffer, "", "some unit tests") {
     LuaAssert(L, sb11.layout_is(0, 0, 11));
 
     // Write a few bytes.
-    write_ztbytes(&sb11, "abcdef");
+    sb11.write_bytes("abcdef");
     LuaAssert(L, sb11.layout_is(0, 6, 5));
 
     // Try reading some bytes.
@@ -365,17 +47,17 @@ LuaDefine(unittests_streambuffer, "", "some unit tests") {
     try {
         sb11.read_bytes(1);
         LuaAssert(L, false && "This should have thrown an exception");
-    } catch (const StreamEof &) {}
+    } catch (const StreamEofOnRead &) {}
     LuaAssert(L, sb11.layout_is(6, 0, 5));
 
     // Write some more bytes into the stream, forcing a shift-left
-    write_ztbytes(&sb11, "ghijkl");
+    sb11.write_bytes("ghijkl");
     LuaAssert(L, sb11.layout_is(0, 6, 5));
 
     // Test buffer wrapping a little more.
     LuaAssert(L, streq("ghi", sb11.read_bytes(3)));
     LuaAssert(L, sb11.layout_is(3, 3, 5));
-    write_ztbytes(&sb11, "mnopqr");
+    sb11.write_bytes("mnopqr");
     LuaAssert(L, sb11.layout_is(0, 9, 2));
     LuaAssert(L, streq("jklmnopqr", sb11.read_bytes(9)));
     LuaAssert(L, sb11.layout_is(9, 0, 2));

luprex/cpp/core/streambuffer.hpp

@@ -184,27 +184,17 @@
 //
 // You can use a streambuffer as a lua_Writer, as follows:
 //
-//       lua_dump(L, stream.lua_writer(), stream.lua_writer_ud());
+//       lua_dump(L, lua_writer_into_streambuffer, &sb);
 //
 // Anything written to the lua_writer gets appended to the streambuffer, the
 // same as if it had been written using write_bytes.
 //
-// You can use a streambuffer as a lua_Reader, as follows:
+// You can't use streambuffer as a lua_Reader directly, but you can get a
+// string_view out of it and then use that to construct a lua_Reader, as
+// follows:
 //
-//      lua_load (L, stream.lua_reader(), stream.lua_reader_ud(nbytes), ...)
+//      LuaStringViewReader svr(mystreambuffer.view());
-//
+//      lua_load (L, svr.lua_reader(), svr.lua_reader_userdata());
-// The exact semantics of the lua_reader are tricky, so be careful:
-// lua_reader_ud calls 'read_bytes' immediately, and it stores the bytes in a
-// "cache of bytes for lua."  Then, when the lua_reader gets invoked, the reader
-// returns the entire contents of the cache, and it clears the cache. Here are
-// some consequences of this design:
-//
-//    1. The number of bytes read from the stream is always exactly equal to
-//       nbytes, even if lua never calls the lua_reader.
-//
-//    2. If the stream doesn't contain nbytes, a StreamEof exception gets thrown
-//       from lua_reader_ud, not from the lua_Reader.  This is good, because it
-//       means exceptions don't get thrown from inside the lua runtime.
 //
 //////////////////////////////////////////////////////////////
 
@@ -219,7 +209,7 @@
 #include <cstdint>
 #include <cassert>
 
-#include "base-writer.hpp"
+#include "base-buffer.hpp"
 #include "luastack.hpp"
 #include "util.hpp"
 
@@ -229,188 +219,88 @@ public:
     virtual char const *what() const { return "General stream exception"; }
 };
 
-class StreamEof : public StreamException
+class StreamEofOnRead : public StreamException
 {
 public:
     virtual char const *what() const { return "Stream ran out of data"; }
 };
 
-class StreamCorruption : public StreamException
+class StreamStringTooLong: public StreamException
 {
 public:
-    virtual char const *what() const { return "Stream contained invalid data"; }
+    virtual char const *what() const { return "Stream contained a string that was too long"; }
 };
 
-class StreamBuffer : public eng::nevernew, public BaseReader<StreamBuffer>, public BaseWriter<StreamBuffer> {
+class StreamIntegerTruncated: public StreamException
+{
 public:
-    using read_string_type = eng::string;
+    virtual char const *what() const { return "You truncated an integer when writing to a stream"; }
+};
 
-    // Construct an empty buffer.
+class StreamCorruption: public StreamException
-    StreamBuffer();
+{
+public:
+    virtual char const *what() const { return "Stream Corruption"; }
+};
 
-    // Construct an empty buffer, preallocate the specified amount of space.
+class StreamBufferCore {
-    StreamBuffer(int64_t size, bool fixed_size);
+protected:
+    void *basebuffer_malloc(size_t size) { return eng::malloc(size); }
+    void basebuffer_free(void *p) { eng::free(p); }
+    void clear_error_flags() { }
+    void raise_eof_on_read() { throw StreamEofOnRead(); }
+    void raise_string_too_long() { throw StreamStringTooLong(); }
+    void raise_integer_truncated() { throw StreamIntegerTruncated(); }
+};
 
-    // Construct a streambuffer that reads from an external block of bytes.
+class StreamBuffer : public eng::nevernew, public BaseBuffer<StreamBufferCore, eng::string> {
-    StreamBuffer(const char *s, int64_t len);
+public:
+    using BaseBuffer::BaseBuffer;
 
-    // Construct a streambuffer that reads from an external block of bytes.
+    void write_xyz(const util::XYZ &xyz) {
-    StreamBuffer(std::string_view data);
+        write_float(xyz.x);
-    
+        write_float(xyz.y);
-    // Delete a StreamBuffer.
+        write_float(xyz.z);
-    ~StreamBuffer();
-    
-    // Get the total number of bytes ever read from this buffer.
-    int64_t total_reads() const;
-    
-    // Get the total number of bytes ever written to this buffer.
-    int64_t total_writes() const;
-
-    // Make the specified amount of space in the buffer for writing.
-    // Return a pointer to the space.
-    char *make_space(int64_t bytes) {
-        int64_t available = buf_hi_ - write_cursor_;
-        if (available < bytes) make_space_slow(bytes);
-        return write_cursor_;
     }
 
-    // Used after calling make_space then filling the space.
+    void write_dxyz(const util::DXYZ &xyz) {
-    void wrote_space(int64_t bytes);
+        write_double(xyz.x);
-
+        write_double(xyz.y);
-    // Amount of data inside the buffer.
+        write_double(xyz.z);
-    int64_t fill() const;
-    
-    // Get a pointer to the data.
-    const char *data() const;
-    
-    // Get entire contents as a string_view
-    std::string_view view() const;
-
-    // Discard all data.  Reset total read and write counts.
-    // Frees up as much space as possible.
-    void clear();
-
-    // Attempt to do a "readline".  If there is no newline in
-    // the buffer, returns empty string.  If there is a newline,
-    // returns a block of text that ends in newline.
-    eng::string readline();
-
-    // Write block of bytes into the buffer.
-    //
-    // Caution: this function doesn't write the length!  
-    // It just writes the bytes.
-    //
-    void write_bytes(const char *bytes, int64_t len);
-    void write_bytes(std::string_view s);
-    
-    // Read a block of bytes from the buffer.
-    //
-    // Caution: the pointer returned is a pointer to the stream's buffer. It is
-    // only valid until you mutate the buffer. Throws StreamEof if the specified
-    // number of bytes aren't present.
-    //
-    const char *read_bytes(int64_t bytes);
-
-    // Copy bytes from the StreamBuffer into an external buffer.
-    //
-    // Returns true if the bytes were successfully read.
-    //
-    void read_bytes_into(char *target, int64_t len);
-    
-    // Read a string as a string_view.
-    //
-    std::string_view read_string_view_limit(uint64_t limit);
-    std::string_view read_string_view() { return read_string_view_limit(0x1000000); }
-
-    // Read and write larger types.
-    //
-    // Throws StreamEof if the specified number of bytes aren't present.
-    // Read string with a length limit will throw 'StreamCorruption' if the
-    // length is too long.
-    //
-    void write_xyz(const util::XYZ &xyz);
-    void write_dxyz(const util::DXYZ &xyz);
-    util::XYZ read_xyz();
-    util::DXYZ read_dxyz();
-    
-    void write_hashvalue(const util::HashValue &hv);
-    util::HashValue read_hashvalue();
-
-    // Overwrite values previously written to the buffer.
-    //
-    // See the comment at the top of this file for an explanation.
-    //
-    void overwrite_int8(int64_t write_count_after, int64_t v);
-    void overwrite_int16(int64_t write_count_after, int64_t v);
-    void overwrite_int32(int64_t write_count_after, int64_t v);
-    void overwrite_int64(int64_t write_count_after, int64_t v);
-    void overwrite_uint8(int64_t write_count_after, uint64_t v);
-    void overwrite_uint16(int64_t write_count_after, uint64_t v);
-    void overwrite_uint32(int64_t write_count_after, uint64_t v);
-    void overwrite_uint64(int64_t write_count_after, uint64_t v);
-
-    // This function checks to see if the buffer is empty.
-    bool empty();
-
-    // Verify that the buffer is empty, if not, throw StreamCorruption.
-    void verify_empty();
-
-    // Make sure the specified number of bytes are available to read.
-    void check_available(int64_t bytes) {
-        int64_t avail = write_cursor_ - read_cursor_;
-        if (avail < bytes) {
-            throw StreamEof();
-        }
     }
 
-    // Rewind the read cursor to a previous position.
+    void write_hashvalue(const util::HashValue &h) {
-    void unread_to(int64_t total_reads);
+        write_uint64(h.first);
-    
+        write_uint64(h.second);
-    // Rewind the write cursor to a previous position.
+    }
-    void unwrite_to(int64_t total_writes);
-    
-    // Copy the entire contents of this streambuffer into another one.
-    void copy_into(StreamBuffer *sb);
 
-    // Transfer the entire contents of this streambuffer into another one.
+    util::XYZ read_xyz() {
-    void transfer_into(StreamBuffer *sb);
+        float x = read_float();
+        float y = read_float();
+        float z = read_float();
+        return util::XYZ(x, y, z);
+    }
 
-    // Compare the contents of this streambuffer to another one.
+    util::DXYZ read_dxyz() {
-    bool contents_equal(const StreamBuffer *sb) const;
+        double x = read_double();
+        double y = read_double();
+        double z = read_double();
+        return util::DXYZ(x, y, z);
+    }
 
-    // Throw a StreamCorruption exception.
+    util::HashValue read_hashvalue() {
-    void raise_truncated() { throw StreamCorruption(); }
+        uint64_t f = read_uint64();
-    void raise_string_too_long() { throw StreamCorruption(); }
+        uint64_t s = read_uint64();
-    
+        return util::HashValue(f, s);
-    // This is for unit testing.
+    }
-    bool layout_is(int64_t a, int64_t b, int64_t c);
 
-private:
+    bool contents_equal(const StreamBuffer *sb) {
-    // Start and end of the allocated block.
+        return view() == sb->view();
-    char *buf_lo_;
+    }
-    char *buf_hi_;
 
-    // The write and read cursors.
+    void copy_into(StreamBuffer *sb) {
-    char *write_cursor_;
+        sb->write_bytes(view());
-    char *read_cursor_;
+    }
-
-    // Number of bytes read before buffer was last aligned.
-    int64_t pre_read_count_;
-
-    // True if we own this buffer.
-    bool owned_;
-    
-    // True if we're not allowed to expand this buffer.
-    bool fixed_size_;
-
-    // Initialize with a new buffer.
-    void init(bool fixed, bool owned, char *buf, int64_t size);
-
-    // Function that resizes the buffer during make_space operations.
-    void make_space_slow(int64_t bytes);
-    
-    // Implementation for the overwrite_int functions.
-    char *get_overwrite(int64_t size, int64_t write_count_after);
 };
 
 // Use a streambuffer as a lua_writer.

luprex/ext/base-buffer.hpp

@@ -0,0 +1,744 @@
+#pragma once
+
+/////////////////////////////////////////////////////////////////
+//
+// IMPORTANT: This is a header-only library that is included
+// by the graphics engine as well.  It cannot contain references
+// to anything else in the engine.
+//
+/////////////////////////////////////////////////////////////////
+
+#include <cstdio>
+#include <cstdint>
+#include <cstdlib>
+#include <cassert>
+#include <cstring>
+#include <string_view>
+
+///////////////////////////////////////////////////////////////
+//
+// SimpleDynamic
+//
+// A struct that holds a dynamically typed value.
+// This can hold a string, number, vector, or boolean.
+//
+// The type is stored in the 'type' field.
+//
+// If it's a STRING, the value is in the field s
+// If it's a NUMBER, the value is in the field x
+// If it's a BOOLEAN, it's true if (x==1.0)
+// If it's a VECTOR, the value is in x,y,z
+//
+///////////////////////////////////////////////////////////////
+
+enum class SimpleDynamicTag {
+    UNINITIALIZED,
+    STRING,
+    NUMBER,
+    BOOLEAN,
+    VECTOR,
+};
+
+template<class STRING>
+struct SimpleDynamic {
+    using string = STRING;
+    SimpleDynamicTag type;
+    double x, y, z;
+    string s;
+    
+    SimpleDynamic() {
+        type = SimpleDynamicTag::UNINITIALIZED;
+        x=y=z=0;
+    }
+
+    static const char *type_name_of(SimpleDynamicTag t) {
+        switch (t) {
+            case SimpleDynamicTag::UNINITIALIZED: return "uninitialized";
+            case SimpleDynamicTag::BOOLEAN: return "boolean";
+            case SimpleDynamicTag::NUMBER: return "number";
+            case SimpleDynamicTag::STRING: return "string";
+            case SimpleDynamicTag::VECTOR: return "vector";
+            default: return "unknown";
+        }
+    }
+
+    const char *type_name() const {
+        return type_name_of(type);
+    }
+
+    void set_uninitialized() {
+        type=SimpleDynamicTag::UNINITIALIZED; s.clear(); x=y=z=0;
+    }
+
+    void set_string(std::string_view is) {
+        type=SimpleDynamicTag::STRING; s=is; x=y=z=0;
+    }
+
+    void set_number(double n) {
+        type = SimpleDynamicTag::NUMBER; s.clear(); x=n; y=z=0;
+    }
+
+    void set_boolean(bool b) {
+        type = SimpleDynamicTag::BOOLEAN; s.clear(); x=(b?1:0); y=z=0;
+    }
+
+    void set_vector(double ix, double iy, double iz) {
+        type = SimpleDynamicTag::VECTOR; s.clear(); x=ix; y=iy; z=iz;
+    }
+
+    void copy_value(const SimpleDynamic &other) {
+        type = other.type;
+        s=other.s; x=other.x; y=other.y; z=other.z;
+    }
+};
+
+///////////////////////////////////////////////////////////////
+//
+// BaseWriter
+//
+// This base class provides the following methods:
+//
+//     void write_uint8(uint64_t data)
+//     void write_uint16(uint64_t data)
+//     void write_uint32(uint64_t data)
+//     void write_uint64(uint64_t data)
+//     void write_int8(int64_t data)
+//     void write_int16(int64_t data)
+//     void write_int32(int64_t data)
+//     void write_int64(int64_t data)
+//     void write_char(char c)
+//     void write_float(float data)
+//     void write_double(double data)
+//     void write_length(size_t data)
+//     void write_string(std::string_view data)
+//     void write_simple_dynamic(const SimpleDynamic &sd);
+//
+// You should derive from BaseWriter using the CRTP pattern:
+//
+//     class DerivedWriter : public BaseWriter<DerivedWriter>
+//
+// You must provide two methods in the derived class:
+//
+//     write_bytes(const char *n, size_t size)
+//     raise_truncated()
+//
+///////////////////////////////////////////////////////////////
+
+template<class Derived>
+class BaseWriter {
+protected:
+    template<class T>
+    void write_value_core(T arg) {
+        static_cast<Derived*>(this)->write_bytes((const char *)&arg, sizeof(arg));
+    }
+
+    template<class T, class XT>
+    void write_int_core(XT arg) {
+        T reduced = arg;
+        if (XT(reduced) != arg) static_cast<Derived*>(this)->raise_truncated();
+        write_value_core(reduced);
+    }
+
+public:
+
+    void write_uint8(uint64_t data) { write_int_core<uint8_t, uint64_t>(data); }
+    void write_uint16(uint64_t data) { write_int_core<uint16_t, uint64_t>(data); }
+    void write_uint32(uint64_t data) { write_int_core<uint32_t, uint64_t>(data); }
+    void write_uint64(uint64_t data) { write_int_core<uint64_t, uint64_t>(data); }
+    void write_int8(int64_t data) { write_int_core<int8_t, int64_t>(data); }
+    void write_int16(int64_t data) { write_int_core<int16_t, int64_t>(data); }
+    void write_int32(int64_t data) { write_int_core<int32_t, int64_t>(data); }
+    void write_int64(int64_t data) { write_int_core<int64_t, int64_t>(data); }
+    
+    void write_bool(bool b) { write_uint8(b ? 1:0); }
+    void write_char(char c) { write_value_core(c); }
+    void write_float(float arg) { write_value_core(arg); }
+    void write_double(double arg) { write_value_core(arg); }
+
+    void write_length(size_t len) {
+        if (len >= 255) {
+            write_uint8(0xFF);
+            write_uint64(len);
+        } else {
+            write_uint8(len);
+        }
+    }
+
+    void write_string(std::string_view s) {
+        write_length(s.size());
+        static_cast<Derived*>(this)->write_bytes(s.data(), s.size());
+    }
+};
+
+
+///////////////////////////////////////////////////////////////
+//
+// BaseReader
+//
+// This base class provides the following methods:
+//
+//     uint8_t read_uint8();
+//     uint16_t read_uint16();
+//     uint32_t read_uint32();
+//     uint64_t read_uint64();
+//     int8_t read_int8();
+//     int16_t read_int16();
+//     int32_t read_int32();
+//     int64_t read_int64();
+//     bool read_bool();
+//     char read_char();
+//     float read_float();
+//     double read_double();
+//     size_t read_length();
+//     String read_string_limit(uint64_t size);
+//     String read_string();
+//     SimpleDynamic read_simple_dynamic();
+//
+// You should derive from BaseReader using the CRTP pattern:
+//
+//     class DerivedReader : public BaseReader<DerivedReader>
+//
+// The derived class must provide:
+//
+//     using read_string_type = std::string; // or compatible
+//     void read_bytes_into(char *n, size_t size)
+//     void handle_string_too_long();
+//
+// Error Handling:
+//
+//     It is up to the derived class whether it wants
+//     to report errors using exceptions or flags.
+//
+//     If read_bytes_into discovers there's not enough bytes,
+//     there are two valid options: throw an exception, OR,
+//     set an error flag and fill the buffer with zeros.
+//
+//     If read_string discovers that the string is longer than
+//     the allowed limit, it will call handle_string_too_long.
+//     This function may either throw an exception, or set an
+//     error flag.
+//
+///////////////////////////////////////////////////////////////
+
+template<class Derived>
+class BaseReader {
+protected:
+    template<class T>
+    T read_value_core() {
+        T result;
+        Derived *dthis = static_cast<Derived*>(this);
+        dthis->read_bytes_into((char *)(&result), sizeof(result));
+        return result;
+    }
+
+public:
+
+    uint8_t read_uint8() { return read_value_core<uint8_t>(); }
+    uint16_t read_uint16() { return read_value_core<uint16_t>(); }
+    uint32_t read_uint32() { return read_value_core<uint32_t>(); }
+    uint64_t read_uint64() { return read_value_core<uint64_t>(); }
+    int8_t read_int8() { return read_value_core<int8_t>(); }
+    int16_t read_int16() { return read_value_core<int16_t>(); }
+    int32_t read_int32() { return read_value_core<int32_t>(); }
+    int64_t read_int64() { return read_value_core<int64_t>(); }
+    
+    bool read_bool() { return (bool)read_uint8(); }
+    char read_char() { return read_value_core<char>(); }
+    float read_float() { return read_value_core<float>(); }
+    double read_double() { return read_value_core<double>(); }
+    
+    size_t read_length() {
+        uint64_t len = read_uint8();
+        if (len == 255) {
+            len = read_uint64();
+        }
+        return len;
+    }
+
+    auto read_string_limit(uint64_t limit) {
+        size_t len = read_length();
+        Derived *dthis = static_cast<Derived*>(this);
+        if (len > limit) {
+            dthis->raise_string_too_long();
+            len = 0;
+        }
+        typename Derived::read_string_type result(len, ' ');
+        dthis->read_bytes_into(&(result[0]), len);
+        return result;
+    }
+
+    auto read_string() { return read_string_limit(0x1000000); } // 16MB limit default
+};
+
+///////////////////////////////////////////////////////////////
+//
+// Class BaseBuffer
+//
+// You must supply a CoreHandler which must define these
+// methods:
+//
+//      void *basebuffer_malloc(size_t size);
+//      void basebuffer_free(void *data);
+//      void raise_eof_on_read();
+//      void raise_string_too_long();
+//      void raise_integer_truncated();
+//
+// You must also select a StringType.  Typically this would
+// be std::string.  This only affects the return value of
+// read_string.  You can always use read_string_view to read
+// strings into other string types.
+//
+///////////////////////////////////////////////////////////////
+
+template<class CoreHandler, class StringType>
+class BaseBuffer : public CoreHandler {
+private:
+    // True if we own this buffer.
+    bool owned_;
+    
+    // True if we're not allowed to expand this buffer.
+    bool fixed_size_;
+
+    // Start and end of the allocated block.
+    char *buf_lo_;
+    char *buf_hi_;
+
+    // The write and read cursors.
+    char *write_cursor_;
+    char *read_cursor_;
+
+    // Number of bytes read before buffer was last aligned.
+    int64_t pre_read_count_;
+
+private:
+    void init(bool fixed, bool owned, char *buf, int64_t size) {
+        CoreHandler::clear_error_flags();
+        owned_ = owned;
+        fixed_size_ = fixed;
+        buf_lo_ = buf;
+        buf_hi_ = buf_lo_ + size;
+        read_cursor_ = buf_lo_;
+        write_cursor_ = buf_lo_;
+        pre_read_count_ = 0;
+    }
+
+public:
+    using string_type = StringType;
+
+    // Construct an empty buffer.
+    //
+    BaseBuffer() {
+        init(false, true, 0, 0);
+    }
+
+    // Construct an empty buffer, preallocate the specified amount of space.
+    //
+    BaseBuffer(int64_t size, bool fixed) {
+        assert(size >= 0);
+        init(fixed, true, (char *)CoreHandler::basebuffer_malloc(size), size);
+    }
+
+    // Construct a streambuffer that reads from an external block of bytes.
+    //
+    BaseBuffer(std::string_view data) {
+        init(true, false, const_cast<char *>(data.data()), data.size());
+        write_cursor_ = buf_hi_;
+    }
+    
+    // Destructor.  Frees the buffer, if any.
+    //
+    ~BaseBuffer() {
+        if (owned_ && (buf_lo_ != 0)) CoreHandler::basebuffer_free(buf_lo_);
+    }
+
+    // Return the total number of bytes ever read.
+    //
+    int64_t total_reads() const {
+        return (read_cursor_ - buf_lo_) + pre_read_count_;
+    }
+
+    // Return the total number of bytes ever written.
+    //
+    int64_t total_writes() const {
+        return (write_cursor_ - buf_lo_) + pre_read_count_;
+    }
+
+    // Return the total bytes in the buffer.
+    //
+    int64_t fill() const {
+        return write_cursor_ - read_cursor_;
+    }
+
+    // Checks to see if the buffer is empty.
+    //
+    bool empty() const {
+        return write_cursor_ == read_cursor_;
+    }
+
+    // Return the contents as a string_view.
+    //
+    std::string_view view() const {
+        return std::string_view(read_cursor_, write_cursor_ - read_cursor_);
+    }
+
+    // Make the specified amount of space in the buffer for writing.
+    //
+    char *make_space(int64_t bytes) {
+        int64_t available = buf_hi_ - write_cursor_;
+        if (available < bytes) make_space_slow(bytes);
+        return write_cursor_;
+    }
+
+    // Used after calling make_space then filling the space.
+    //
+    void wrote_space(int64_t bytes) {
+        int64_t available = buf_hi_ - write_cursor_;
+        assert(bytes >= 0);
+        assert(available >= bytes);
+        write_cursor_ += bytes;
+    }
+
+    // Rewind the read cursor to a previous position.
+    //
+    void unread_to(int64_t rd_count) {
+        assert(rd_count >= pre_read_count_);
+        assert(rd_count <= total_reads());
+        read_cursor_ = buf_lo_ + (rd_count - pre_read_count_);
+    }
+
+    // Rewind the write cursor to a previous position.
+    //
+    void unwrite_to(int64_t wr_count) {
+        assert(wr_count >= total_reads());
+        assert(wr_count <= total_writes());
+        write_cursor_ = buf_lo_ + (wr_count - pre_read_count_);
+    }
+
+    // Discard all data.  Reset total read and write counts.
+    // Releases the allocated buffer, if any.
+    //
+    void clear() {
+        assert(owned_);
+        if (!fixed_size_) {
+            if (buf_lo_ != nullptr) CoreHandler::basebuffer_free(buf_lo_);
+            buf_lo_ = 0;
+            buf_hi_ = 0;
+        }
+        owned_ = true;
+        read_cursor_ = buf_lo_;
+        write_cursor_ = buf_lo_;
+        pre_read_count_ = 0;
+    }
+
+    // Write block of bytes into the buffer.
+    //
+    void write_bytes(std::string_view s) {
+        int64_t len = s.size();
+        make_space(len);
+        memcpy(write_cursor_, s.data(), len);
+        write_cursor_ += len;
+    }
+    
+    // Write integers.
+    //
+    void write_uint8(uint64_t data) { write_uint_core<uint8_t>(data); }
+    void write_uint16(uint64_t data) { write_uint_core<uint16_t>(data); }
+    void write_uint32(uint64_t data) { write_uint_core<uint32_t>(data); }
+    void write_uint64(uint64_t data) { write_uint_core<uint64_t>(data); }
+    void write_int8(int64_t data) { write_int_core<int8_t>(data); }
+    void write_int16(int64_t data) { write_int_core<int16_t>(data); }
+    void write_int32(int64_t data) { write_int_core<int32_t>(data); }
+    void write_int64(int64_t data) { write_int_core<int64_t>(data); }
+
+    // Write other primitive types.
+    //
+    void write_bool(bool b) { write_uint8(b ? 1:0); }
+    void write_char(char c) { write_value_core(c); }
+    void write_float(float arg) { write_value_core(arg); }
+    void write_double(double arg) { write_value_core(arg); }
+
+    // Write lengths.
+    //
+    // Lengths are usually short, so we have a special way of storing
+    // lengths that minimizes the number of bytes when the length is short.
+    //
+    void write_length(size_t len) {
+        if (len >= 255) {
+            write_uint8(0xFF);
+            write_uint64(len);
+        } else {
+            write_uint8(len);
+        }
+    }
+
+    // Write a string.
+    //
+    void write_string(std::string_view s) {
+        write_length(s.size());
+        write_bytes(s);
+    }
+
+    // Write a SimpleDynamic value.
+    //
+    // This works regardless of what kind of string is present in the
+    // SimpleDynamic.
+    //
+    template<class STRING>
+    void write_simple_dynamic(const SimpleDynamic<STRING> &sd) {
+        write_uint8(uint8_t(sd.type));
+        switch(sd.type) {
+            case SimpleDynamicTag::NUMBER: write_double(sd.x); break;
+            case SimpleDynamicTag::BOOLEAN: write_bool(sd.x == 1.0); break;
+            case SimpleDynamicTag::VECTOR: write_double(sd.x); write_double(sd.y); write_double(sd.z); break;
+            case SimpleDynamicTag::STRING: write_string(sd.s); break;
+            default: assert(false);
+        }
+    }
+
+    // Read a block of bytes from the buffer.
+    //
+    // Caution: the pointer returned is a pointer to the stream's buffer.
+    // It is only valid until you mutate the buffer. If the bytes aren't
+    // there, calls 'raise_eof_on_read', and then returns nullptr.
+    //
+    const char *read_bytes(int64_t bytes) {
+        int64_t avail = write_cursor_ - read_cursor_;
+        if (avail < bytes) {
+            CoreHandler::raise_eof_on_read();
+            return nullptr;
+        }
+        char *data = read_cursor_;
+        read_cursor_ += bytes;
+        return data;
+    }
+    
+    // Read integers.
+    //
+    uint8_t read_uint8() { return read_value_core<uint8_t>(); }
+    uint16_t read_uint16() { return read_value_core<uint16_t>(); }
+    uint32_t read_uint32() { return read_value_core<uint32_t>(); }
+    uint64_t read_uint64() { return read_value_core<uint64_t>(); }
+    int8_t read_int8() { return read_value_core<int8_t>(); }
+    int16_t read_int16() { return read_value_core<int16_t>(); }
+    int32_t read_int32() { return read_value_core<int32_t>(); }
+    int64_t read_int64() { return read_value_core<int64_t>(); }
+    
+    // Read other primitive types.
+    //
+    bool read_bool() { return (bool)read_uint8(); }
+    char read_char() { return read_value_core<char>(); }
+    float read_float() { return read_value_core<float>(); }
+    double read_double() { return read_value_core<double>(); }
+
+    // Read a length.
+    //
+    size_t read_length() {
+        uint64_t len = read_uint8();
+        if (len == 255) {
+            len = read_uint64();
+        }
+        return len;
+    }
+
+    // Read a string as a string_view.
+    //
+    // If the string in the buffer is longer than the limit,
+    // calls 'raise_string_too_long' and returns an empty string.
+    //
+    // If the buffer doesn't contain a complete string, calls
+    // 'raise_eof_on_read' and returns an empty string.
+    //
+    std::string_view read_string_view_limit(uint64_t limit) {
+        size_t length = read_length();
+        if (length > limit) {
+            CoreHandler::raise_string_too_long();
+            return std::string_view();
+        }
+        int64_t avail = write_cursor_ - read_cursor_;
+        if (avail < int64_t(length)) {
+            CoreHandler::raise_eof_on_read();
+            return std::string_view();
+        }
+        std::string_view result(read_cursor_, length);
+        read_cursor_ += length;
+        return result;
+    }
+
+    // Read a string as a string_view.
+    //
+    std::string_view read_string_view() {
+        return read_string_view_limit(0x1000000);
+    }
+
+    // Read a string.
+    //
+    string_type read_string_limit(uint64_t limit) {
+        size_t len = read_length();
+        if (len > limit) {
+            CoreHandler::raise_string_too_long();
+            return string_type();
+        }
+        int64_t avail = write_cursor_ - read_cursor_;
+        if (avail < int64_t(len)) {
+            CoreHandler::raise_eof_on_read();
+            return string_type();
+        }
+        string_type result(len, ' ');
+        memcpy(&result[0], read_cursor_, len);
+        read_cursor_ += len;
+        return result;
+    }
+
+    // Read a string.
+    //
+    string_type read_string() {
+        return read_string_limit(0x1000000);
+    }
+    
+    // Read a SimpleDynamic
+    //
+    template<class STRING>
+    void read_simple_dynamic(SimpleDynamic<STRING> *result) {
+        SimpleDynamicTag type = SimpleDynamicTag(read_uint8());
+        switch (type) {
+            case SimpleDynamicTag::NUMBER: result->set_number(read_double()); break;
+            case SimpleDynamicTag::BOOLEAN: result->set_boolean(read_bool()); break;
+            case SimpleDynamicTag::VECTOR: {
+                double x=read_double();
+                double y=read_double();
+                double z=read_double();
+                result->set_vector(x,y,z);
+                break;
+            }
+            case SimpleDynamicTag::STRING: result->set_string(read_string()); break;
+            default: assert(false);
+        }
+    }
+
+    // Attempt to do a "readline".  If there is no newline in
+    // the buffer, returns empty string.  If there is a newline,
+    // returns a block of text that ends in newline.
+    //
+    string_type readline() {
+        char *p = read_cursor_;
+        while ((p < write_cursor_) && (*p != '\n')) p++;
+        if (p == write_cursor_) {
+            return string_type();
+        } else {
+            p++;
+            string_type result(read_cursor_, p - read_cursor_);
+            read_cursor_ = p;
+            return result;
+        }
+    }
+
+    // Overwrite values previously written to the buffer.
+    //
+    // See the comment at the top of this file for an explanation.
+    //
+    void overwrite_int8(int64_t write_count_after, int64_t v) { overwrite_int_core<int8_t>(write_count_after, v); }
+    void overwrite_int16(int64_t write_count_after, int64_t v) { overwrite_int_core<int16_t>(write_count_after, v); }
+    void overwrite_int32(int64_t write_count_after, int64_t v) { overwrite_int_core<int32_t>(write_count_after, v); }
+    void overwrite_int64(int64_t write_count_after, int64_t v) { overwrite_int_core<int64_t>(write_count_after, v); }
+    void overwrite_uint8(int64_t write_count_after, uint64_t v) { overwrite_uint_core<uint8_t>(write_count_after, v); }
+    void overwrite_uint16(int64_t write_count_after, uint64_t v) { overwrite_uint_core<uint16_t>(write_count_after, v); }
+    void overwrite_uint32(int64_t write_count_after, uint64_t v) { overwrite_uint_core<uint32_t>(write_count_after, v); }
+    void overwrite_uint64(int64_t write_count_after, uint64_t v) { overwrite_uint_core<uint64_t>(write_count_after, v); }
+
+    // This is for unit testing.
+    //
+    bool layout_is(int64_t a, int64_t b, int64_t c) {
+        if (read_cursor_ - buf_lo_ != a) return false;
+        if (write_cursor_ - read_cursor_ != b) return false;
+        if (buf_hi_ - write_cursor_ != c) return false;
+        return true;
+    }
+
+private:
+    void make_space_slow(int64_t bytes) {
+        assert(owned_ && "We don't own this buffer, can't grow it");
+
+        // Decide whether the current buffer is big enough.
+        int64_t data_size = (write_cursor_ - read_cursor_);
+        int64_t existing_size = (buf_hi_ - buf_lo_);
+        int64_t desired_size = 8192 + ((data_size + bytes) * 2);
+
+        // Update some simple things.
+        pre_read_count_ += (read_cursor_ - buf_lo_);
+
+        // Move the data to the beginning of the buffer, or to
+        // the beginning of a new buffer.
+        if (fixed_size_) {
+            assert((data_size + bytes <= existing_size) && "Not enough space in fixed-size buffer");
+            if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size);
+        } else if (existing_size >= desired_size) {
+            if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size);
+        } else {
+            char *nbuf = (char *)CoreHandler::basebuffer_malloc(desired_size);
+            if (data_size > 0) memcpy(nbuf, read_cursor_, data_size);
+            if (buf_lo_ != nullptr) CoreHandler::basebuffer_free(buf_lo_);
+            buf_lo_ = nbuf;
+            buf_hi_ = nbuf + desired_size;
+        }
+
+        // Update the pointers to the data region.
+        read_cursor_ = buf_lo_;
+        write_cursor_ = buf_lo_ + data_size;
+    }
+
+    template<class T>
+    void write_value_core(T arg) {
+        make_space(sizeof(arg));
+        memcpy(write_cursor_, &arg, sizeof(arg));
+        write_cursor_ += sizeof(arg);
+    }
+
+    template<class T>
+    void write_int_core(int64_t arg) {
+        T reduced = arg;
+        if (int64_t(reduced) != arg) CoreHandler::raise_integer_truncated();
+        write_value_core(reduced);
+    }
+
+    template<class T>
+    void write_uint_core(uint64_t arg) {
+        T reduced = arg;
+        if (uint64_t(reduced) != arg) CoreHandler::raise_integer_truncated();
+        write_value_core(reduced);
+    }
+
+    template<class T>
+    T read_value_core() {
+        T result;
+        int64_t avail = write_cursor_ - read_cursor_;
+        if (avail < int64_t(sizeof(result))) {
+            CoreHandler::raise_eof_on_read();
+            return 0;
+        }
+        memcpy(&result, read_cursor_, sizeof(result));
+        read_cursor_ += sizeof(result);
+        return result;
+    }
+
+    template<class T>
+    void overwrite_int_core(int64_t write_count_after, int64_t vv) {
+        T v = vv;
+        assert(int64_t(v) == vv);
+        int64_t write_count_before = write_count_after - sizeof(v);
+        assert(write_count_before >= total_reads());
+        assert(write_count_after <= total_writes());
+        void *target = buf_lo_ + (write_count_before - pre_read_count_);
+        memcpy(target, &v, sizeof(v));
+    }
+
+    template<class T>
+    void overwrite_uint_core(int64_t write_count_after, uint64_t vv) {
+        T v = vv;
+        assert(uint64_t(v) == vv);
+        int64_t write_count_before = write_count_after - sizeof(v);
+        assert(write_count_before >= total_reads());
+        assert(write_count_after <= total_writes());
+        void *target = buf_lo_ + (write_count_before - pre_read_count_);
+        memcpy(target, &v, sizeof(v));
+    }
+};

luprex/ext/base-writer.hpp

@@ -1,301 +0,0 @@
-#pragma once
-
-/////////////////////////////////////////////////////////////////
-//
-// IMPORTANT: This is a header-only library that is included
-// by the graphics engine as well.  It cannot contain references
-// to anything else in the engine.
-//
-/////////////////////////////////////////////////////////////////
-
-#include <cstdio>
-#include <cstdint>
-#include <cstdlib>
-#include <cassert>
-#include <string_view>
-
-///////////////////////////////////////////////////////////////
-//
-// SimpleDynamic
-//
-// A struct that holds a dynamically typed value.
-// This can hold a string, number, vector, or boolean.
-//
-// The type is stored in the 'type' field.
-//
-// If it's a STRING, the value is in the field s
-// If it's a NUMBER, the value is in the field x
-// If it's a BOOLEAN, it's true if (x==1.0)
-// If it's a VECTOR, the value is in x,y,z
-//
-///////////////////////////////////////////////////////////////
-
-enum class SimpleDynamicTag {
-    UNINITIALIZED,
-    STRING,
-    NUMBER,
-    BOOLEAN,
-    VECTOR,
-};
-
-template<class STRING>
-struct SimpleDynamic {
-    using string = STRING;
-    SimpleDynamicTag type;
-    double x, y, z;
-    string s;
-    
-    SimpleDynamic() {
-        type = SimpleDynamicTag::UNINITIALIZED;
-        x=y=z=0;
-    }
-
-    static const char *type_name_of(SimpleDynamicTag t) {
-        switch (t) {
-            case SimpleDynamicTag::UNINITIALIZED: return "uninitialized";
-            case SimpleDynamicTag::BOOLEAN: return "boolean";
-            case SimpleDynamicTag::NUMBER: return "number";
-            case SimpleDynamicTag::STRING: return "string";
-            case SimpleDynamicTag::VECTOR: return "vector";
-            default: return "unknown";
-        }
-    }
-
-    const char *type_name() const {
-        return type_name_of(type);
-    }
-
-    void set_uninitialized() {
-        type=SimpleDynamicTag::UNINITIALIZED; s.clear(); x=y=z=0;
-    }
-
-    void set_string(std::string_view is) {
-        type=SimpleDynamicTag::STRING; s=is; x=y=z=0;
-    }
-
-    void set_number(double n) {
-        type = SimpleDynamicTag::NUMBER; s.clear(); x=n; y=z=0;
-    }
-
-    void set_boolean(bool b) {
-        type = SimpleDynamicTag::BOOLEAN; s.clear(); x=(b?1:0); y=z=0;
-    }
-
-    void set_vector(double ix, double iy, double iz) {
-        type = SimpleDynamicTag::VECTOR; s.clear(); x=ix; y=iy; z=iz;
-    }
-
-    void copy_value(const SimpleDynamic &other) {
-        type = other.type;
-        s=other.s; x=other.x; y=other.y; z=other.z;
-    }
-};
-
-///////////////////////////////////////////////////////////////
-//
-// BaseWriter
-//
-// This base class provides the following methods:
-//
-//     void write_uint8(uint64_t data)
-//     void write_uint16(uint64_t data)
-//     void write_uint32(uint64_t data)
-//     void write_uint64(uint64_t data)
-//     void write_int8(int64_t data)
-//     void write_int16(int64_t data)
-//     void write_int32(int64_t data)
-//     void write_int64(int64_t data)
-//     void write_char(char c)
-//     void write_float(float data)
-//     void write_double(double data)
-//     void write_length(size_t data)
-//     void write_string(std::string_view data)
-//     void write_simple_dynamic(const SimpleDynamic &sd);
-//
-// You should derive from BaseWriter using the CRTP pattern:
-//
-//     class DerivedWriter : public BaseWriter<DerivedWriter>
-//
-// You must provide two methods in the derived class:
-//
-//     write_bytes(const char *n, size_t size)
-//     raise_truncated()
-//
-///////////////////////////////////////////////////////////////
-
-template<class Derived>
-class BaseWriter {
-protected:
-    template<class T>
-    void write_value_core(T arg) {
-        static_cast<Derived*>(this)->write_bytes((const char *)&arg, sizeof(arg));
-    }
-
-    template<class T, class XT>
-    void write_int_core(XT arg) {
-        T reduced = arg;
-        if (XT(reduced) != arg) static_cast<Derived*>(this)->raise_truncated();
-        write_value_core(reduced);
-    }
-
-public:
-
-    void write_uint8(uint64_t data) { write_int_core<uint8_t, uint64_t>(data); }
-    void write_uint16(uint64_t data) { write_int_core<uint16_t, uint64_t>(data); }
-    void write_uint32(uint64_t data) { write_int_core<uint32_t, uint64_t>(data); }
-    void write_uint64(uint64_t data) { write_int_core<uint64_t, uint64_t>(data); }
-    void write_int8(int64_t data) { write_int_core<int8_t, int64_t>(data); }
-    void write_int16(int64_t data) { write_int_core<int16_t, int64_t>(data); }
-    void write_int32(int64_t data) { write_int_core<int32_t, int64_t>(data); }
-    void write_int64(int64_t data) { write_int_core<int64_t, int64_t>(data); }
-    
-    void write_bool(bool b) { write_uint8(b ? 1:0); }
-    void write_char(char c) { write_value_core(c); }
-    void write_float(float arg) { write_value_core(arg); }
-    void write_double(double arg) { write_value_core(arg); }
-
-    void write_length(size_t len) {
-        if (len >= 255) {
-            write_uint8(0xFF);
-            write_uint64(len);
-        } else {
-            write_uint8(len);
-        }
-    }
-
-    void write_string(std::string_view s) {
-        write_length(s.size());
-        static_cast<Derived*>(this)->write_bytes(s.data(), s.size());
-    }
-
-    template<class STRING>
-    void write_simple_dynamic(const SimpleDynamic<STRING> &sd) {
-        write_uint8(uint8_t(sd.type));
-        switch(sd.type) {
-            case SimpleDynamicTag::NUMBER: write_double(sd.x); break;
-            case SimpleDynamicTag::BOOLEAN: write_bool(sd.x == 1.0); break;
-            case SimpleDynamicTag::VECTOR: write_double(sd.x); write_double(sd.y); write_double(sd.z); break;
-            case SimpleDynamicTag::STRING: write_string(sd.s); break;
-            default: assert(false);
-        }
-    }
-};
-
-
-///////////////////////////////////////////////////////////////
-//
-// BaseReader
-//
-// This base class provides the following methods:
-//
-//     uint8_t read_uint8();
-//     uint16_t read_uint16();
-//     uint32_t read_uint32();
-//     uint64_t read_uint64();
-//     int8_t read_int8();
-//     int16_t read_int16();
-//     int32_t read_int32();
-//     int64_t read_int64();
-//     bool read_bool();
-//     char read_char();
-//     float read_float();
-//     double read_double();
-//     size_t read_length();
-//     String read_string_limit(uint64_t size);
-//     String read_string();
-//     SimpleDynamic read_simple_dynamic();
-//
-// You should derive from BaseReader using the CRTP pattern:
-//
-//     class DerivedReader : public BaseReader<DerivedReader>
-//
-// The derived class must provide:
-//
-//     using read_string_type = std::string; // or compatible
-//     void read_bytes_into(char *n, size_t size)
-//     void handle_string_too_long();
-//
-// Error Handling:
-//
-//     It is up to the derived class whether it wants
-//     to report errors using exceptions or flags.
-//
-//     If read_bytes_into discovers there's not enough bytes,
-//     there are two valid options: throw an exception, OR,
-//     set an error flag and fill the buffer with zeros.
-//
-//     If read_string discovers that the string is longer than
-//     the allowed limit, it will call handle_string_too_long.
-//     This function may either throw an exception, or set an
-//     error flag.
-//
-///////////////////////////////////////////////////////////////
-
-template<class Derived>
-class BaseReader {
-protected:
-    template<class T>
-    T read_value_core() {
-        T result;
-        Derived *dthis = static_cast<Derived*>(this);
-        dthis->read_bytes_into((char *)(&result), sizeof(result));
-        return result;
-    }
-
-public:
-
-    uint8_t read_uint8() { return read_value_core<uint8_t>(); }
-    uint16_t read_uint16() { return read_value_core<uint16_t>(); }
-    uint32_t read_uint32() { return read_value_core<uint32_t>(); }
-    uint64_t read_uint64() { return read_value_core<uint64_t>(); }
-    int8_t read_int8() { return read_value_core<int8_t>(); }
-    int16_t read_int16() { return read_value_core<int16_t>(); }
-    int32_t read_int32() { return read_value_core<int32_t>(); }
-    int64_t read_int64() { return read_value_core<int64_t>(); }
-    
-    bool read_bool() { return (bool)read_uint8(); }
-    char read_char() { return read_value_core<char>(); }
-    float read_float() { return read_value_core<float>(); }
-    double read_double() { return read_value_core<double>(); }
-    
-    size_t read_length() {
-        uint64_t len = read_uint8();
-        if (len == 255) {
-            len = read_uint64();
-        }
-        return len;
-    }
-
-    auto read_string_limit(uint64_t limit) {
-        size_t len = read_length();
-        Derived *dthis = static_cast<Derived*>(this);
-        if (len > limit) {
-            dthis->raise_string_too_long();
-            len = 0;
-        }
-        typename Derived::read_string_type result(len, ' ');
-        dthis->read_bytes_into(&(result[0]), len);
-        return result;
-    }
-
-    auto read_string() { return read_string_limit(0x1000000); } // 16MB limit default
-
-    template<class STRING>
-    void read_simple_dynamic(SimpleDynamic<STRING> *result) {
-        SimpleDynamicTag type = SimpleDynamicTag(read_uint8());
-        switch (type) {
-            case SimpleDynamicTag::NUMBER: result->set_number(read_double()); break;
-            case SimpleDynamicTag::BOOLEAN: result->set_boolean(read_bool()); break;
-            case SimpleDynamicTag::VECTOR: {
-                double x=read_double();
-                double y=read_double();
-                double z=read_double();
-                result->set_vector(x,y,z);
-                break;
-            }
-            case SimpleDynamicTag::STRING: result->set_string(read_string()); break;
-            default: assert(false);
-        }
-    }
-};
-