#pragma once ///////////////////////////////////////////////////////////////// // // IMPORTANT: This is a header-only library that is included // by the graphics engine as well. It cannot contain references // to anything else in the engine. // ///////////////////////////////////////////////////////////////// #include #include #include #include #include #include #include /////////////////////////////////////////////////////////////// // // BaseLuaValue // // A struct that holds a dynamically typed value. // This can hold a string, token, number, vector, or boolean. // // The type is stored in the 'type' field. // // If it's a STRING, the value is in the field s // If it's a TOKEN, the value is stored in the field s // If it's a NUMBER, the value is in the field x // If it's a BOOLEAN, it's true if (x==1.0) // If it's a VECTOR, the value is in x,y,z // /////////////////////////////////////////////////////////////// enum class LuaValueType { UNINITIALIZED, STRING, TOKEN, NUMBER, BOOLEAN, VECTOR, }; template struct BaseLuaValue { using string = STRING; LuaValueType type; double x, y, z; string s; BaseLuaValue() { type = LuaValueType::UNINITIALIZED; x=y=z=0; } static const char *type_name_of(LuaValueType t) { switch (t) { case LuaValueType::UNINITIALIZED: return "uninitialized"; case LuaValueType::STRING: return "string"; case LuaValueType::TOKEN: return "token"; case LuaValueType::BOOLEAN: return "boolean"; case LuaValueType::NUMBER: return "number"; case LuaValueType::VECTOR: return "vector"; default: return "unknown"; } } const char *type_name() const { return type_name_of(type); } void set_uninitialized() { type=LuaValueType::UNINITIALIZED; s.clear(); x=y=z=0; } void set_string(std::string_view is) { type=LuaValueType::STRING; s=is; x=y=z=0; } void set_token(std::string_view is) { type=LuaValueType::TOKEN; s=is; x=y=z=0; } void set_number(double n) { type = LuaValueType::NUMBER; s.clear(); x=n; y=z=0; } void set_boolean(bool b) { type = LuaValueType::BOOLEAN; s.clear(); x=(b?1:0); y=z=0; } void set_vector(double ix, double iy, double iz) { type = LuaValueType::VECTOR; s.clear(); x=ix; y=iy; z=iz; } void copy_value(const BaseLuaValue &other) { type = other.type; s=other.s; x=other.x; y=other.y; z=other.z; } }; /////////////////////////////////////////////////////////////// // // DataSerializer // // DataSerializer is an object that can serialize ints, // strings, floats, and other basic types. It provides a // consistent standard for the byte formats. // // To serialize, first construct a DataSerializer, passing // in a pointer to an output device. An output device is // any class that has these methods and type aliases: // // void write_bytes(const char *data, size_t len); // void raise_integer_truncated(); // // using fvector_type = ...; // needed for write_xyz // using dvector_type = ...; // needed for write_dxyz // using luavalue_type = ...; // needed for write_lua_value // // After constructing the DataSerializer, call write_int, // write_float, write_string, or the like. The data will be // written to the output device using write_bytes. If // there's an error, a 'raise' method may be called on the // output device. // // It is intended that the compiler will optimize this // process to such a degree that it costs no more than // simply calling write_bytes directly on the output // device. In other words, it is our intent that the // use of a DataSerializer should be free. // /////////////////////////////////////////////////////////////// template class DataSerializer { private: OutputDevice *output_; template void write_value_core(T arg) { output_->write_bytes((const char *)&arg, sizeof(arg)); } template void write_int_core(XT arg) { T reduced = arg; if (XT(reduced) != arg) output_->raise_integer_truncated(); output_->write_bytes((const char *)&reduced, sizeof(reduced)); } public: DataSerializer(OutputDevice *o) : output_(o) {} void write_uint8(uint64_t data) { write_int_core(data); } void write_uint16(uint64_t data) { write_int_core(data); } void write_uint32(uint64_t data) { write_int_core(data); } void write_uint64(uint64_t data) { write_int_core(data); } void write_int8(int64_t data) { write_int_core(data); } void write_int16(int64_t data) { write_int_core(data); } void write_int32(int64_t data) { write_int_core(data); } void write_int64(int64_t data) { write_int_core(data); } void write_bool(bool b) { write_uint8(b ? 1:0); } void write_char(char c) { write_value_core(c); } void write_float(float arg) { write_value_core(arg); } void write_double(double arg) { write_value_core(arg); } void write_length(size_t len) { if (len >= 255) { write_uint8(0xFF); write_uint64(len); } else { write_uint8(len); } } void write_string(std::string_view s) { write_length(s.size()); output_->write_bytes(s.data(), s.size()); } void write_lua_value_type(LuaValueType tag) { write_uint8(uint8_t(tag)); } template void write_xyz(const FV &v) { static_assert(std::is_same_v); if constexpr (requires { v.x; }) { write_float(v.x); write_float(v.y); write_float(v.z); } else { write_float(v.X); write_float(v.Y); write_float(v.Z); } } template void write_dxyz(const DV &v) { static_assert(std::is_same_v); if constexpr (requires { v.x; }) { write_double(v.x); write_double(v.y); write_double(v.z); } else { write_double(v.X); write_double(v.Y); write_double(v.Z); } } template void write_lua_value(const LV &sd) { static_assert(std::is_same_v); write_lua_value_type(sd.type); switch(sd.type) { case LuaValueType::STRING: write_string(sd.s); break; case LuaValueType::TOKEN: write_string(sd.s); break; case LuaValueType::NUMBER: write_double(sd.x); break; case LuaValueType::BOOLEAN: write_bool(sd.x == 1.0); break; case LuaValueType::VECTOR: write_double(sd.x); write_double(sd.y); write_double(sd.z); break; default: assert(false); } } }; /////////////////////////////////////////////////////////////// // // DataDeserializer // // DataDeserializer is an object that can deserialize ints, // strings, floats, and other basic types. It provides a // consistent standard for the byte formats. // // To deserialize, first construct a DataDeserializer, passing // in a pointer to an input device. An input device is // any class that has these methods and type aliases: // // void read_bytes_into(char *data, size_t len); // void raise_string_too_long(); // // using string_type = ...; // needed for read_string // using fvector_type = ...; // needed for read_xyz // using dvector_type = ...; // needed for read_dxyz // using luavalue_type = ...; // needed for read_lua_value // // After constructing the DataDeserializer, call read_int, // read_float, read_string, or the like. The data will be // read from the input device using read_bytes_into. If // there's an error, a 'raise' method may be called on the // input device. // // It is intended that the compiler will optimize this // process to such a degree that it costs no more than // simply calling read_bytes_into directly on the input // device. In other words, it is our intent that the // use of a DataDeserializer should be free. // /////////////////////////////////////////////////////////////// template class DataDeserializer { private: InputDevice *input_; template T read_value_core() { T result; input_->read_bytes_into((char *)(&result), sizeof(result)); return result; } public: DataDeserializer(InputDevice *i) : input_(i) {} uint8_t read_uint8() { return read_value_core(); } uint16_t read_uint16() { return read_value_core(); } uint32_t read_uint32() { return read_value_core(); } uint64_t read_uint64() { return read_value_core(); } int8_t read_int8() { return read_value_core(); } int16_t read_int16() { return read_value_core(); } int32_t read_int32() { return read_value_core(); } int64_t read_int64() { return read_value_core(); } bool read_bool() { return (bool)read_uint8(); } char read_char() { return read_value_core(); } float read_float() { return read_value_core(); } double read_double() { return read_value_core(); } size_t read_length() { uint64_t len = read_uint8(); if (len == 255) len = read_uint64(); return len; } auto read_string_limit(uint64_t limit) { using ST = typename InputDevice::string_type; size_t len = read_length(); if (len > limit) { input_->raise_string_too_long(); len = 0; } ST result(len, ' '); input_->read_bytes_into(&(result[0]), len); return result; } auto read_string() { return read_string_limit(0x1000000); // 16MB limit default } auto read_xyz() { using FV = typename InputDevice::fvector_type; float x = read_float(), y = read_float(), z = read_float(); return FV(x, y, z); } auto read_dxyz() { using DV = typename InputDevice::dvector_type; double x = read_double(), y = read_double(), z = read_double(); return DV(x, y, z); } LuaValueType read_lua_value_type() { return LuaValueType(read_uint8()); } template void read_lua_value(LV *result) { static_assert(std::is_same_v); LuaValueType type = read_lua_value_type(); switch (type) { case LuaValueType::STRING: result->set_string(read_string()); break; case LuaValueType::TOKEN: result->set_token(read_string()); break; case LuaValueType::NUMBER: result->set_number(read_double()); break; case LuaValueType::BOOLEAN: result->set_boolean(read_bool()); break; case LuaValueType::VECTOR: { double x=read_double(); double y=read_double(); double z=read_double(); result->set_vector(x,y,z); break; } default: result->set_uninitialized(); break; } } }; /////////////////////////////////////////////////////////////// // // Class BaseBuffer // // You must supply a BaseBufferConfig which must define these: // // using string_type = ...; // string type for read_string // using fvector_type = ...; // vector type for read/write_xyz // using dvector_type = ...; // vector type for read/write_dxyz // using luavalue_type = ...; // lua value type for read/write_lua_value // void *basebuffer_malloc(size_t size); // void basebuffer_free(void *data); // void clear_error_flags(); // void raise_eof_on_read(); // void raise_string_too_long(); // void raise_integer_truncated(); // /////////////////////////////////////////////////////////////// template class BaseBuffer : public BaseBufferConfig { private: // True if we own this buffer. bool owned_; // True if we're not allowed to expand this buffer. bool fixed_size_; // Start and end of the allocated block. char *buf_lo_; char *buf_hi_; // The write and read cursors. char *write_cursor_; char *read_cursor_; // Number of bytes read before buffer was last aligned. int64_t pre_read_count_; public: using string_type = typename BaseBufferConfig::string_type; using fvector_type = typename BaseBufferConfig::fvector_type; using dvector_type = typename BaseBufferConfig::dvector_type; using luavalue_type = typename BaseBufferConfig::luavalue_type; private: using DS = DataSerializer>; using DD = DataDeserializer>; void init(bool fixed, bool owned, char *buf, int64_t size) { BaseBufferConfig::clear_error_flags(); owned_ = owned; fixed_size_ = fixed; buf_lo_ = buf; buf_hi_ = buf_lo_ + size; read_cursor_ = buf_lo_; write_cursor_ = buf_lo_; pre_read_count_ = 0; } public: // Construct an empty buffer. // BaseBuffer() { init(false, true, 0, 0); } // Construct an empty buffer, preallocate the specified amount of space. // BaseBuffer(int64_t size, bool fixed) { assert(size >= 0); init(fixed, true, (char *)BaseBufferConfig::basebuffer_malloc(size), size); } // Construct a streambuffer that reads from an external block of bytes. // BaseBuffer(std::string_view data) { init(true, false, const_cast(data.data()), data.size()); write_cursor_ = buf_hi_; } // Modify an existing streambuffer to read from an external block of bytes. // void open(std::string_view data) { if (owned_ && (buf_lo_ != 0)) BaseBufferConfig::basebuffer_free(buf_lo_); init(true, false, const_cast(data.data()), data.size()); write_cursor_ = buf_hi_; } // Destructor. Frees the buffer, if any. // ~BaseBuffer() { if (owned_ && (buf_lo_ != 0)) BaseBufferConfig::basebuffer_free(buf_lo_); } // Return the total number of bytes ever read. // int64_t total_reads() const { return (read_cursor_ - buf_lo_) + pre_read_count_; } // Return the total number of bytes ever written. // int64_t total_writes() const { return (write_cursor_ - buf_lo_) + pre_read_count_; } // Return the total bytes in the buffer. // int64_t fill() const { return write_cursor_ - read_cursor_; } // Checks to see if the buffer is empty. // bool empty() const { return write_cursor_ == read_cursor_; } // Return the contents as a string_view. // std::string_view view() const { return std::string_view(read_cursor_, write_cursor_ - read_cursor_); } // Make the specified amount of space in the buffer for writing. // char *make_space(int64_t bytes) { int64_t available = buf_hi_ - write_cursor_; if (available < bytes) make_space_slow(bytes); return write_cursor_; } // Used after calling make_space then filling the space. // void wrote_space(int64_t bytes) { int64_t available = buf_hi_ - write_cursor_; assert(bytes >= 0); assert(available >= bytes); write_cursor_ += bytes; } // Rewind the read cursor to a previous position. // void unread_to(int64_t rd_count) { assert(rd_count >= pre_read_count_); assert(rd_count <= total_reads()); read_cursor_ = buf_lo_ + (rd_count - pre_read_count_); } // Rewind the write cursor to a previous position. // void unwrite_to(int64_t wr_count) { assert(wr_count >= total_reads()); assert(wr_count <= total_writes()); write_cursor_ = buf_lo_ + (wr_count - pre_read_count_); } // Discard all data. Reset total read and write counts. // May release the allocated buffer, if it is large. // void clear() { if (!owned_) { open(""); } else { if ((!fixed_size_) && (buf_lo_ != nullptr) && ((buf_hi_ - buf_lo_) > 100000)) { BaseBufferConfig::basebuffer_free(buf_lo_); buf_lo_ = nullptr; buf_hi_ = nullptr; } read_cursor_ = buf_lo_; write_cursor_ = buf_lo_; pre_read_count_ = 0; } } // Write block of bytes into the buffer. // void write_bytes(const char *data, size_t size) { make_space(size); memcpy(write_cursor_, data, size); write_cursor_ += size; } void write_bytes(std::string_view s) { write_bytes(s.data(), s.size()); } // Copy the contents of this buffer into another buffer. // void copy_into(BaseBuffer *sb) { sb->write_bytes(view()); } // Check if the contents of this buffer are equal to another. // bool contents_equal(const BaseBuffer *sb) const { return view() == sb->view(); } // Read bytes into a caller-supplied buffer. // // This is the primitive read operation used by DataDeserializer. // If there aren't enough bytes, calls raise_eof_on_read and // fills the buffer with zeros. // void read_bytes_into(char *data, size_t size) { int64_t avail = write_cursor_ - read_cursor_; if (avail < int64_t(size)) { BaseBufferConfig::raise_eof_on_read(); memset(data, 0, size); return; } memcpy(data, read_cursor_, size); read_cursor_ += size; } // Read a block of bytes from the buffer. // // Caution: the pointer returned is a pointer to the stream's buffer. // It is only valid until you mutate the buffer. If the bytes aren't // there, calls 'raise_eof_on_read', and then returns nullptr. // const char *read_bytes(int64_t bytes) { int64_t avail = write_cursor_ - read_cursor_; if (avail < bytes) { BaseBufferConfig::raise_eof_on_read(); return nullptr; } char *data = read_cursor_; read_cursor_ += bytes; return data; } // Write methods — delegate to DataSerializer. // void write_uint8(uint64_t data) { DS(this).write_uint8(data); } void write_uint16(uint64_t data) { DS(this).write_uint16(data); } void write_uint32(uint64_t data) { DS(this).write_uint32(data); } void write_uint64(uint64_t data) { DS(this).write_uint64(data); } void write_int8(int64_t data) { DS(this).write_int8(data); } void write_int16(int64_t data) { DS(this).write_int16(data); } void write_int32(int64_t data) { DS(this).write_int32(data); } void write_int64(int64_t data) { DS(this).write_int64(data); } void write_bool(bool b) { DS(this).write_bool(b); } void write_char(char c) { DS(this).write_char(c); } void write_float(float arg) { DS(this).write_float(arg); } void write_double(double arg) { DS(this).write_double(arg); } void write_length(size_t len) { DS(this).write_length(len); } void write_string(std::string_view s) { DS(this).write_string(s); } void write_lua_value_type(LuaValueType tag) { DS(this).write_lua_value_type(tag); } void write_xyz(const fvector_type &v) { DS(this).write_xyz(v); } void write_dxyz(const dvector_type &v) { DS(this).write_dxyz(v); } void write_lua_value(const luavalue_type &sd) { DS(this).write_lua_value(sd); } // Read methods — delegate to DataDeserializer. // uint8_t read_uint8() { return DD(this).read_uint8(); } uint16_t read_uint16() { return DD(this).read_uint16(); } uint32_t read_uint32() { return DD(this).read_uint32(); } uint64_t read_uint64() { return DD(this).read_uint64(); } int8_t read_int8() { return DD(this).read_int8(); } int16_t read_int16() { return DD(this).read_int16(); } int32_t read_int32() { return DD(this).read_int32(); } int64_t read_int64() { return DD(this).read_int64(); } bool read_bool() { return DD(this).read_bool(); } char read_char() { return DD(this).read_char(); } float read_float() { return DD(this).read_float(); } double read_double() { return DD(this).read_double(); } size_t read_length() { return DD(this).read_length(); } string_type read_string_limit(uint64_t limit) { return DD(this).read_string_limit(limit); } string_type read_string() { return DD(this).read_string(); } LuaValueType read_lua_value_type() { return DD(this).read_lua_value_type(); } fvector_type read_xyz() { return DD(this).read_xyz(); } dvector_type read_dxyz() { return DD(this).read_dxyz(); } void read_lua_value(luavalue_type *result) { DD(this).read_lua_value(result); } // Read a string as a string_view. // // This is BaseBuffer-specific — it returns a view directly into // the buffer, avoiding a copy. // std::string_view read_string_view_limit(uint64_t limit) { size_t length = read_length(); if (length > limit) { BaseBufferConfig::raise_string_too_long(); return std::string_view(); } int64_t avail = write_cursor_ - read_cursor_; if (avail < int64_t(length)) { BaseBufferConfig::raise_eof_on_read(); return std::string_view(); } std::string_view result(read_cursor_, length); read_cursor_ += length; return result; } // Read a string as a string_view. // std::string_view read_string_view() { return read_string_view_limit(0x1000000); } // Attempt to do a "readline". If there is no newline in // the buffer, returns empty string. If there is a newline, // returns a block of text that ends in newline. // string_type readline() { char *p = read_cursor_; while ((p < write_cursor_) && (*p != '\n')) p++; if (p == write_cursor_) { return string_type(); } else { p++; string_type result(read_cursor_, p - read_cursor_); read_cursor_ = p; return result; } } // Overwrite values previously written to the buffer. // // See the comment at the top of this file for an explanation. // void overwrite_int8(int64_t write_count_after, int64_t v) { overwrite_int_core(write_count_after, v); } void overwrite_int16(int64_t write_count_after, int64_t v) { overwrite_int_core(write_count_after, v); } void overwrite_int32(int64_t write_count_after, int64_t v) { overwrite_int_core(write_count_after, v); } void overwrite_int64(int64_t write_count_after, int64_t v) { overwrite_int_core(write_count_after, v); } void overwrite_uint8(int64_t write_count_after, uint64_t v) { overwrite_int_core(write_count_after, v); } void overwrite_uint16(int64_t write_count_after, uint64_t v) { overwrite_int_core(write_count_after, v); } void overwrite_uint32(int64_t write_count_after, uint64_t v) { overwrite_int_core(write_count_after, v); } void overwrite_uint64(int64_t write_count_after, uint64_t v) { overwrite_int_core(write_count_after, v); } // This is for unit testing. // bool layout_is(int64_t a, int64_t b, int64_t c) { if (read_cursor_ - buf_lo_ != a) return false; if (write_cursor_ - read_cursor_ != b) return false; if (buf_hi_ - write_cursor_ != c) return false; return true; } private: void make_space_slow(int64_t bytes) { assert(owned_ && "We don't own this buffer, can't grow it"); // Decide whether the current buffer is big enough. int64_t data_size = (write_cursor_ - read_cursor_); int64_t existing_size = (buf_hi_ - buf_lo_); int64_t desired_size = 8192 + ((data_size + bytes) * 2); // Update some simple things. pre_read_count_ += (read_cursor_ - buf_lo_); // Move the data to the beginning of the buffer, or to // the beginning of a new buffer. if (fixed_size_) { assert((data_size + bytes <= existing_size) && "Not enough space in fixed-size buffer"); if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size); } else if (existing_size >= desired_size) { if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size); } else { char *nbuf = (char *)BaseBufferConfig::basebuffer_malloc(desired_size); if (data_size > 0) memcpy(nbuf, read_cursor_, data_size); if (buf_lo_ != nullptr) BaseBufferConfig::basebuffer_free(buf_lo_); buf_lo_ = nbuf; buf_hi_ = nbuf + desired_size; } // Update the pointers to the data region. read_cursor_ = buf_lo_; write_cursor_ = buf_lo_ + data_size; } template void overwrite_int_core(int64_t write_count_after, XT vv) { T v = vv; assert(XT(v) == vv); int64_t write_count_before = write_count_after - sizeof(v); assert(write_count_before >= total_reads()); assert(write_count_after <= total_writes()); void *target = buf_lo_ + (write_count_before - pre_read_count_); memcpy(target, &v, sizeof(v)); } };