Change directory structure

luprex/cpp/core/animqueue.cpp

@@ -0,0 +1,677 @@
+#include "wrap-sstream.hpp"
+#include "wrap-map.hpp"
+#include "util.hpp"
+#include "animqueue.hpp"
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+
+#include <limits>
+#include <cmath>
+
+AnimStep::AnimStep() {
+    clear();
+}
+
+AnimStep::~AnimStep() {}
+
+
+void AnimStep::clear() {
+    id_ = 0;
+    bits_ = 0;
+    action_ = "";
+    facing_ = 0;
+    xyz_ = util::XYZ(0,0,0);
+    graphic_ = "";
+    plane_ = "";
+}
+
+void AnimStep::set_action(const eng::string &act) {
+    action_ = act;
+}
+
+void AnimStep::set_facing(float f) {
+    bits_ |= HAS_FACING;
+    facing_ = f;
+}
+
+void AnimStep::set_x(float f) {
+    bits_ |= HAS_X;
+    xyz_.x = f;
+}
+
+void AnimStep::set_y(float f) {
+    bits_ |= HAS_Y;
+    xyz_.y = f;
+}
+
+void AnimStep::set_z(float f) {
+    bits_ |= HAS_Z;
+    xyz_.z = f;
+}
+
+void AnimStep::set_xyz(const util::XYZ &xyz) {
+    bits_ |= (HAS_X | HAS_Y | HAS_Z);
+    xyz_ = xyz;
+}
+
+void AnimStep::set_graphic(const eng::string &g) {
+    bits_ |= HAS_GRAPHIC;
+    graphic_ = g;
+}
+
+void AnimStep::set_plane(const eng::string &p) {
+    bits_ |= HAS_PLANE;
+    plane_ = p;
+}
+
+
+bool AnimStep::exactly_equal(const AnimStep &other) const {
+    if (id_ != other.id_) return false;
+    if (bits_ != other.bits_) return false;
+    if (action_ != other.action_) return false;
+    if (facing_ != other.facing_) return false;
+    if (xyz_ != other.xyz_) return false;
+    if (graphic_ != other.graphic_) return false;
+    if (plane_ != other.plane_) return false;
+    return true;
+}
+
+bool AnimStep::logically_equal(const AnimStep &other) const {
+    if (id_ != other.id_) return false;
+    if (bits_ != other.bits_) return false;
+    if (action_ != other.action_) return false;
+    if (has_facing() && (facing_ != other.facing_)) return false;
+    if (has_x() && (xyz_.x != other.xyz_.x)) return false;
+    if (has_y() && (xyz_.y != other.xyz_.y)) return false;
+    if (has_z() && (xyz_.z != other.xyz_.z)) return false;
+    if (has_graphic() && (graphic_ != other.graphic_)) return false;
+    if (has_plane() && (plane_ != other.plane_)) return false;
+    return true;
+}
+
+bool AnimStep::state_equal(const AnimStep &other) const {
+    if (facing_ != other.facing_) return false;
+    if (xyz_ != other.xyz_) return false;
+    if (graphic_ != other.graphic_) return false;
+    if (plane_ != other.plane_) return false;
+    return true;
+}
+
+void AnimStep::write_into(StreamBuffer *sb) const {
+    sb->write_int64(id_);
+    sb->write_int16(bits_);
+    sb->write_string(action_);
+    if (has_facing()) {
+        sb->write_float(facing_);
+    }
+    if (has_x()) {
+        sb->write_float(xyz_.x);
+    }
+    if (has_y()) {
+        sb->write_float(xyz_.y);
+    }
+    if (has_z()) {
+        sb->write_float(xyz_.z);
+    }
+    if (has_graphic()) {
+        sb->write_string(graphic_);
+    }
+    if (has_plane()) {
+        sb->write_string(plane_);
+    }
+}
+
+void AnimStep::read_from(StreamBuffer *sb) {
+    id_ = sb->read_int64();
+    bits_ = sb->read_int16();
+    action_ = sb->read_string();
+    if (has_facing()) {
+        facing_ = sb->read_float();
+    }
+    if (has_x()) {
+        xyz_.x = sb->read_float();
+    }
+    if (has_y()) {
+        xyz_.y = sb->read_float();
+    }
+    if (has_z()) {
+        xyz_.z = sb->read_float();
+    }
+    if (has_graphic()) {
+        graphic_ = sb->read_string();
+    }
+    if (has_plane()) {
+        plane_ = sb->read_string();
+    }
+}
+
+
+void AnimStep::config_store_string(lua_State *L, int idx, eng::string *target, int16_t bits, const char *name) {
+    if ((bits_ & bits)||(*target != "")) {
+        luaL_error(L, "specified %s twice", name);
+    }
+    if (lua_type(L, idx) != LUA_TSTRING) {
+        luaL_error(L, "Expected %s to be a string", name);
+    }
+    *target = lua_tostring(L, idx);
+    bits_ |= bits;
+}
+
+void AnimStep::config_store_number(lua_State *L, int idx, float *target, float offset, int16_t bits, const char *name) {
+    if (bits_ & bits) {
+        luaL_error(L, "specified %s twice", name);
+    }
+    if (lua_type(L, idx) != LUA_TNUMBER) {
+        luaL_error(L, "Expected %s to be a number", name);
+    }
+    *target = lua_tonumber(L, idx) + offset;
+    bits_ |= bits;
+}
+
+void AnimStep::configure(LuaKeywordParser &kp, const AnimStep &aqback) {
+    lua_State *L = kp.state();
+    LuaVar value;
+    LuaStack LS(L, value);
+
+    if (kp.parse(value, "action")) {
+        config_store_string(L, value.index(), &action_, 0, "action");
+    }
+    if (kp.parse(value, "graphic")) {
+        config_store_string(L, value.index(), &graphic_, HAS_GRAPHIC, "graphic");
+    }
+    if (kp.parse(value, "plane")) {
+        config_store_string(L, value.index(), &plane_, HAS_PLANE, "plane");
+    }
+    if (kp.parse(value, "x")) {
+        config_store_number(L, value.index(), &xyz_.x, 0.0, HAS_X, "X coordinate");
+    }
+    if (kp.parse(value, "y")) {
+        config_store_number(L, value.index(), &xyz_.y, 0.0, HAS_Y, "Z coordinate");
+    }
+    if (kp.parse(value, "z")) {
+        config_store_number(L, value.index(), &xyz_.z, 0.0, HAS_Z, "Z coordinate");
+    }
+    if (kp.parse(value, "dx")) {
+        config_store_number(L, value.index(), &xyz_.x, aqback.xyz().x, HAS_X, "X coordinate");
+    }
+    if (kp.parse(value, "dy")) {
+        config_store_number(L, value.index(), &xyz_.y, aqback.xyz().y, HAS_Y, "Y coordinate");
+    }
+    if (kp.parse(value, "dz")) {
+        config_store_number(L, value.index(), &xyz_.z, aqback.xyz().z, HAS_Z, "Z coordinate");
+    }
+    if (kp.parse(value, "facing")) {
+        config_store_number(L, value.index(), &facing_, 0.0, HAS_FACING, "facing");
+    }
+}
+
+void AnimStep::keep_state_only() {
+    bits_ = HAS_EVERYTHING;
+    id_ = 0;
+    action_ = "";
+}
+
+void AnimStep::echo(const AnimStep &prev) {
+    if (!has_facing()) {
+        facing_ = prev.facing_;
+    }
+    if (!has_x()) {
+        xyz_.x = prev.xyz_.x;
+    }
+    if (!has_y()) {
+        xyz_.y = prev.xyz_.y;
+    }
+    if (!has_z()) {
+        xyz_.z = prev.xyz_.z;
+    }
+    if (!has_graphic()) {
+        graphic_ = prev.graphic_;
+    }
+    if (!has_plane()) {
+        plane_ = prev.plane_;
+    }
+}
+
+bool AnimStep::echoes(const AnimStep &prev) const {
+    if (!has_facing() && (facing_ != prev.facing_)) {
+        return false;
+    }
+    if (!has_x() && (xyz_.x != prev.xyz_.x)) {
+        return false;
+    }
+    if (!has_y() && (xyz_.y != prev.xyz_.y)) {
+        return false;
+    }
+    if (!has_z() && (xyz_.z != prev.xyz_.z)) {
+        return false;
+    }
+    if (!has_graphic() && (graphic_ != prev.graphic_)) {
+        return false;
+    }
+    if (!has_plane() && (plane_ != prev.plane_)) {
+        return false;
+    }
+    return true;
+}
+
+eng::string AnimStep::debug_string() const {
+    eng::ostringstream oss;
+    oss << "id=" << id();
+    oss << " action=" << action();
+    if (has_plane()) {
+        oss << " plane=" << plane();
+    }
+    if (has_x()) {
+        oss << " x=" << xyz().x;
+    }
+    if (has_y()) {
+        oss << " y=" << xyz().y;
+    }
+    if (has_z()) {
+        oss << " z=" << xyz().z;
+    }
+    if (has_facing()) {
+        oss << " facing=" << facing();
+    }
+    if (has_graphic()) {
+        oss << " graphic=" << graphic();
+    }
+    return oss.str();
+}
+
+
+bool AnimStep::from_string(const eng::string &config) {
+    clear();
+    util::StringVec parts = util::split(config, ' ');
+    for (int i = 0; i < int(parts.size()); i++) {
+        const eng::string &part = parts[i];
+        if (part == "") continue;
+        util::StringVec lr = util::split(part, '=');
+        if (lr.size() != 2) return false;
+        const eng::string &key = lr[0];
+        const eng::string &val = lr[1];
+        if (key == "action") {
+            action_ = val;
+        } else if (key == "id") {
+            int64_t id = sv::to_int64(val, -1);
+            if (id < 0) return false;
+            id_ = id;
+        } else if (key == "plane") {
+            set_plane(val);
+        } else if (key == "x") {
+            double v = sv::to_double(val);
+            if (std::isnan(v)) return false;
+            set_x(v);
+        } else if (key == "y") {
+            double v = sv::to_double(val);
+            if (std::isnan(v)) return false;
+            set_y(v);
+        } else if (key == "z") {
+            double v = sv::to_double(val);
+            if (std::isnan(v)) return false;
+            set_z(v);
+        } else if (key == "facing") {
+            double v = sv::to_double(val);
+            if (std::isnan(v)) return false;
+            set_facing(v);
+        } else if (key == "graphic") {
+            set_graphic(val);
+        } else {
+            return false;
+        }
+    }
+    return true;
+}
+
+AnimQueue::AnimQueue(util::WorldType wt) {
+    version_autoinc_ = (wt == util::WORLD_TYPE_MASTER);
+    size_limit_ = 10; // Default size limit.
+    steps_.emplace_back();
+    steps_.front().keep_state_only();
+    version_number_ = version_autoinc_ ? 1 : 0;
+}
+
+void AnimQueue::mutated() {
+    if (version_autoinc_) {
+        version_number_ += 1;
+    } else {
+        version_number_ = 0;
+    }
+}
+
+bool AnimQueue::size_and_steps_equal(const AnimQueue &other) const {
+    if (size_limit_ != other.size_limit_) {
+        return false;
+    }
+    if (steps_.size() != other.steps_.size()) {
+        return false;
+    }
+    for (int i = 0; i < int(steps_.size()); i++) {
+        if (!steps_[i].exactly_equal(other.steps_[i])) {
+            return false;
+        }
+    }
+    return true;
+}
+
+void AnimQueue::full_clear_and_set_limit(int n) {
+    assert(n >= 1);
+    steps_.clear();
+    steps_.emplace_back();
+    steps_.front().keep_state_only();
+    size_limit_ = n;
+    version_number_ = version_autoinc_ ? 1 : 0;
+}
+
+void AnimQueue::clear(const eng::string &plane) {
+    steps_.clear();
+    steps_.emplace_back();
+    steps_.front().set_plane(plane);
+    steps_.front().keep_state_only();
+    mutated();
+}
+
+void AnimQueue::set_limit(int n) {
+    assert(n >= 1);
+    size_limit_ = n;
+    if (int(steps_.size()) > n) {
+        while (int(steps_.size()) > n) {
+            steps_.pop_front();
+        }
+        steps_.front().keep_state_only();
+    }
+    mutated();
+}
+
+void AnimQueue::add(int64_t id, const AnimStep &step) {
+    AnimStep copy = step;
+    copy.echo(steps_.back());
+    copy.id_ = id;
+    steps_.push_back(copy);
+    while (int(steps_.size()) > size_limit_) {
+        steps_.pop_front();
+    }
+    steps_.front().keep_state_only();
+    mutated();
+}
+
+bool AnimQueue::valid() const {
+    // Size limit must be between 2 and 250
+    if ((size_limit_ < 1) || (size_limit_ > 250)) {
+        return false;
+    }
+    // Animqueue must have at least one step, and no more than 250.
+    if (steps_.empty()) {
+        return false;
+    }
+    if (steps_.size() > 250) {
+        return false;
+    }
+    // First action should be blank.
+    if (steps_[0].action_ != "") {
+        return false;
+    }
+    // First step should have all bits.
+    if (steps_[0].bits_ != AnimStep::HAS_EVERYTHING) {
+        return false;
+    }
+    // Any unset bit should correspond to a value copied from the previous step.
+    for (size_t i = 1; i < steps_.size(); i++) {
+        const AnimStep &prev = steps_[i - 1];
+        const AnimStep &curr = steps_[i];
+        if (!curr.echoes(prev)) return false;
+    }
+    return true;
+}
+
+eng::string AnimQueue::steps_debug_string() const {
+    eng::ostringstream oss;
+    for (int i = 0; i < int(size()); i++) {
+        oss << nth(i).debug_string() << "; ";
+    }
+    return oss.str();
+}
+
+eng::string AnimQueue::full_debug_string() const {
+    eng::ostringstream oss;
+    for (int i = 0; i < int(size()); i++) {
+        oss << nth(i).debug_string() << "; ";
+    }
+    oss << "version=" << version_number();
+    oss << "; limit=" << size_limit();
+    return oss.str();
+}
+
+void AnimQueue::serialize(StreamBuffer *sb) const {
+    assert(valid()); // can't serialize an invalid animqueue.
+    sb->write_uint8(size_limit_);
+    sb->write_uint8(steps_.size());
+    for (const AnimStep &step : steps_) {
+        step.write_into(sb);
+    }
+}
+
+void AnimQueue::deserialize(StreamBuffer *sb) {
+    size_limit_ = sb->read_uint8();
+    size_t nsteps = sb->read_uint8();
+    steps_.resize(nsteps);
+    for (size_t i = 0; i < nsteps; i++) {
+        AnimStep &step = steps_[i];
+        step.read_from(sb);
+        if (i > 0) step.echo(steps_[i - 1]);
+    }
+    version_number_ = 0;
+}
+
+bool AnimQueue::version_identical(const AnimQueue &auth) const {
+    return version_number_ == auth.version_number_;
+}
+
+bool AnimQueue::diff(const AnimQueue &auth, StreamBuffer *sb) const {
+    assert(valid());
+    assert(auth.valid());
+
+    // Fast path to equivalence detection
+    if (version_identical(auth)) {
+        assert(size_and_steps_equal(auth));
+        sb->write_uint8(255);
+        return false;
+    }
+
+    // Another path to equivalence detection
+    if (size_and_steps_equal(auth)) {
+        sb->write_uint8(255);
+        return false;
+    }
+
+    // Write the first element.
+    assert(auth.steps_.size() < 255);
+    sb->write_uint8(auth.steps_.size());
+    sb->write_uint8(auth.size_limit_);
+    const AnimStep &first = auth.steps_[0];
+    int match = 0;
+    while ((match < int(steps_.size())) && (!steps_[match].state_equal(first))) {
+        match++;
+    }
+    if (match == int(steps_.size())) {
+        sb->write_uint8(255);
+        first.write_into(sb);
+    } else {
+        sb->write_uint8(match);
+    }
+
+    // Index the remaining elements by id.
+    eng::map<uint64_t, int> index;
+    for (int i = 1; i < int(steps_.size()); i++) {
+        index[steps_[i].id_] = i;
+    }
+
+    // Write the remaining elements.
+    for (int i = 1; i < int(auth.steps_.size()); i++) {
+        const AnimStep &step = auth.steps_[i];
+        auto iter = index.find(step.id());
+        if (iter == index.end()) {
+            sb->write_uint8(255);
+            step.write_into(sb);
+        } else {
+            const AnimStep &local = steps_[iter->second];
+            if (local.exactly_equal(step)) {
+                sb->write_uint8(iter->second);
+            } else {
+                sb->write_uint8(255);
+                step.write_into(sb);
+            }
+        }
+    }
+    return true;
+}
+
+void AnimQueue::patch(StreamBuffer *sb, DebugCollector *dbc) {
+    int len = sb->read_uint8();
+    if (len == 255) {
+        return;
+    }
+    DebugLine(dbc) << "AnimQueue modified";
+
+    size_limit_ = sb->read_uint8();
+    // Decode the diff, stop at eof.
+    eng::deque<AnimStep> old = std::move(steps_);
+    steps_.clear();
+    for (int i = 0; i < len; i++) {
+        uint8_t index = sb->read_uint8();
+        if (index < 255) {
+            assert(index < old.size());
+            steps_.push_back(old[index]);
+        } else {
+            AnimStep step;
+            step.read_from(sb);
+            steps_.push_back(step);
+        }
+        int size = steps_.size();
+        if (size > 1) {
+            steps_[size-1].echo(steps_[size-2]);
+        } else {
+            steps_[0].keep_state_only();
+        }
+    }
+    mutated();
+}
+
+void AnimQueue::update_version(const AnimQueue &auth) {
+    assert(size_and_steps_equal(auth));
+    version_number_ = auth.version_number_;
+}
+
+const AnimStep &AnimQueue::back() const {
+    return steps_.back();
+}
+
+static bool diff_works(const AnimQueue &master, AnimQueue &sync) {
+    StreamBuffer sb;
+    sync.diff(master, &sb);
+    sync.patch(&sb, nullptr);
+    return sync.size_and_steps_equal(master);
+}
+
+LuaDefine(unittests_animqueue, "", "some unit tests") {
+    // Useful objects.
+    AnimStep stp;
+    AnimQueue aq(util::WORLD_TYPE_MASTER);
+    AnimQueue aqds(util::WORLD_TYPE_S_SYNC);
+    StreamBuffer sb;
+    
+    // Debug string of a newly initialized queue
+    LuaAssert(L, aq.valid());
+    LuaAssertStrEq(L, aq.full_debug_string(), "id=0 action= plane= x=0 y=0 z=0 facing=0 graphic=; version=1; limit=10");
+
+    // Test the step setters.
+    stp.clear();
+    LuaAssertStrEq(L, stp.debug_string(), "id=0 action=");
+    stp.set_facing(180);
+    LuaAssertStrEq(L, stp.debug_string(), "id=0 action= facing=180");
+    stp.set_x(3);
+    LuaAssertStrEq(L, stp.debug_string(), "id=0 action= x=3 facing=180");
+    stp.set_y(4);
+    LuaAssertStrEq(L, stp.debug_string(), "id=0 action= x=3 y=4 facing=180");
+    stp.set_z(5);
+    LuaAssertStrEq(L, stp.debug_string(), "id=0 action= x=3 y=4 z=5 facing=180");
+    stp.set_plane("somewhere");
+    LuaAssertStrEq(L, stp.debug_string(), "id=0 action= plane=somewhere x=3 y=4 z=5 facing=180");
+    stp.set_graphic("something");
+    LuaAssertStrEq(L, stp.debug_string(), "id=0 action= plane=somewhere x=3 y=4 z=5 facing=180 graphic=something");
+
+    // Test the step debug string parser.
+    LuaAssert(L, stp.from_string("id=123 action=walk x=1 y=2 z=3 facing=4 plane=p graphic=g"));
+    LuaAssertStrEq(L, stp.debug_string(), "id=123 action=walk plane=p x=1 y=2 z=3 facing=4 graphic=g");
+
+    // Test that we can clear a queue.
+    aq.full_clear_and_set_limit(3);
+    LuaAssert(L, aq.valid());
+    LuaAssertStrEq(L, aq.full_debug_string(), "id=0 action= plane= x=0 y=0 z=0 facing=0 graphic=; version=1; limit=3");
+    
+    // Add a step to a queue.
+    aq.full_clear_and_set_limit(3);
+    LuaAssert(L, stp.from_string("action=walk"));
+    aq.add(12345, stp);
+    LuaAssertStrEq(L, aq.full_debug_string(),
+        "id=0 action= plane= x=0 y=0 z=0 facing=0 graphic=; "
+        "id=12345 action=walk; "
+        "version=2; limit=3");
+
+    // Exceed the length limit, dropping first element.
+    aq.full_clear_and_set_limit(3);
+    LuaAssert(L, stp.from_string("action=walk plane=foo"));
+    aq.add(12345, stp);
+    aq.add(12346, stp);
+    aq.add(12347, stp);
+    LuaAssertStrEq(L, aq.full_debug_string(),
+        "id=0 action= plane=foo x=0 y=0 z=0 facing=0 graphic=; "
+        "id=12346 action=walk plane=foo; "
+        "id=12347 action=walk plane=foo; "
+        "version=4; limit=3"
+    );
+
+    // Test serialization and deserialization.
+    aq.full_clear_and_set_limit(5);
+    LuaAssert(L, stp.from_string("action=walk x=3 y=4 z=5"));
+    aq.add(12345, stp);
+    LuaAssert(L, stp.from_string("action=setgraphic graphic=banana"));
+    aq.add(12346, stp);
+    LuaAssert(L, stp.from_string("action=setfacing facing=301"));
+    aq.add(12347, stp);
+
+    aq.serialize(&sb);
+    aqds.deserialize(&sb);
+
+    LuaAssertStrEq(L, aqds.full_debug_string(), 
+        "id=0 action= plane= x=0 y=0 z=0 facing=0 graphic=; "
+        "id=12345 action=walk x=3 y=4 z=5; "
+        "id=12346 action=setgraphic graphic=banana; "
+        "id=12347 action=setfacing facing=301; "
+        "version=0; limit=5"
+    );
+    
+    // Test difference transmission
+    aq.full_clear_and_set_limit(10);
+    aqds.full_clear_and_set_limit(10);
+    
+    // Add a single action to the queue and DT
+    LuaAssert(L, stp.from_string("action=walk x=3 y=4 z=5"));
+    aq.add(12345, stp);
+    LuaAssert(L, diff_works(aq, aqds));
+
+    // Add another action and DT
+    LuaAssert(L, stp.from_string("action=fnord plane=where facing=123"));
+    aq.add(232, stp);
+    LuaAssert(L, diff_works(aq, aqds));
+
+    // Change the queue size limit.
+    aq.set_limit(13);
+    LuaAssert(L, diff_works(aq, aqds));
+    
+    // Discard all but the last action.
+    aq.set_limit(1);
+    LuaAssert(L, diff_works(aq, aqds));
+
+    return 0;
+}

luprex/cpp/core/animqueue.hpp

@@ -0,0 +1,241 @@
+///////////////////////////////////////////////////////////////////
+//
+// ANIMATION QUEUES
+//
+// An animation queue is a fifo queue of animation steps.  New animations are
+// pushed on the back, and old ones are popped from the front.
+//
+// An animation step has an "action" which is usually the name of an animation,
+// or it's a special token like "walk" or "warp." Each animation step shows the
+// resulting AnimState that the player finds himself in after executing the
+// action.
+//
+// The first step in an animation queue always has id=0 and action="". This step
+// represents the initial state of the sprite before any animations or
+// movements.
+//
+// To add new items to the AnimQueue, use this process: first, call add(id,
+// action).  This adds a new step to the queue.  Then, call set_xyz, set_facing,
+// set_plane, or an other setter. These setters are meant to only be used
+// immediately after calling 'add' to populate the new step. 
+//
+// VERSION NUMBERS
+//
+// The version number field: if the version number in the synchronous model is
+// equal to the version number in the master model, then the two animqueues are
+// guaranteed to be equal. Here's how we achieve that invariant:
+//
+// * In the master model, the version number starts at 1 and is auto-incremented
+//   every time the animation queue is mutated.
+//
+// * In the synchronous model, the version number is set to zero every time the
+//   animation queue is mutated.  Note that master version numbers are never
+//   zero. Applying a patch also sets the version number to zero.
+//
+// * Serializing and deserializing causes the version number to be saved and
+//   restored in both master and synchronous models.
+//
+// * The routine 'update_version' should be called after difference
+//   transmission.  This copies the version number from the master to the
+//   synchronous model.  This is guaranteed to be safe because we just finished
+//   difference transmission, therefore, the queues should match.
+//
+///////////////////////////////////////////////////////////////////
+
+#ifndef ANIMQUEUE_HPP
+#define ANIMQUEUE_HPP
+
+#include "wrap-set.hpp"
+#include "wrap-string.hpp"
+#include "wrap-deque.hpp"
+#include "wrap-unordered-map.hpp"
+
+#include "streambuffer.hpp"
+#include "debugcollector.hpp"
+#include "util.hpp"
+
+#include <cassert>
+#include <ostream>
+
+class AnimStep : public eng::nevernew {
+    friend class AnimQueue;
+public:
+    enum {
+        HAS_FACING = 1,
+        HAS_X = 2,
+        HAS_Y = 4,
+        HAS_Z = 8,
+        HAS_XYZ = 14,
+        HAS_GRAPHIC = 16,
+        HAS_PLANE = 32,
+        HAS_EVERYTHING = 63,
+    };
+
+    AnimStep();
+    ~AnimStep();
+
+    int64_t id() const { return id_; }
+    int bits() const { return bits_; }
+
+    const eng::string &action() const { return action_; }
+    double facing() const { return facing_; }
+    float x() const { return xyz_.x; }
+    float y() const { return xyz_.y; }
+    float z() const { return xyz_.z; }
+    const util::XYZ &xyz() const { return xyz_; }
+    const eng::string &graphic() const { return graphic_; }
+    const eng::string &plane() const { return plane_; }
+
+    bool has_facing() const { return bits_ & HAS_FACING; }
+    bool has_x() const { return bits_ & HAS_X; }
+    bool has_y() const { return bits_ & HAS_Y; }
+    bool has_z() const { return bits_ & HAS_Z; }
+    bool has_xyz() const { return (bits_ & HAS_XYZ) == HAS_XYZ; }
+    bool has_graphic() const { return bits_ & AnimStep::HAS_GRAPHIC; }
+    bool has_plane() const { return bits_ & AnimStep::HAS_PLANE; }
+
+    void set_action(const eng::string &action);
+    void set_facing(float f);
+    void set_x(float f);
+    void set_y(float f);
+    void set_z(float z);
+    void set_xyz(const util::XYZ &xyz);
+    void set_graphic(const eng::string &g);
+    void set_plane(const eng::string &p);
+
+    void clear();
+
+    // ExactlyEqual compares all fields.
+    bool exactly_equal(const AnimStep &other) const;
+
+    // LogicallyEqual only compares fields whose HAS_XXX bits are set.
+    bool logically_equal(const AnimStep &other) const;
+
+    // StateEqual is true if the plane, graphic, facing, and xyz match.
+    bool state_equal(const AnimStep &other) const;
+
+    // read/write the step using a stream buffer.
+    //
+    void write_into(StreamBuffer *sb) const;
+    void read_from(StreamBuffer *sb);
+
+    // Create an AnimStep from a lua table.
+    //
+    // Lua stack must contain a table, which may contain:
+    //   action: "action"
+    //   facing: 0.0 - 360.0
+    //   x: x-coordinate
+    //   y: y-coordinate
+    //   z: z-coordinate
+    //   graphic: "graphic"
+    //   plane: "plane"
+    //
+    // aqback: the animation queue back, from which relative
+    // moves are computed.
+    //
+    void configure(LuaKeywordParser &kp, const AnimStep &aqback);
+
+    // Make this step into a first-step of an anim queue.
+    void keep_state_only();
+    
+    // For any values that are unchanged in this step,
+    // echo the values of the previous step.
+    void echo(const AnimStep &prev);
+
+    // Verify that this step echoes the previous step.
+    bool echoes(const AnimStep &prev) const;
+
+    // Convert to a string for debugging purposes.
+    eng::string debug_string() const;
+
+    // Convert a string to an animstep (for testing only).
+    bool from_string(const eng::string &s);
+
+private:
+    int64_t id_;
+    int16_t bits_;
+    eng::string action_;
+
+    float facing_;
+    util::XYZ xyz_;
+    eng::string graphic_;
+    eng::string plane_;
+
+    void config_store_string(lua_State *L, int idx, eng::string *target, int16_t bits, const char *name);
+    void config_store_number(lua_State *L, int idx, float *target, float offset, int16_t bits, const char *name);    
+};
+
+
+class AnimQueue : public eng::nevernew {
+public:
+    // World type determines whether versions increment or autozero
+    AnimQueue(util::WorldType wt);
+        
+    // Simple getters.
+    const AnimStep &nth(int n) const { return steps_[n]; }
+    size_t size() const { return steps_.size(); }
+    int32_t size_limit() const { return size_limit_; }
+    int64_t version_number() const { return version_number_; }
+    bool version_identical(const AnimQueue &aq) const;
+
+    // Return true if the size limit and steps are identical.
+    bool size_and_steps_equal(const AnimQueue &aq) const;
+
+    // Mutator to create a new step.
+    void add(int64_t id, const AnimStep &step);
+    
+    // Clear and set the plane.
+    void clear(const eng::string &plane);
+    
+    // Serialize or deserialize to a StreamBuffer
+    //
+    // Caution: version numbers are not stored.  On deserialize,
+    // version number is set to zero.
+    //
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
+    // Difference transmission
+    bool diff(const AnimQueue &auth, StreamBuffer *sb) const;
+    void patch(StreamBuffer *sb, DebugCollector *dbc);
+    void update_version(const AnimQueue &auth);
+    
+    // Get the final resting place after all animations are complete.
+    const AnimStep &back() const;
+
+public:
+    ////////////////////////////////////////////////////////////////////////////
+    //
+    // TESTING SUPPORT
+    //
+    // The following functions are not designed to be useful for production
+    // code, they're designed to be helpful for unit testing.
+    //
+    ////////////////////////////////////////////////////////////////////////////
+
+    // Change the size limit.
+    void full_clear_and_set_limit(int szl);
+    void set_limit(int szl);
+    
+    // Make sure the invariants are preserved.
+    bool valid() const;
+
+    // Convert to a string for debugging purposes.
+    eng::string steps_debug_string() const;
+    eng::string full_debug_string() const;
+    
+    // Convert to a 
+
+private:
+    bool version_autoinc_;
+    int32_t size_limit_;
+    eng::deque<AnimStep> steps_;
+    int64_t version_number_;
+
+    // Called whenever the animation queue is mutated.
+    // serialization and deserialization
+    void mutated();
+};
+
+#endif // ANIMQUEUE_HPP
+

luprex/cpp/core/debugcollector.cpp

@@ -0,0 +1,89 @@
+
+
+#include <cstring>
+#include <algorithm>
+
+#include "debugcollector.hpp"
+#include "util.hpp"
+
+void DebugCollector::flush() {
+    if (need_flush_) {
+        eng::string str = oss_.str();
+        if (!str.empty()) {
+            if (is_regular_) n_regular_ += 1;
+            lines_.push_back(str);
+            oss_.str("");
+        }
+        need_flush_ = false;
+    }
+}
+
+DebugCollector::DebugCollector() : n_regular_(0), need_flush_(false), is_regular_(true), active_(false) {
+}
+
+DebugCollector::DebugCollector(const eng::string &targets) 
+    : n_regular_(0), need_flush_(false), is_regular_(true), active_(false) {
+    targets_ = util::split(targets, ',');
+    std::sort(targets_.begin(), targets_.end());
+}
+
+bool DebugCollector::use(const char *target) {
+    int lo = 0;
+    int hi = targets_.size();
+    while (hi > lo) {
+        int mid = (hi + lo) >> 1;
+        int cmp = strcmp(targets_[mid].c_str(), target);
+        if (cmp == 0) return true;
+        if (cmp > 0) hi = mid;
+        else lo = mid + 1;
+    }
+    return false;
+}
+
+bool DebugCollector::do_line() {
+    if (active_) {
+        flush();
+        need_flush_ = true;
+        is_regular_ = true;
+        return true;
+    } else return false;
+}
+
+bool DebugCollector::do_header() {
+    flush();
+    need_flush_ = true;
+    is_regular_ = false;
+    return true;
+}
+
+void DebugCollector::dump(std::ostream &os) {
+    flush();
+    for (const eng::string &line : lines_) {
+        os << line << std::endl;
+    }
+}
+
+DebugBlock::DebugBlock(DebugCollector *dbc, const char *target) {
+    dbc_ = dbc;
+    if (dbc) {
+        n_regular_ = dbc->n_regular_;
+        n_lines_ = dbc->lines_.size();
+        active_ = (!dbc->active_) && (dbc->use(target));
+        if (active_) dbc_->active_ = true;
+    }
+}
+
+DebugBlock::~DebugBlock() {
+    if (dbc_) {
+        dbc_->flush();
+        // If no regular lines have been added,
+        // remove all header lines that were added.
+        if (dbc_->n_regular_ == n_regular_) {
+            if (dbc_->lines_.size() != n_lines_) {
+                dbc_->lines_.resize(n_lines_);
+            }
+        }
+        if (active_) dbc_->active_ = false;
+    }
+}
+

luprex/cpp/core/debugcollector.hpp

@@ -0,0 +1,51 @@
+#ifndef DEBUGCOLLECTOR_HPP
+#define DEBUGCOLLECTOR_HPP
+
+#include "wrap-vector.hpp"
+#include "wrap-string.hpp"
+#include "wrap-sstream.hpp"
+
+#include <ostream>
+#include <memory>
+
+class DebugCollector : public eng::nevernew {
+private:
+    // At any given time, the stringstream may contain one
+    // extra line that hasn't yet been appended to the list of lines.
+    // The flag 'need_flush_' is true if the stringstream contains
+    // an extra line.  In that case, is_regular_ indicates whether
+    // the extra line is a header or a regular line.
+    eng::vector<eng::string> lines_;
+    eng::vector<eng::string> targets_;
+    int n_regular_;
+    bool need_flush_;
+    bool is_regular_;
+    bool active_;
+    void flush();
+    bool use(const char *target);
+public:
+    eng::ostringstream oss_;
+    DebugCollector();
+    DebugCollector(const eng::string &targets);
+    bool do_header();
+    bool do_line();
+    void dump(std::ostream &os);
+    friend class DebugBlock;
+};
+
+class DebugBlock : public eng::nevernew {
+private:
+    DebugCollector *dbc_;
+    int n_regular_;
+    size_t n_lines_;
+    bool active_;
+public:
+    DebugBlock(DebugCollector *dbc, const char *target);
+    ~DebugBlock();
+};
+
+#define DebugHeader(dbc) if (((dbc)!=nullptr) && (dbc)->do_header()) ((dbc)->oss_)
+#define DebugLine(dbc) if (((dbc)!=nullptr) && (dbc)->do_line()) ((dbc)->oss_)
+
+#endif // DEBUGCOLLECTOR_HPP
+

luprex/cpp/core/drivenengine.hpp

@@ -0,0 +1,345 @@
+//////////////////////////////////////////////////////////////
+//
+// DrivenEngine
+//
+// This module embodies the idea of an "event-driven game engine." The
+// DrivenEngine module provides two APIs: the engine-side API, and the
+// driver-side API.
+//
+// The engine-side API looks like a typical collection of I/O primitives. It
+// includes methods to open sockets, read and write sockets, read lua source,
+// get the clock, and so forth.
+//
+// But in reality, these I/O functions don't ever call operating system
+// functions like "read" or "write" or "connect."  They don't call the operating
+// system at all - not even indirectly, through a wrapper.  Therefore, they
+// can't really do any I/O.  When you use one of these I/O functions to (say)
+// write some data to a communication channel, the only thing that happens is
+// that the data is put into a buffer.  The actual transmission of the data
+// happens elsewhere, in what is called the "Driver."  Likewise, when you use
+// one of these I/O functions to read data, it only returns data that was
+// previously stored by the "Driver."
+//
+// The "Driver" is a module that implements the actual I/O.  It is highly
+// OS-dependent code, because it contains code to manipulate sockets, time
+// clocks, and the like.
+//
+// From the perspective of the driver, the DrivenEngine is a C++ object that
+// acts like a state machine.  This state machine is driven forward by I/O
+// events. The DrivenEngine provides an API where the driver can feed in these
+// I/O events.
+//
+// Notice that the usual call graph is inverted: in most application programs,
+// the application calls the operating system to do I/O.  But when using class
+// DrivenEngine, it's the other way around: the driver calls into class
+// DrivenEngine to drive it forward.  I/O routines drive computation. 
+//
+// So the upshot of all this is that the DrivenEngine is a deterministic state
+// machine, free of all OS-specific code.
+//
+//////////////////////////////////////////////////////////////
+
+#ifndef DRIVENENGINE_HPP
+#define DRIVENENGINE_HPP
+
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+
+#include <ostream>
+#include <memory>
+#include <string_view>
+
+#include "util.hpp"
+#include "streambuffer.hpp"
+#include "enginewrapper.hpp"
+
+class DrivenEngine;
+using UniqueDrivenEngine = std::unique_ptr<DrivenEngine>;
+using DrivenEngineMaker = UniqueDrivenEngine (*)();
+using DrivenEngineInitializer = void (*)();
+
+class Channel : public eng::opnew {
+public:
+    // Get the buffers associated with this channel.
+    //
+    StreamBuffer *out() { return sb_out_.get(); }
+    StreamBuffer *in() { return sb_in_.get(); }
+
+    // The channel ID.  These are reused.
+    //
+    int chid() const { return chid_; }
+
+    // If this is a socket connection, the receiver's port number.
+    //
+    int port() const { return port_; }
+
+    // If this is an outgoing socket connection, get the target host.
+    //
+    const eng::string &target() const { return target_; }
+
+    // True if the remote closed the connection, or a failure occurred.
+    //
+    bool closed() const { return closed_; }
+
+    // Get the channel's error message.
+    //
+    // If this is an empty string, there is no error.  If this is set,
+    // then the channel is also closed.
+    //
+    const eng::string &error() const { return error_; }
+
+    // Set the prompt for readline mode.
+    //
+    void set_prompt(const eng::string &prompt);
+
+    // Do not construct your own Channels.  Instead,
+    // use methods of class DrivenEngine like new_outgoing_channel.
+    // Channels are referenced by shared_ptr.  You can
+    // release your shared_ptr at any time.
+    //
+    Channel(DrivenEngine *de, int chid, int port, const eng::string &target, bool stop);
+    ~Channel() {};
+    
+private:
+    // Constructor is deliberately private.  Use
+    // DrivenEngine::new_outgoing_channel to create outgoing socket channels.
+    //
+
+    void feed_readline(std::string_view data);
+    std::string_view peek_outgoing() const;
+    void sent_outgoing(int nbytes);
+    void erase_command();
+    void echo_command();
+    void pump_readline();
+
+private:
+    static const int READLINE_MAX=512;
+    int chid_;
+
+    // These are the in/out buffers presented to the user.
+    std::shared_ptr<StreamBuffer> sb_in_;
+    std::shared_ptr<StreamBuffer> sb_out_;
+
+    // If this is stdio, we inject tty echoes into the output stream.
+    // This buffer holds the users output interleaved with the tty echoes.
+    // In any other channel, this is just another pointer to sb_out.
+    std::shared_ptr<StreamBuffer> sb_drvout_;
+
+    int port_;
+    bool closed_;
+    eng::string error_;
+    eng::string target_;
+    bool stop_driver_;
+
+    // Readline stuff.  Only used on channel 0 (stdio).
+    eng::string desired_command_;
+    eng::string current_command_;
+    eng::string desired_prompt_;
+    eng::string current_prompt_;
+    char readline_lastc_;
+
+    friend class DrivenEngine;
+};
+
+using SharedChannel = std::shared_ptr<Channel>;
+
+class DrivenEngine : public eng::opnew {
+public:
+    //////////////////////////////////////////////////////////////
+    //
+    // Build the named engine
+    //
+    //////////////////////////////////////////////////////////////
+
+    static UniqueDrivenEngine make(std::string_view name);
+
+    static void print_usage(std::ostream &strm, std::string_view progname);
+
+    //////////////////////////////////////////////////////////////
+    //
+    // The following methods are the 'engine' side of the pipe.
+    //
+    //////////////////////////////////////////////////////////////
+
+    // The init callback.  You may override this in a subclass.
+    // This will be called once at program initialization.
+    //
+    virtual void event_init(int argc, char *argv[]) {}
+
+    // The update callback.  You may override this in a subclass.
+    // This will be called whenever anything changes.
+    //
+    virtual void event_update() {}
+
+    // Specify the set of listening ports.
+    // This can only be used during the init routine.
+    //
+    void listen_port(int port);
+
+    // Get the current time.
+    //
+    // DRIVER: This returns the time most recently stored by the driver
+    // using drv_set_clock. 
+    //
+    double get_clock();
+
+    // Create a channel and open an outgoing connection.  The channel creation
+    // always succeeds.  You can write to the channel immediately.  You can
+    // read, too, but of course there won't be anything in the incoming buffer
+    // yet.  In future update events, data will show up in the incoming buffer,
+    // and will have been sent from the outgoing buffer.  In future update
+    // events, the channel may get closed by the remote.  If the connection
+    // fails (say, the remote host doesn't exist), then the Channel will get
+    // closed with an error.
+    //
+    // DRIVER: The channel object is created instantly, but it does nothing
+    // until the driver notices the new channel.  The driver is responsible for
+    // actually opening the connection and relaying data into the channel using
+    // drv_get_target, drv_peek_outgoing, drv_sent_outgoing, drv_recv_incoming.
+    //
+    SharedChannel new_outgoing_channel(const eng::string &target);
+
+    // Create a new channel from any pending incoming connection. If there is no
+    // incoming connection, returns nullptr.
+    //
+    // DRIVER: The driver must be hardwired to know what ports to listen on.
+    // When the driver notices a new incoming connection, it calls
+    // drv_notify_accept, which triggers the creation of the channel.  The
+    // channel is put into the incoming channel queue, which is fetched by this
+    // method. The driver is responsible for relaying data into the channel
+    // using drv_get_target, drv_peek_outgoing, drv_sent_outgoing,
+    // drv_recv_incoming.
+    //
+    SharedChannel new_incoming_channel();
+
+    // Obtain the stdio channel. There is only one stdio channel.
+    //
+    // DRIVER: the stdio channel is created automatically when the DrivenEngine
+    // is created.  The driver is responsible for relaying data into the channel
+    // using drv_get_target, drv_peek_outgoing, drv_sent_outgoing,
+    // drv_recv_incoming.
+    //
+    SharedChannel get_stdio_channel();
+
+    // Obtain the output buffer of the stdio channel as an ostream.
+    //
+    std::ostream &stdostream() { return get_stdio_channel()->out()->ostream(); }
+
+    // Fetches the lua source, and takes ownership of it.  The DrivenEngine
+    // no longer contains the source after calling this.
+    //
+    util::LuaSourcePtr get_lua_source();
+
+    // Rescan the lua source directory. The lua source directory is read once,
+    // automatically, at engine creation time.  If you want to read it again,
+    // you must trigger a rescan.  The rescan is not instantaneous.
+    //
+    // DRIVER: this merely sets a flag, which the driver will notice later,
+    // causing the driver to update the lua source.
+    //
+    void rescan_lua_source();
+
+    // Stop the driver.  The engine should call this when it's done
+    // and there's nothing left to do.
+    //
+    void stop_driver();
+
+    //////////////////////////////////////////////////////////////
+    //
+    // Creation and Destruction.
+    //
+    //////////////////////////////////////////////////////////////
+
+    // Constructor.
+    //
+    // Most initialization is achieved by 'drv_xxx' functions, so
+    // this constructor takes no arguments.
+    //
+    DrivenEngine();
+
+    // Destructor.
+    //
+    // It is necessary to delete all channels before deleting the
+    // DrivenEngine.  The destructor will verify that this has been done.
+    //
+    virtual ~DrivenEngine();
+
+    //////////////////////////////////////////////////////////////
+    //
+    // The following accessors are for use by PlayWrapper and ReplayWrapper.
+    //
+    // The PlayWrapper and ReplayWrapper use C stubs to access
+    // the engine.  The C stubs, in turn, call these C++ methods.
+    //
+    // The stubs for the getters are trivial, one-line stubs.
+    //
+    // The stubs for the mutators add logging.
+    //
+    //////////////////////////////////////////////////////////////
+
+    void drv_get_listen_ports(uint32_t *nports, const uint32_t **ports) const;
+    void drv_get_new_outgoing(uint32_t *nchids, const uint32_t **chids) const;
+    const char *drv_get_target(uint32_t chid) const;
+    bool drv_get_channel_released(uint32_t chid) const;
+    void drv_get_outgoing(uint32_t chid, uint32_t *len, const char **data) const;
+    bool drv_get_outgoing_empty(uint32_t chid) const;
+    double drv_get_clock() const;
+    bool drv_get_rescan_lua_source() const;
+    bool drv_get_stop_driver() const;
+
+    void drv_initialize(uint32_t srcpklen, const char *srcpk, int argc, char **argv);
+    void drv_clear_new_outgoing();
+    void drv_sent_outgoing(uint32_t chid, uint32_t nbytes);
+    void drv_recv_incoming(uint32_t chid, uint32_t nbytes, const char *bytes);
+    void drv_notify_close(uint32_t chid, uint32_t len, const char *data);
+    uint32_t drv_notify_accept(uint32_t port);
+    void drv_invoke_event_update(double clock);
+    void drv_set_lua_source(uint32_t srcpklen, const char *srcpk);
+    
+private:
+    // Find a currently-unused channel ID.  Channel IDs
+    // are small integers that are reused.
+    int find_unused_chid();
+
+    // Get the channel associated with the specified channel ID.
+    Channel *get_chid(int chid) const;
+    
+private:
+    SharedChannel channels_[DRV_MAX_CHAN];
+    int next_unused_chid_;
+    SharedChannel stdio_channel_;
+    eng::vector<SharedChannel> accepted_channels_;
+    eng::vector<uint32_t> new_outgoing_;
+    util::LuaSourcePtr lua_source_;
+    eng::vector<uint32_t> listen_ports_;
+    bool rescan_lua_source_;
+    double clock_;
+    bool stop_driver_;
+    friend class Channel;
+};
+
+
+//////////////////////////////////////////////////////////////////////////////////
+
+
+struct DrivenEngineReg {
+    const char *name;
+    DrivenEngineMaker maker;
+    DrivenEngineReg *next;
+    static DrivenEngineReg *All;
+    DrivenEngineReg(const char *name, DrivenEngineMaker f);
+};
+
+#define DrivenEngineDefine(name, cname) \
+    UniqueDrivenEngine dengmake_##cname() { \
+        return UniqueDrivenEngine(new cname); \
+    } \
+    DrivenEngineReg dengreg_##cname(name, dengmake_##cname);
+
+struct DrivenEngineInitializerReg {
+    static DrivenEngineInitializer func;
+    DrivenEngineInitializerReg(DrivenEngineInitializer f);
+};
+
+
+#endif // DRIVENENGINE_HPP

luprex/cpp/core/eng-malloc.cpp

@@ -0,0 +1,122 @@
+
+// We only use a custom allocator on linux (for now)
+// The allocator we chose is 'dlmalloc' (doug lea's malloc).
+// It's a good allocator for single-threaded programs.
+// It needs to be configured by setting a bunch of defines.
+//
+#ifdef __linux__
+
+// We need this to define dlmalloc, not malloc.
+#define USE_DL_PREFIX 1
+
+// We don't need mspaces.
+#define ONLY_MSPACES 0
+#define MSPACES 0
+
+// We don't need mallinfo
+#define NO_MALLINFO 1
+
+// We don't need dlmalloc_inspect_all
+#define MALLOC_INSPECT_ALL 0
+
+// Disable locking.  The entire engine is single-threaded.
+#define USE_LOCKS 0
+
+// This allocator can use sbrk to obtain RAM.  We're going to
+// emulate sbrk, in order to put it in a different memory region
+// than the system malloc, which also uses sbrk.
+#define HAVE_MORECORE 1
+#define MORECORE emulated_sbrk
+#define MORECORE_CANNOT_TRIM 1
+
+// We won't let the memory allocator use mmap.  This is because
+// opening the replay log may be implemented in stdio using mmap.
+// So using mmap might trigger different results under replay.
+#define HAVE_MMAP 0
+
+// Fix a warning in dlmalloc.
+#define MAX_RELEASE_CHECK_RATE INT_MAX
+
+// Don't generate random seeds, or time-based seeds.
+#define USE_DEV_RANDOM 0
+#define LACKS_TIME_H 1
+
+// Don't export any dlmalloc functions.
+#define DLMALLOC_EXPORT static inline
+
+#include <sys/mman.h>
+#include <cstdint>
+#include <cassert>
+#include <climits>
+
+static char     *emulated_sbrk_base;
+static intptr_t  emulated_sbrk_used;
+static intptr_t  emulated_sbrk_limit;
+
+// We assume that the increments are already aligned to the system
+// page size, because dlmalloc does that for us.
+//
+// Our emulated sbrk cannot trim the memory size.  It can only grow.
+//
+static void *emulated_sbrk(intptr_t increment) {
+    if (emulated_sbrk_base == 0) {
+        emulated_sbrk_limit = intptr_t(64) * 1024 * 1024 * 1024;
+        void *map = mmap(nullptr, emulated_sbrk_limit, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        assert(map != MAP_FAILED);
+        assert(map != nullptr);
+        emulated_sbrk_base = (char *)map;
+        emulated_sbrk_used = 0;
+    }
+    int64_t old_used = emulated_sbrk_used;
+    int64_t new_used = emulated_sbrk_used + increment;
+    if (new_used > emulated_sbrk_limit) {
+        return (void *)(-1);
+    }
+    assert(new_used >= old_used);
+    emulated_sbrk_used = new_used;
+    if (new_used > old_used) {
+        int status = mprotect(emulated_sbrk_base + old_used, new_used - old_used, PROT_READ | PROT_WRITE);
+        assert(status == 0);
+    }
+    return emulated_sbrk_base + old_used;
+}
+
+#include "../../extern/dlmalloc.c"
+
+namespace eng {
+    static uint32_t hash;
+    void *malloc(size_t size) {
+        void *result = dlmalloc(size);
+        hash += uint32_t(uintptr_t(result)) + 0x83748374;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        hash += uint32_t(size) + 0x85893489;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        return result;
+    }
+    void *realloc(void *p, size_t size) {
+        void *result = dlrealloc(p, size);
+        hash += uint32_t(uintptr_t(p)) + 0x74741912;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        hash += uint32_t(uintptr_t(result)) + 0x68843823;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        hash += uint32_t(size) + 0x81444120;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        return result;
+    }
+    void  free(void *p) {
+        hash += uint32_t(uintptr_t(p)) + 0x87823448;
+        dlfree(p);
+    }
+    int memhash() {
+        return (hash & 0x3FFFFFFF) | 0x40000000;
+    }
+} // namespace eng
+
+#endif // ifdef __linux__
+
+

luprex/cpp/core/eng-malloc.hpp

@@ -0,0 +1,195 @@
+//
+// eng-malloc
+//
+// The engine has its own private eng::malloc which it uses to allocate
+// everything.  The engine's malloc heap only contains engine data structures.
+// This helps achieve determinism when playing a replay log.
+//
+// The engine's eng::malloc is a thin wrapper around Doug Lea's Malloc, a good
+// general-purpose single-threaded malloc.  It's probably not the fastest any
+// more (it was, once), but it's still quite good.  It's also fairly easy
+// to work with.
+//
+// In order to get all engine data structures into the eng::malloc heap, you
+// need to jump through quite a few hoops:
+//
+// * When writing a class, always derive from eng::opnew.  This adds a
+//   custom operator new to your class, which causes your class to be
+//   allocated using eng::malloc.
+//
+// * When using STL containers, you need to use the eng variant:
+//   eng::map, eng::set, eng::vector, eng::unordered_map, eng::unordered_set,
+//   and eng::deque.  These classes derive from eng::opnew, and they also
+//   use eng::malloc for their internal nodes.
+//
+// * Use eng::string instead of std::string.  Use eng::ostringstream
+//   instead of std::ostringstream.
+//
+// * Simple classes like std::pair, std::string_view, std::less, std::hash, and
+//   so forth are not wrapped.  Do not use operator new or delete on
+//   these classes.
+//
+// * Instead of std::make_shared, use eng::make_shared.  You need this
+//   because std::make_shared doesn't respect your custom operator new.
+//
+// * Be aware that most C++ streams use the system malloc heap, and there's no
+//   way to change that.   Fortunately, eng::ostringstream uses the eng::malloc
+//   heap.
+//
+// * Failing to jump through all these hoops won't break your code in any
+//   obvious way - you'll just have some of your data structures in the malloc
+//   heap instead of the eng::malloc heap.  This can break determinism of
+//   replay.
+//
+#ifndef ENG_MALLOC_HPP
+#define ENG_MALLOC_HPP
+
+#include <cstddef>
+#include <memory>
+#include <cassert>
+
+namespace eng {
+#ifdef __linux__
+void* malloc(size_t x);
+void  free(void *p);
+void* realloc(void*, size_t);
+int   memhash();
+#else
+inline void *malloc(size_t x) { return ::malloc(x); }
+inline void  free(void *p) { return ::free(p); }
+inline void *realloc(void *p, size_t x) { return ::realloc(p, x); }
+inline int   memhash() { return 0; }
+#endif
+} // namespace eng
+
+// An allocator for lua states that uses eng::malloc and eng::free
+namespace eng {
+    inline void *l_alloc(void *ud, void *ptr, size_t osize, size_t nsize) {
+        if (nsize == 0) {
+            ::eng::free(ptr);
+            return NULL;
+        } else {
+            return ::eng::realloc(ptr, nsize);
+        }
+    }
+} // namespace eng
+
+// eng_allocator is similar to std::allocator, but allocates
+// objects using eng::malloc and eng::free.
+template <class T>
+class eng_allocator
+{
+public:
+    using value_type    = T;
+    eng_allocator() noexcept {}
+    template <class U> eng_allocator(eng_allocator<U> const&) noexcept {}
+
+    value_type* allocate(std::size_t n)
+    {
+        return static_cast<value_type*>(eng::malloc(n*sizeof(value_type)));
+    }
+
+    void deallocate(value_type* p, std::size_t) noexcept
+    {
+        eng::free(p);
+    }
+};
+
+// Another name for eng_allocator is eng::allocator.
+namespace eng {
+    template<class T>
+    using allocator = ::eng_allocator<T>;
+} // namespace eng
+
+// Mandated equality and inequality operators for eng_allocator.
+template <class T, class U>
+bool operator==(const eng_allocator<T> &, const eng_allocator<U> &) noexcept
+{
+    return true;
+}
+template <class T, class U>
+bool operator!=(const eng_allocator<T> &, const eng_allocator<U> &) noexcept
+{
+    return false;
+}
+
+// eng::opnew.  A class containing operator new and operator delete,
+// meant to be used as a base class for inheritance.
+namespace eng {
+    class opnew {
+    public:
+        void *operator new(size_t size)
+        {
+            return ::eng::malloc(size);
+        }
+
+        void operator delete(void *p, size_t size)
+        {
+            return ::eng::free(p);
+        }
+
+        void *operator new[](size_t size)
+        {
+            return ::eng::malloc(size);
+        }
+
+        void operator delete[](void *p, size_t size)
+        {
+            return ::eng::free(p);
+        }
+    };
+} // namespace eng
+
+// eng::nevernew.  A class containing private operator new and
+// operator delete, making it impossible to 'new' the class.
+// This means the class must be embedded as a field in some other
+// class, and it gets allocated when its enclosing object gets
+// allocated.
+namespace eng {
+    class nevernew {
+    private:
+        void *operator new(size_t size)
+        {
+            assert(false && "not supposed to 'new' this class");
+            return NULL;
+        }
+
+        void operator delete(void *p, size_t size)
+        {
+            assert(false && "not supposed to 'delete' this class");
+        }
+
+        void *operator new[](size_t size)
+        {
+            assert(false && "not supposed to 'new' this class");
+            return NULL;
+        }
+
+        void operator delete[](void *p, size_t size)
+        {
+            assert(false && "not supposed to 'delete' this class");
+        }
+    };
+} // namespace eng
+
+
+// eng::make_shared allocates shared objects using eng::malloc.
+namespace eng {
+    template<class T, class... Args>
+    inline ::std::shared_ptr<T> make_shared(Args&&... args) {
+        return std::allocate_shared<T>(eng::allocator<T>(), args...);
+    }
+} // namespace eng
+
+// eng::make_unique doesn't do anything different than std::make_unique: they
+// both use operator new and delete.  You must derive from eng::opnew to change
+// operator new and delete.
+namespace eng {
+    template<class T, class... Args>
+    inline ::std::unique_ptr<T> make_unique(Args&&... args) {
+        return std::make_unique<T>(args...);
+    }
+} // namespace eng
+
+#endif // ENG_MALLOC_HPP
+

luprex/cpp/core/eng-tests.cpp

@@ -0,0 +1,129 @@
+#include "wrap-string.hpp"
+
+#include "drivenengine.hpp"
+#include "streambuffer.hpp"
+#include "world.hpp"
+
+#include <iomanip>
+
+static void write_closed_message(Channel *ch, StreamBuffer *out) {
+    eng::ostringstream oss;
+    oss << "Chan " << ch->chid() << " closed [" << ch->error() << "]\n";
+    out->write_bytes(oss.str());
+}
+
+static void dump_lines(StreamBuffer *in, StreamBuffer *out, int chid) {
+    while (true) {
+        eng::string l = in->readline();
+        if (l == "") break;
+        eng::ostringstream oss;
+        oss << "Chan " << chid << ": " << l;
+        out->write_bytes(oss.str());
+    }
+}
+
+// This test is the minimal possible DrivenEngine.
+class DriverStubTest : public DrivenEngine {
+    virtual void event_init(int argc, char *argv[]) {
+        stop_driver();
+    }
+};
+
+// This test connects to a public webserver and prints
+// the output from the server.
+class DriverWebServerTest : public DrivenEngine {
+public:
+    eng::vector<SharedChannel> channels_;
+    virtual void event_init(int argc, char *argv[]) {
+        SharedChannel ch = new_outgoing_channel("cert:stanford.edu:443");
+        ch->out()->write_bytes("GET https://stanford.edu/xbanankjdsh.html HTTP/1.1\n\n");
+        channels_.emplace_back(std::move(ch));
+    }
+    
+    virtual void event_update() {
+        SharedChannel stdioch = get_stdio_channel();
+        dump_lines(stdioch->in(), stdioch->out(), 0);
+        eng::vector<SharedChannel> keep;
+        for (SharedChannel &ch : channels_) {
+            dump_lines(ch->in(), stdioch->out(), ch->chid());
+            if (ch->closed()) {
+                write_closed_message(ch.get(), stdioch->out());
+            } else {
+                keep.emplace_back(std::move(ch));
+            }
+        }
+        channels_ = std::move(keep);
+    }
+};
+
+// This test produces a DNS resolution failure.
+class DriverDNSFailTest : public DrivenEngine {
+public:
+    eng::vector<SharedChannel> channels_;
+    virtual void event_init(int argc, char *argv[]) {
+        SharedChannel ch = new_outgoing_channel("akjsdkajshdakjshd.alk:80");
+        ch->out()->write_bytes("GET http://stanford.edu/index.html HTTP/1.1\n\n");
+        channels_.emplace_back(std::move(ch));
+    }
+    
+    virtual void event_update() {
+        SharedChannel stdioch = get_stdio_channel();
+        dump_lines(stdioch->in(), stdioch->out(), 0);
+        eng::vector<SharedChannel> keep;
+        for (SharedChannel &ch : channels_) {
+            dump_lines(ch->in(), stdioch->out(), ch->chid());
+            if (ch->closed()) {
+                write_closed_message(ch.get(), stdioch->out());
+            } else {
+                keep.emplace_back(std::move(ch));
+            }
+        }
+        channels_ = std::move(keep);
+    }
+};
+
+// This test just prints the time.
+class DriverPrintClockTest : public DrivenEngine {
+public:
+    int count_;
+    double last_clock_;
+    virtual void event_init(int argc, char *argv[]) {
+        count_ = 0;
+        last_clock_ = 0.0;
+    }
+    
+    virtual void event_update() {
+        double clock = get_clock();
+        if (clock > last_clock_ + 0.5) {
+            int ms = eng::memhash();
+            stdostream() << std::fixed << std::setprecision(2) << clock << "   " << std::hex << ms << "  ";
+            count_++;
+            last_clock_ = clock;
+        }
+        if (count_ == 4) {
+            stdostream() << std::endl;
+            count_ = 0;
+        }
+    }
+};
+
+
+class RunUnitTests : public DrivenEngine {
+private:
+    UniqueWorld world_;
+
+    void event_init(int argc, char *argv[])
+    {
+        world_.reset(new World(util::WORLD_TYPE_STANDALONE));
+        world_->update_source(get_lua_source());
+        world_->run_unittests();
+        stop_driver();
+    }
+};
+
+DrivenEngineDefine("driverstubtest", DriverStubTest);
+DrivenEngineDefine("driverwebservertest", DriverWebServerTest);
+DrivenEngineDefine("driverdnsfailtest", DriverDNSFailTest);
+DrivenEngineDefine("driverprintclocktest", DriverPrintClockTest);
+DrivenEngineDefine("rununittests", RunUnitTests);
+

luprex/cpp/core/eng-tests.hpp

@@ -0,0 +1,5 @@
+#ifndef DRIVERTESTS_HPP
+#define DRIVERTESTS_HPP
+
+#endif // DRIVERTESTS_HPP
+

luprex/cpp/core/enginewrapper.hpp

@@ -0,0 +1,217 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// enginewrapper.hpp
+//
+// This header file contains driver's interface to class DrivenEngine.
+// This is meant to be used across a DLL boundary.  Since the DLL may have
+// been compiled by a different compiler than the driver, we use only simple
+// POD types and we only use C calling conventions.
+//
+// When calling a wrapper function, you must always pass in the wrapper as
+// the first parameter.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ENGINEWRAPPER_H
+#define ENGINEWRAPPER_H
+
+#define DRV_MAX_CHAN 256
+#define DRV_MAX_LISTEN_PORTS 256
+#define DRV_ERRMSG_SIZE 8192
+#define DRV_SHORTSTRING_SIZE 65536
+
+class DrivenEngine;
+class PlayLogfile;
+class ReplayLogfile;
+
+struct EngineWrapper {
+    char error[DRV_ERRMSG_SIZE];
+    char databuffer[DRV_SHORTSTRING_SIZE];
+    DrivenEngine *engine;
+    PlayLogfile *wlog;
+    ReplayLogfile *rlog;
+
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    //
+    // CONSTRUCTION
+    //
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    
+    // Of course, there's no constructor, since this is a C struct.
+    // To initialize it, you use 'dlsym' or 'GetProcAddress' to get the
+    // address of the function 'init_engine_wrapper'.  Then, you call
+    // the function init_engine_wrapper(&wrapper).
+    
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    //
+    // GETTERS
+    //
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+
+    // Get a list of all the listening ports.  The driver is expected
+    // to fetch this set shortly after the event_init callback is invoked.
+    //
+    void (*get_listen_ports)(EngineWrapper *w, uint32_t *nports, const uint32_t **ports);
+
+    // Get a list of all recently-opened channels that were created using
+    // new_outgoing_channel.  The driver should initiate outgoing
+    // connections for these channels.
+    //
+    void (*get_new_outgoing)(EngineWrapper *w, uint32_t *nchanids, const uint32_t **chanids);
+
+    // Get a string_view of the target of a channel. A target is a string like
+    // "cert:whatever.com:80" or "nocert:whatever.com:80".
+    // The first word indicate whether or not a valid SSL certificate
+    // is required.  The second word is the hostname.  The third word is
+    // the port number.  The char string returned here is valid until
+    // the channel is closed.
+    //
+    const char *(*get_target)(EngineWrapper *w, uint32_t chid);
+        
+    // Return true if the user has released all references to this channel.
+    // In this case, the driver should initiate shutdown of the channel,
+    // and the driver should eventually call notify_close.
+    //
+    bool (*get_channel_released)(EngineWrapper *w, uint32_t chid);
+
+    // Get a pointer to the bytes in the outgoing buffer.  The char pointer
+    // returned here is naturally only valid until the buffer is changed.
+    // This function is used for all channels, including sockets and stdio.
+    //
+    void (*get_outgoing)(EngineWrapper *w, uint32_t chid, uint32_t *len, const char **data);
+
+    // Return true if the outgoing buffer is empty.
+    //
+    bool (*get_outgoing_empty)(EngineWrapper *w, uint32_t chid);
+
+    // Get the clock.
+    //
+    // Get the current time.  This is equal to the last value passed
+    // in by invoke_event_update.
+    //
+    double (*get_clock)(EngineWrapper *w);
+
+    // Check the 'rescan_lua_source' flag.  If this flag is set, it means
+    // that the engine wants the driver to rescan the lua source code.
+    // When the driver sees this flag, it should rescan the source and call
+    // set_lua_source.
+    //
+    bool (*get_rescan_lua_source)(EngineWrapper *w);
+
+    // If true, the engine is done.  Stop the driver.
+    //
+    bool (*get_stop_driver)(EngineWrapper *w);
+
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    //
+    // MUTATORS USED ONLY IN PLAY MODE
+    //
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    
+    // Create the driven engine.  argc and argv allow you to specify what
+    // kind of engine you want.  You must pass in the initial state of the lua
+    // source, if you have any.  You may optionally also specify a replay log.
+    // If you don't want to create a replay log, pass a null pointer.
+    //
+    // Check to see if the error buffer contains a message after calling
+    // this function.
+    //
+    void (*play_initialize)(EngineWrapper *w, uint32_t argc, char **argv, uint32_t srcpklen, const char *srcpk, const char *logfn);
+
+    // Clear the list of recently-opened channels.  You are meant to fetch
+    // new outgoing channels using get_new_outgoing, then you call
+    // clear_new_outgoing after you've opened those channels.
+    //
+    void (*play_clear_new_outgoing)(EngineWrapper *w);
+        
+    // Notifies the channel that some bytes were transmitted.  This causes those
+    // bytes to be removed from the outgoing buffer.  This function is used for
+    // all channels, including sockets and stdio.
+    //
+    void (*play_sent_outgoing)(EngineWrapper *w, uint32_t chid, uint32_t nbytes);
+        
+    // Notifies the channel that some bytes were received.  This causes those
+    // bytes to be appended to the incoming buffer.  This function is used for
+    // all channels, including sockets and stdio.
+    //
+    void (*play_recv_incoming)(EngineWrapper *w, uint32_t chid, uint32_t len, const char *data);
+
+    // Notify the channel that the connection was closed.  This includes all
+    // sorts of closes, including friendly termination, all the way to network
+    // failure.  Closing the channel doesn't delete it.  The engine is
+    // responsible for noticing that the channel closed and the engine must
+    // delete it.  Closing a channel prevents it from showing up in
+    // 'list_channels'.  
+    //
+    void (*play_notify_close)(EngineWrapper *w, uint32_t chid, uint32_t len, const char *data);
+    
+    // Notify the DrivenEngine that somebody connected to an incoming port.
+    // This will cause the DrivenEngine to allocate a new channel and put the
+    // new channel into the incoming channels queue.  Returns the new channel
+    // ID.  The new incoming channel appears in the 'list_channels' list,
+    // even before the engine pops the channel from the incoming channels queue.
+    //
+    uint32_t (*play_notify_accept)(EngineWrapper *w, uint32_t port);
+        
+    // Invoke the update event.
+    //
+    // The clock value must absolutely be monotonically increasing,
+    // and it should roughly be equal to the number of seconds since
+    // the program started.
+    //
+    void (*play_invoke_event_update)(EngineWrapper *w, double clock);
+
+    // Store the lua source code.
+    //
+    void (*play_set_lua_source)(EngineWrapper *w, uint32_t srcpklen, const char *srcpk); 
+
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    //
+    // MUTATORS USED ONLY IN REPLAY MODE
+    //
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    
+    // Begin a replay.
+    //
+    // Opens the logfile and prepares to replay the log.
+    // If an error occurs, the error buffer contains a message,
+    // and the done flag is set to true.
+    //
+    void (*replay_initialize)(EngineWrapper *w, const char *logfn);
+        
+    // Execute a single step from the replay log.
+    //
+    // Calling this when 'done' is true is a no-op.
+    //
+    void (*replay_step)(EngineWrapper *w);
+
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+    //
+    // FUNCTIONS THAT CAN BE USED AT ANY TIME
+    //
+    //////////////////////////////////////////////////////////////////////////////
+    //////////////////////////////////////////////////////////////////////////////
+
+    // Restore the wrapper to its initial blank state.
+    //
+    // Note that the wrapper must have already been initialized using
+    // init_engine_wrapper.  Otherwise, the 'release' function pointer would not
+    // be initialized.  If writing a logfile, this stores a 'clean exit' marker
+    // in the logfile, indicating that the engine exited cleanly, as opposed to
+    // crashing.
+    //
+    // If the wrapper is already in its clear state, this is a no-op.
+    //
+    void (*release)(EngineWrapper *w);
+};
+
+#endif // ENGINEWRAPPER_HPP

luprex/cpp/core/globaldb.cpp

@@ -0,0 +1,72 @@
+#include "luastack.hpp"
+#include "globaldb.hpp"
+
+LuaDefine(global_once, "name", "for a given string, returns true exactly once") {
+    LuaArg name;
+    LuaRet flag;
+    LuaVar oncedb, val;
+    LuaStack LS(L, name, flag, oncedb, val);
+
+    // Get a pointer to the oncedb.
+    LS.rawget(oncedb, LuaRegistry, "oncedb");
+    if (!LS.istable(oncedb)) {
+        LS.set(flag, false);
+        return LS.result();
+    }
+
+    LS.checkstring(name, "name");
+    LS.rawget(val, oncedb, name);
+    if (!LS.isnil(val)) {
+        LS.set(flag, false);
+        return LS.result();
+    }
+    LS.rawset(oncedb, name, true);
+    LS.set(flag, true);
+    return LS.result();
+}
+
+LuaDefine(global_clearonce, "name", "reset the specified once-flag") {
+    LuaArg name;
+    LuaVar oncedb;
+    LuaStack LS(L, name, oncedb);
+
+    // Get a pointer to the oncedb.
+    LS.rawget(oncedb, LuaRegistry, "oncedb");
+    if (!LS.istable(oncedb)) {
+        return LS.result();
+    }
+    LS.checkstring(name, "name");
+    LS.rawset(oncedb, name, LuaNil);
+    return LS.result();
+}
+
+LuaDefine(global_table, "globalname", "get a table where global data can be stored") {
+    LuaArg globalname;
+    LuaRet globaltab;
+    LuaVar globaldb;
+    LuaStack LS(L, globalname, globaltab, globaldb);
+    LS.checkstring(globalname, "globalname");
+
+    // Get a pointer to the globaldb.
+    LS.rawget(globaldb, LuaRegistry, "globaldb");
+
+    // nopredict
+    if (lua_isyieldable(L) && (!LS.istable(globaldb))) {
+        return lua_yield(L, 0);
+    }
+
+    // Get the globaltab from the globaldb, sanity check it.
+    LS.rawget(globaltab, globaldb, globalname);
+    if (LS.istable(globaltab)) {
+        return LS.result();
+    } else if (!LS.isnil(globaltab)) {
+        luaL_error(L, "%s is not a global", LS.ckstring(globalname).c_str());
+    }
+
+    // Create a new globaltab and store it in the globaldb.
+    LS.newtable(globaltab);
+    LS.rawset(globaldb, globalname, globaltab);
+    LS.rawset(globaltab, "__global", globalname);
+    LS.settabletype(globaltab, LUA_TT_GLOBALDB);
+    return LS.result();
+}

luprex/cpp/core/globaldb.hpp

@@ -0,0 +1,32 @@
+////////////////////////////////////////////////////////////
+//
+// GLOBALDB
+//
+// The master world model is allowed to maintain global data structures.
+//
+// Unfortunately, any attempt to put a global data structure into the global
+// environment will fail, because the global environment periodically gets wiped
+// clean by the "source rebuild" procedure (see module 'source').
+//
+// This module creates a safe space where you can put global data structures
+// that won't get wiped out.
+//
+// THE GLOBAL OPERATOR
+//
+// This module adds a new function to lua: "global".  Global takes a global data
+// structure name (a string) and returns a table for that global data structure.
+// You can put anything you want into the table.
+//
+// Since you're only allowed to use global data structures in the master world
+// model, any attempt to call "global" in a synchronous world model will
+// result in a 'donotpredict' error.
+//
+////////////////////////////////////////////////////////////
+
+#ifndef GLOBALDB_HPP
+#define GLOBALDB_HPP
+
+
+#endif // GLOBALDB_HPP
+
+

luprex/cpp/core/gui.cpp

@@ -0,0 +1,70 @@
+#include "wrap-string.hpp"
+#include "wrap-map.hpp"
+#include "wrap-vector.hpp"
+#include "util.hpp"
+#include "gui.hpp"
+#include "luastack.hpp"
+
+void Gui::store_global_pointer(lua_State *L, Gui *g) {
+    lua_pushstring(L, "gui");
+    lua_pushlightuserdata(L, g);
+    lua_rawset(L, LUA_REGISTRYINDEX);
+}
+
+Gui *Gui::fetch_global_pointer(lua_State *L) {
+    lua_pushstring(L, "gui");
+    lua_rawget(L, LUA_REGISTRYINDEX);
+    Gui *result = (Gui *)lua_touserdata(L, -1);
+    if (result == nullptr) {
+        luaL_error(L, "Not currently building a GUI");
+    }
+    lua_pop(L, 1);
+    return result;
+}
+
+void Gui::menu_item(const eng::string &action, const eng::string &label) {
+    GuiElt elt;
+    elt.type_ = GuiElt::TYPE_MENU_ITEM;
+    elt.action_ = action;
+    elt.label_ = label;
+    elts_.push_back(elt);
+}
+
+bool Gui::has_action(const eng::string &action) const {
+    for (const GuiElt &elt : elts_) {
+        if (elt.action_ == action) {
+            return true;
+        }
+    }
+    return false;
+}
+
+eng::string Gui::get_action(int64_t index) {
+    if ((index < 0) || (index >= int64_t(elts_.size()))) {
+        return "";
+    }
+    return elts_[index].action();
+}
+
+eng::string Gui::menu_debug_string() const {
+    eng::ostringstream oss;
+    int index = 0;
+    for (const GuiElt &elt : elts()) {
+        oss << index << " " << elt.label() << std::endl;
+        index += 1;
+    }
+    return oss.str();
+}
+
+LuaDefine(gui_menu_item, "action,label", "add a menu item to the current gui") {
+    Gui *gui = Gui::fetch_global_pointer(L);
+    LuaArg laction, llabel;
+    LuaStack LS(L, laction, llabel);
+    eng::string action = LS.ckstring(laction);
+    eng::string label = LS.ckstring(llabel);
+    if (!sv::has_prefix(action, "cb_")) {
+        luaL_error(L, "menuitem callbacks must start with cb_");
+    }
+    gui->menu_item(action, label);
+    return LS.result();
+}

luprex/cpp/core/gui.hpp

@@ -0,0 +1,57 @@
+#ifndef GUI_HPP
+#define GUI_HPP
+
+#include "wrap-string.hpp"
+#include "wrap-map.hpp"
+#include "wrap-vector.hpp"
+
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+
+class GuiElt : public eng::nevernew {
+    friend class Gui;
+public:
+    enum Type {
+        TYPE_MENU_ITEM,
+    };
+private:
+    Type type_;
+    eng::string action_;
+    eng::string label_;
+
+    GuiElt() {}
+public:
+    ~GuiElt() {}
+    Type type() const { return type_; }
+    const eng::string &action() const { return action_; }
+    const eng::string &label() const { return label_; }
+};
+
+class Gui : public eng::nevernew {
+public:
+    using EltVec = eng::vector<GuiElt>;
+private:
+    int64_t place_;
+    EltVec elts_;
+public:
+    Gui() { place_ = 0; }
+    int64_t place() { return place_; }
+    const EltVec &elts() const { return elts_; }
+    void clear(int64_t p) { place_ = p; elts_.clear(); }
+    bool has_action(const eng::string &action) const;
+    void menu_item(const eng::string &action, const eng::string &label);
+    eng::string get_action(int64_t index);
+    eng::string menu_debug_string() const;
+
+    // Put a pointer to a gui into the lua registry.
+    //
+    // All lua commands that manipulate the GUI implicitly
+    // operate on this global gui pointer.
+    //
+    static void store_global_pointer(lua_State *L, Gui *g);
+    static Gui *fetch_global_pointer(lua_State *L);
+};
+
+
+#endif // GUI_HPP
+

luprex/cpp/core/http.hpp

@@ -0,0 +1,450 @@
+/////////////////////////////////////////////////////////
+//
+// HTTP Implementation.
+//
+// This is a fairly limited implementation of HTTP.
+//
+// It only supports HTTP 1.1
+//
+// It only supports GET, POST, and HEAD.
+//
+/////////////////////////////////////////////////////////
+
+#ifndef HTTP_HPP
+#define HTTP_HPP
+
+#include "eng-malloc.hpp"
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+#include "wrap-map.hpp"
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+#include "drivenengine.hpp"
+#include <ostream>
+
+using UrlParameters = eng::map<eng::string, eng::string>;
+
+class HttpClientRequest : public eng::nevernew {    
+private:
+    // Request IDs.
+    int64_t request_id_;
+    int64_t place_id_;
+    int64_t thread_id_;
+    
+    // When we detect that we generated an invalid
+    // outgoing request, the error is stored here.
+    eng::string check_fail_;
+
+    // If true, verify the server's certificate.
+    // True is the default.
+    bool verify_certificate_;
+
+    // Method: GET, HEAD, or POST.  Must be all-caps.
+    eng::string method_;
+
+    // The hostname. Not yet url-encoded.
+    eng::string host_;
+
+    // Port number.
+    int port_;
+
+    // The path.  Not yet url-encoded.  Can not include URL parameters.
+    eng::string path_;
+    
+    // URL parameters to append to the path.  Not yet url-encoded.
+    UrlParameters params_;
+
+    // The mime type of the content, only for POST requests.
+    eng::string mime_type_;
+
+    // The content as a string, only for POST requests.
+    eng::string content_;
+    bool content_assigned_;
+
+private:
+    void check_fail(std::string_view error);
+    void send_internal(StreamBuffer *target, bool debug_string) const;
+
+public:
+    // Construct an empty HTTP request.
+    // All of the fields have empty values.
+    HttpClientRequest();
+    
+    // Populate an request-related fields one piece at a time.
+    // If you pass an invalid value, or if the field is
+    // already set, the routine will generate an error message
+    // and store it in the error field.  In that case, the set
+    // will not happen.  If there's already an error in the error
+    // field, it will not be overwritten.
+    //
+    void set_verify_certificate(bool flag);
+    void set_method(const eng::string &method);
+    void set_host(const eng::string &host);
+    void set_port(int port);
+    void set_path(std::string_view path);
+    void set_param(const eng::string &key, const eng::string &value);
+    void set_url(std::string_view url);
+    void set_mime_type(const eng::string &mime_type);
+    void set_content(const eng::string &content);
+ 
+    void set_verify_certificate(LuaStack &LS, LuaSlot val);
+    void set_method(LuaStack &LS, LuaSlot val);
+    void set_host(LuaStack &LS, LuaSlot val);
+    void set_port(LuaStack &LS, LuaSlot val);
+    void set_path(LuaStack &LS, LuaSlot path);
+    void set_param(LuaStack &LS, LuaSlot key, LuaSlot val);
+    void set_params(LuaStack &LS, LuaSlot tab);
+    void set_url(LuaStack &LS, LuaSlot val);
+    void set_mime_type(LuaStack &LS, LuaSlot val);
+    void set_content(LuaStack &LS, LuaSlot val);
+    void set_jsonvalue(LuaStack &LS, LuaSlot val);
+
+    // Set default values for method and port.
+    // This must be done after setting regular values.
+    void set_defaults();
+
+    // Populate request-related fields from a Lua table.
+    // Doesn't throw errors, instead, returns errors via check(). 
+    void configure(LuaKeywordParser &kp);
+    
+    // Get or Set the request IDs.
+    //
+    // This class does not use these fields, it just stores
+    // them as a convenience.
+    //
+    int64_t request_id() const { return request_id_; }
+    int64_t place_id() const { return place_id_; }
+    int64_t thread_id() const { return thread_id_; }
+    void set_request_id(int64_t request_id) { request_id_ = request_id; }
+    void set_place_id(int64_t place_id) { place_id_ = place_id; }
+    void set_thread_id(int64_t thread_id) { thread_id_ = thread_id; }
+    
+    // Get the network target, eg, "cert:host:port"
+    //
+    eng::string target() const;
+
+    // Verify that the request is error free and that
+    // defaults have been set.
+    //
+    eng::string check() const;
+
+    // Get the method.
+    //
+    eng::string method() const { return method_; }
+
+    // Put the request into the stream, assuming HTTP/1.1
+    //
+    void send(StreamBuffer *target) const { send_internal(target, false); }
+
+    // Serialize and deserialize.
+    //
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
+    // Get the request as a debug string.
+    //
+    eng::string debug_string();
+};
+
+class HttpServerResponse {
+private:
+    // When we detect that our own response is
+    // invalid, the error is stored here.
+    //
+    eng::string check_fail_;
+
+    // The status code to send back to the client
+    // as part of the response status line.
+    //
+    int status_;
+
+    // Maximum age for the cache-control directive.
+    //
+    int max_age_;
+
+    // Mime type of the content.
+    //
+    eng::string mime_type_;
+
+    // Content to send to the client.
+    //
+    eng::string content_;
+    bool content_assigned_;
+
+    // If the keep-alive flag is set, then a connection: keep-alive
+    // header is sent.  Otherwise, a connection: close is sent.
+    // The keep-alive flag cannot be controlled from Lua.  It is
+    // designed to be handled automatically by the server code.
+    //
+    bool keep_alive_;
+
+    // The protocol is taken from the request.
+    //
+    bool http11_;
+        
+private:
+    void check_fail(std::string_view error);
+    void send_internal(StreamBuffer *target, bool debug_string) const;
+
+public:
+    // Construct an empty response.
+    // All of the fields have empty values.
+    //
+    HttpServerResponse();
+
+    void set_method(const eng::string &method);
+    void set_status(int status);
+    void set_max_age(int max_age);
+    void set_mime_type(const eng::string &mime_type);
+    void set_content(const eng::string &content);
+ 
+    void set_status(LuaStack &LS, LuaSlot val);
+    void set_max_age(LuaStack &LS, LuaSlot val);
+    void set_mime_type(LuaStack &LS, LuaSlot val);
+    void set_content(LuaStack &LS, LuaSlot val);
+    void set_jsonvalue(LuaStack &LS, LuaSlot val);
+
+    // Set default values.
+    //
+    void set_defaults();
+
+    // Populate request-related fields from a Lua table.
+    // Doesn't throw errors, instead, returns them via check()
+    //
+    void configure(LuaKeywordParser &kp);
+
+    // Set the keep_alive flag.
+    //
+    void set_keep_alive(bool k) { keep_alive_ = k; };
+
+    // Set the protocol.
+    //
+    void set_http11(bool k) { http11_= k; }
+    
+    // Store a fail response.
+    //
+    void fail(int status, const eng::string &message);
+
+    // Verify that the response is error free.
+    //
+    eng::string check() const;
+
+    // Get the status of the response.
+    //
+    int status() const { return status_; }
+    bool errstatus() const { return (status_ >= 400)&&(status_ <= 599); }
+
+    // Put the response into the stream, assuming HTTP/1.1
+    //
+    void send(StreamBuffer *target) const { send_internal(target, false); }
+
+    // Get the response as a debug string.
+    //
+    eng::string debug_string();
+};
+
+// HttpParser is used for parsing both requests and responses.
+//
+class HttpParser : public eng::nevernew {
+protected:
+    // The request ID is not used by the parser, but
+    // you can store a request ID in there for convenience.
+    int64_t request_id_;
+
+    // True if this parser parsed a request.  If false,
+    // then it parsed a response.
+    bool is_request_;
+
+    // The status code, a 3-digit number.
+    int status_;
+
+    // If the communication contains an error, the error can
+    // be stored here.  In the event of a successful communication,
+    // this should always be empty string.
+    eng::string error_;
+
+    // Only if content-length header present, otherwise, -1.
+    int64_t content_length_;
+
+    // If empty, it means there was no transfer-encoding header.
+    eng::string transfer_encoding_;
+
+    // If empty, it means there was no content-encoding header.
+    eng::string content_encoding_;
+
+    // Only if location header present.
+    eng::string location_;
+
+    // MIME type of the content.
+    eng::string mime_type_;
+
+    // Charset of the content before the content was translated to utf-8.
+    eng::string charset_;
+
+    // The protocol: true for HTTP/1.1, false for HTTP/1.0
+    bool http11_;
+
+    // The method.  In a response, this is assigned before parsing.
+    eng::string method_;
+
+    // The URL path, not url-encoded. (only when parsing requests)
+    eng::string path_;
+
+    // The URL parameters, not url-encoded (only when parsing requests)
+    UrlParameters params_;
+
+    // The content as string.  If it's text, it's been translated to utf-8.
+    eng::string content_;
+
+    // The length in bytes of the entire communication.
+    // If the response is complete, but the comm_length_
+    // is zero, it means we couldn't find the end of 
+    // the request.
+    int comm_length_;
+
+protected:
+    // Store a message indicating that we haven't received enough
+    // bytes yet.  If the connection is closed and we still haven't
+    // received enough bytes, that's a fatal error.
+    //
+    void incomplete(bool closed);
+
+    // Store a message indicating that we couldn't parse because
+    // something was syntactically malformed, ie, not valid HTTP.
+    //
+    void syntax(std::string_view detail);
+
+    // Store a message indicating that the content was too large
+    // and we refused to download it.
+    //
+    void oversized();
+
+    // Parse a response status line, such as "HTTP/1.1 200 OK"
+    // returns false if we couldn't parse the whole line.
+    //
+    bool parse_status_line(std::string_view &view, bool closed);
+
+    // Parse a request line, such as "GET /index.html HTTP/1.1"
+    // returns false if we couldn't parse the whole line.
+    //
+    bool parse_request_line(std::string_view &view, bool closed);
+
+    // Parse specific headers.
+    //
+    void parse_content_encoding(std::string_view value);
+    void parse_content_length(std::string_view value);
+    void parse_content_type(std::string_view value);
+    void parse_location(std::string_view value);
+    void parse_transfer_encoding(std::string_view value);
+
+    // Parse a single response header.  Most headers are ignored.
+    // If the header contains an error, the error is stored.
+    //
+    void parse_header(std::string_view header, std::string_view value);
+
+    // Parse all of the headers, including the blank line after
+    // the headers.  Return true if the view is at the end of the headers.
+    //
+    bool parse_headers(std::string_view &view, bool closed);
+    
+    // parse the content, for different transfer encodings.
+    // Returns true if the view is at the end of the content.
+    //
+    bool parse_content_basic(std::string_view &view, bool closed);
+    bool parse_content_chunked(std::string_view &view, bool closed);
+
+    // Parse the content.  Return true if the view is at the end of the
+    // content.
+    //
+    // - Uses the appropriate transfer-encoding specified in the headers.
+    // - Decompresses the content using the appropriate content-encoding.
+    // - Guesses a mimetype and charset if one was not specified in headers.
+    // - If it's text, converts the charset to utf-8.
+    //
+    bool parse_content(std::string_view &view, bool closed);
+
+public:
+    // Construct a blank parser.
+    //
+    HttpParser();
+
+    // Get the parsed status or error message.
+    //
+    int status() const { return status_; }
+    bool errstatus() const { return (status_ >= 400) && (status_ <= 599); }
+    eng::string error() const { return error_; }
+
+    // Return true if the communication was complete.
+    //
+    bool complete() const { return status_ != 0; }
+
+    // Get the first component of the path.
+    // If the path is empty, return defval.
+    eng::string first_path_component(std::string_view defval) const;
+
+    // Get the parsed protocol.
+    //
+    bool http11() const { return http11_; }
+
+    // Store the parsed fields into a lua table.
+    //
+    void store(LuaStack &LS, LuaSlot tab) const;
+
+    // The parser will not try to parse content longer than this.
+    //
+    static constexpr int64_t MAX_CONTENT_LENGTH = 1000000;
+
+    // Parse a request or a response.
+    //
+    // You can only parse once.  If you need to parse again,
+    // construct a new parser.
+    //
+    void parse_request(std::string_view view, bool closed);
+    void parse_response(std::string_view view, bool closed, std::string_view method);
+
+    // Store a status code and an error message, and clear the content.
+    // This is generally used when the client detects an error,
+    // such as a DNS lookup fail, a connection failed, an SSL negotiation
+    // failed, or the like.
+    //
+    void fail(int status, std::string_view error);
+
+    // Get or set the request ID.
+    //
+    int64_t request_id() const { return request_id_; }
+    void set_request_id(int64_t v) { request_id_ = v; }
+
+    // Return a debug string.
+    //
+    eng::string debug_string() const;
+
+    // Synthesize an error and store it in lua.
+    //
+    static void store_fail(LuaStack &LS, LuaSlot tab, int status, std::string_view error);
+};
+
+class HttpClientRequestMap : public eng::map<int64_t, HttpClientRequest> {
+public:
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+};
+
+using HttpParserVec = eng::vector<HttpParser>;
+
+// This class is used by LpxServer to store the
+// incoming and outgoing http channels.
+//
+class HttpChannel {
+public:
+    SharedChannel channel_;
+    eng::string method_;
+    int64_t parsed_bytes_;
+
+    bool marked_for_deletion() const { return channel_ == nullptr; }
+    HttpChannel() : parsed_bytes_(0) {}
+};
+
+using HttpChannelMap = eng::map<int64_t, HttpChannel>;
+using HttpChannelVec = eng::vector<HttpChannel>;
+
+#endif // HTTP_HPP

luprex/cpp/core/idalloc.cpp

@@ -0,0 +1,433 @@
+#include "wrap-map.hpp"
+#include "wrap-sstream.hpp"
+#include "wrap-deque.hpp"
+
+#include "idalloc.hpp"
+
+#include <ostream>
+
+
+static bool ranges_equal(const eng::deque<int64_t> &dq, int64_t a, int64_t b, int64_t c) {
+    if (dq.size() != 3) return false;
+    if (dq[0] != a) return false;
+    if (dq[1] != b) return false;
+    if (dq[2] != c) return false;
+    return true;
+}
+
+IdGlobalPool::IdGlobalPool() {
+    salvaged_.clear();
+    next_batch_ = 0;
+    next_id_ = 0;
+    next_seqno_ = 0;
+}
+
+IdGlobalPool::~IdGlobalPool() {
+}
+
+void IdGlobalPool::init_master() {
+    salvaged_.clear();
+    next_batch_ = 0x0001000000000000;
+    next_id_ = 0x0010000000000000;
+}
+
+void IdGlobalPool::init_synch() {
+    salvaged_.clear();
+    next_batch_ = 0;
+    next_id_ = 0x001E000000000000;
+}
+
+int64_t IdGlobalPool::get_one() {
+    return next_id_++;
+}
+
+int64_t IdGlobalPool::get_batch() {
+    int64_t batch;
+    if (salvaged_.empty()) {
+        if (next_batch_ == 0) {
+            batch = 0;
+        } else {
+            batch = next_batch_;
+            next_batch_ += 256;
+        }
+    } else {
+        batch = salvaged_.back();
+        salvaged_.pop_back();
+    }
+    return batch;
+}
+
+void IdGlobalPool::salvage(int64_t batch) {
+    if (batch == 0) return;
+    if (next_batch_ == 0) return;
+    if ((batch & 0xFF) >= 128) return;
+    salvaged_.push_back(batch);
+}
+
+void IdGlobalPool::serialize(StreamBuffer *sb) const {
+    sb->write_int64(next_batch_);
+    sb->write_int64(next_id_);
+    sb->write_uint64(next_seqno_);
+    sb->write_uint32(salvaged_.size());
+    for (int64_t batch : salvaged_) {
+        sb->write_int64(batch);
+    }
+}
+
+void IdGlobalPool::deserialize(StreamBuffer *sb) {
+    next_batch_ = sb->read_int64();
+    next_id_ = sb->read_int64();
+    next_seqno_ = sb->read_uint64();
+    uint32_t salvaged_size = sb->read_uint32();
+    salvaged_.resize(salvaged_size);
+    for (int i=0; i < int(salvaged_size); i++) {
+        salvaged_[i] = sb->read_int64();
+    }
+}
+
+eng::string IdGlobalPool::debug_string() const {
+    eng::ostringstream oss;
+    oss << "next_batch:" << util::hex64.val(next_batch_) << " ";
+    oss << "next_id:" << util::hex64.val(next_id_) << " ";
+    oss << "next_seqno: " << util::hex64.val(next_seqno_) << " ";
+    oss << "salvaged:";
+    for (const int64_t val : salvaged_) {
+        oss << " " << util::hex64.val(val);
+    }
+    return oss.str();
+}
+
+IdPlayerPool::IdPlayerPool(IdGlobalPool *g) {
+    global_ = g;
+    fifo_capacity_ = 0;
+    next_seqno_ = 0;
+}
+
+IdPlayerPool::~IdPlayerPool() {
+}
+
+void IdPlayerPool::set_fifo_capacity(int n) {
+    assert((n >= 0) && (n <= 250));
+    fifo_capacity_ = n;
+    while (int(ranges_.size()) > n) {
+        global_->salvage(ranges_.back());
+        ranges_.pop_back();
+    }
+}
+
+void IdPlayerPool::refill() {
+    while (int(ranges_.size()) < fifo_capacity_) {
+        int64_t batch = global_->get_batch();
+        if (batch == 0) break;
+        ranges_.push_back(batch);
+    }
+}
+
+void IdPlayerPool::test_push_back(int64_t range) {
+    ranges_.push_back(range);
+}
+
+void IdPlayerPool::test_pop_front() {
+    ranges_.pop_front();
+}
+
+void IdPlayerPool::test_clear_ranges() {
+    ranges_.clear();
+}
+
+int64_t IdPlayerPool::get_one() {
+    refill();
+    if (ranges_.empty()) {
+        return global_->get_one();
+    } else {
+        int64_t id = ranges_.front();
+        if ((id & 0xFF) == 0xFF) {
+            ranges_.pop_front();
+            refill();
+        } else {
+            ranges_.front() = id + 1;
+        }
+        return id;
+    }
+}
+
+void IdPlayerPool::serialize(StreamBuffer *sb) const {
+    sb->write_uint8(fifo_capacity_);
+    sb->write_uint8(ranges_.size());
+    sb->write_uint64(next_seqno_);
+    for (int64_t batch : ranges_) {
+        sb->write_int64(batch);
+    }
+}
+
+void IdPlayerPool::deserialize(StreamBuffer *sb) {
+    fifo_capacity_ = sb->read_uint8();
+    int ranges_size = sb->read_uint8();
+    next_seqno_ = sb->read_uint64();
+    ranges_.resize(ranges_size);
+    for (int i=0; i < ranges_size; i++) {
+        ranges_[i] = sb->read_int64();
+    }
+}
+
+bool IdPlayerPool::exactly_equal(const IdPlayerPool &other) const {
+    if (fifo_capacity_ != other.fifo_capacity_) return false;
+    if (ranges_.size() != other.ranges_.size()) return false;
+    if (next_seqno_ != other.next_seqno_) return false;
+    for (int i = 0; i < int(ranges_.size()); i++) {
+        if (ranges_[i] != other.ranges_[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+bool IdPlayerPool::valid() const {
+    if ((fifo_capacity_ < 0) || (fifo_capacity_ > 250)) return false;
+    if (int(ranges_.size()) > fifo_capacity_) return false;
+    return true;
+}
+
+void IdPlayerPool::diff(const IdPlayerPool &auth, StreamBuffer *sb) const {
+    assert(valid());
+    assert(auth.valid());
+
+    // Special case: there's nothing to fix.
+    if (exactly_equal(auth)) {
+        sb->write_uint8(255);
+        return;
+    }
+
+    // Write the fifo capacity, nranges, and next seqno
+    assert(auth.fifo_capacity_ != 255);
+    sb->write_uint8(auth.fifo_capacity_);
+    sb->write_uint8(auth.ranges_.size());
+    sb->write_uint64(auth.next_seqno_);
+    
+    // Build up an index of the known IDs.
+    eng::map<int64_t, int> index;
+    for (int i = 0; i < int(ranges_.size()); i++) {
+        index[ranges_[i]] = i;
+    }
+
+    // Write the ranges, but encode known IDs in one byte.
+    for (int i = 0; i < int(auth.ranges_.size()); i++) {
+        int64_t n = auth.ranges_[i];
+        auto iter = index.find(n);
+        if (iter == index.end()) {
+            sb->write_uint8(255);
+            sb->write_int64(n);
+        } else {
+            int slot = iter->second;
+            sb->write_uint8(slot);
+        }
+    }
+}
+
+void IdPlayerPool::patch(StreamBuffer *sb, DebugCollector *dbc) {
+    // read the header byte
+    int fifo_cap = sb->read_uint8();
+    // If the fifo_cap is 255, it means there's nothing to fix.
+    if (fifo_cap == 255) {
+        return;
+    }
+    DebugLine(dbc) << "IdPlayerPool modified";
+    fifo_capacity_ = fifo_cap;
+    int nranges = sb->read_uint8();
+    next_seqno_ = sb->read_uint64();
+    eng::deque<int64_t> old = std::move(ranges_);
+    ranges_.clear();
+    for (int i = 0; i < nranges; i++) {
+        int index = sb->read_uint8();
+        if (index < 255) {
+            assert(index < int(old.size()));
+            ranges_.push_back(old[index]);
+        } else {
+            ranges_.push_back(sb->read_int64());
+        }
+    }
+}
+
+eng::string IdPlayerPool::debug_string() const {
+    eng::ostringstream oss;
+    oss << "cap:" << fifo_capacity_ << " ids:";
+    for (int i = 0; i < int(ranges_.size()); i++) {
+        if (i > 0) oss << ",";
+        oss << util::hex64.val(ranges_[i]);
+    }
+    oss << " seqno:" << util::hex64.val(next_seqno_);
+    return oss.str();
+}
+
+static int64_t nthbatch(int64_t n) {
+    return int64_t(0x0001000000000000) + n*256;
+}
+
+LuaDefine(unittests_idalloc, "", "some unit tests") {
+    IdGlobalPool gp;
+    IdPlayerPool pp(&gp);
+    IdGlobalPool gpds;
+    IdPlayerPool ppds(&gpds);
+    StreamBuffer sb;
+
+    // Synchronous pools produce IDs starting at 0x001E000000000000
+    gp.init_synch();
+    LuaAssert(L, gp.get_one() == 0x001E000000000000);
+    LuaAssert(L, gp.get_one() == 0x001E000000000001);
+    LuaAssert(L, gp.get_one() == 0x001E000000000002);
+
+    // Master pools produce IDs starting at 0x0010000000000000
+    gp.init_master();
+    LuaAssert(L, gp.get_one() == 0x0010000000000000);
+    LuaAssert(L, gp.get_one() == 0x0010000000000001);
+    LuaAssert(L, gp.get_one() == 0x0010000000000002);
+
+    // Synchronous pools produce only null batches.
+    gp.init_synch();
+    LuaAssert(L, gp.get_batch() == 0);
+    LuaAssert(L, gp.get_batch() == 0);
+    gp.salvage(nthbatch(5));
+    LuaAssert(L, gp.get_batch() == 0);
+    
+    // Simple fetch batches with a few salvages.
+    gp.init_master();
+    LuaAssert(L, gp.get_batch() == nthbatch(0));
+    LuaAssert(L, gp.get_batch() == nthbatch(1));
+    LuaAssert(L, gp.get_batch() == nthbatch(2));
+    gp.salvage(nthbatch(182));
+    gp.salvage(nthbatch(183));
+    LuaAssert(L, gp.get_batch() == nthbatch(183));
+    LuaAssert(L, gp.get_batch() == nthbatch(182));
+    LuaAssert(L, gp.get_batch() == nthbatch(3));
+    
+    // Salvage of a zero-batch does nothing.
+    gp.init_master();
+    LuaAssert(L, gp.get_batch() == nthbatch(0));
+    LuaAssert(L, gp.get_batch() == nthbatch(1));
+    gp.salvage(0);
+    LuaAssert(L, gp.get_batch() == nthbatch(2));
+
+    // Salvage of a partial batch.
+    gp.init_master();
+    LuaAssert(L, gp.get_batch() == nthbatch(0));
+    LuaAssert(L, gp.get_batch() == nthbatch(1));
+    gp.salvage(nthbatch(142) + 10);
+    LuaAssert(L, gp.get_batch() == nthbatch(142) + 10);
+    LuaAssert(L, gp.get_batch() == nthbatch(2));
+
+    // Salvage of a half-empty batch does nothing.
+    gp.init_master();
+    LuaAssert(L, gp.get_batch() == nthbatch(0));
+    LuaAssert(L, gp.get_batch() == nthbatch(1));
+    gp.salvage(nthbatch(142) + 145);
+    LuaAssert(L, gp.get_batch() == nthbatch(2));
+
+    // In the synchronous model, refill should do nothing.
+    // The player pool should shunt through to the global.
+    pp.test_clear_ranges();
+    gp.init_synch();
+    pp.set_fifo_capacity(3);
+    pp.refill();
+    LuaAssert(L, pp.size() == 0);
+    LuaAssert(L, pp.get_one() == 0x001E000000000000);
+    LuaAssert(L, pp.size() == 0);
+
+    // In the master model, with fifo disabled.  Fifo should remain
+    // empty, and the player pool should shunt through to the global.
+    gp.init_master();
+    pp.test_clear_ranges();
+    pp.set_fifo_capacity(0);
+    pp.refill();
+    LuaAssert(L, pp.size() == 0);
+    LuaAssert(L, pp.get_one() == 0x0010000000000000);
+    LuaAssert(L, pp.size() == 0);
+
+    // Test refill from master (with enabled fifo).
+    gp.init_master();
+    pp.test_clear_ranges();
+    pp.set_fifo_capacity(3);
+    pp.refill();
+    LuaAssert(L, ranges_equal(pp.ranges_, nthbatch(0), nthbatch(1), nthbatch(2)));
+
+    // Now test that get_one() keeps the pool filled from master.
+    for (int i = 0; i < 256; i++) {
+        LuaAssert(L, pp.get_one() == nthbatch(0) + i);
+    }
+    for (int i = 0; i < 256; i++) {
+        LuaAssert(L, pp.get_one() == nthbatch(1) + i);
+    }
+    LuaAssert(L, ranges_equal(pp.ranges_, nthbatch(2), nthbatch(3), nthbatch(4)));
+
+    // Test unqueueing the batches.
+    LuaAssert(L, gp.get_batch() == nthbatch(5));
+    LuaAssert(L, gp.get_batch() == nthbatch(6));
+    pp.set_fifo_capacity(0);
+    LuaAssert(L, gp.get_batch() == nthbatch(2));
+    LuaAssert(L, gp.get_batch() == nthbatch(3));
+    LuaAssert(L, gp.get_batch() == nthbatch(4));
+    LuaAssert(L, gp.get_batch() == nthbatch(7));
+
+    // Serialize and deserialize a global pool.
+    gp.init_master();
+    gpds.init_master();
+    LuaAssert(L, gp.get_one() == 0x0010000000000000);
+    LuaAssert(L, gp.get_batch() == nthbatch(0));
+    gp.salvage(nthbatch(182));
+    gp.salvage(nthbatch(183));
+    gp.serialize(&sb);
+    gpds.deserialize(&sb);
+    LuaAssert(L, gpds.get_one() == 0x0010000000000001);
+    LuaAssert(L, gpds.get_batch() == nthbatch(183));
+    LuaAssert(L, gpds.get_batch() == nthbatch(182));
+    LuaAssert(L, gpds.get_batch() == nthbatch(1));
+
+    // Serialize and deserialize a player pool.
+    gp.init_master();
+    gpds.init_synch();
+    LuaAssert(L, gp.get_batch() == nthbatch(0));
+    pp.test_clear_ranges();
+    pp.set_fifo_capacity(3);
+    pp.refill();
+    ppds.test_clear_ranges();
+    pp.serialize(&sb);
+    ppds.deserialize(&sb);
+    LuaAssert(L, ppds.get_fifo_capacity()==3);
+    LuaAssert(L, ppds.size() == 3);
+    LuaAssert(L, ranges_equal(ppds.ranges_, nthbatch(1), nthbatch(2), nthbatch(3)));
+
+    // Difference transmit compare two empty pools.
+    gp.init_master();
+    gpds.init_master();
+    pp.test_clear_ranges();
+    ppds.test_clear_ranges();
+    pp.set_fifo_capacity(3);
+    ppds.set_fifo_capacity(3);
+    
+    // Check case: no differences.
+    sb.clear();
+    ppds.diff(pp, &sb);
+    ppds.patch(&sb, nullptr);
+    LuaAssert(L, ppds.exactly_equal(pp));
+
+    // Add some values to master pool
+    pp.test_push_back(123);
+    pp.test_push_back(456);
+
+    // transmit and compare.
+    sb.clear();
+    ppds.diff(pp, &sb);
+    ppds.patch(&sb, nullptr);
+    LuaAssert(L, ppds.exactly_equal(pp));
+
+    // Pop a value from master pool
+    pp.test_pop_front();
+    pp.test_push_back(789);
+
+    // transmit and compare.
+    sb.clear();
+    ppds.diff(pp, &sb);
+    ppds.patch(&sb, nullptr);
+    LuaAssert(L, ppds.exactly_equal(pp));
+
+    return 0;
+}

luprex/cpp/core/idalloc.hpp

@@ -0,0 +1,221 @@
+///////////////////////////////////////////////////////////////////
+//
+// THE ID ALLOCATOR
+//
+// This ID allocator's goal is to allocate IDs in such a way that the
+// synchronous model gets the same IDs as the master model.
+//
+// DESIGN PRINCIPLES
+//
+// There are two classes defined here: IdGlobalPool, and IdPlayerPool.
+//
+// Every logged-in player maintains an IdPlayerPool.  That's basically a fifo
+// queue of ID batches.  An ID batch is a contiguous range of IDs, containing
+// between 128 and 256 contiguous IDs.  The IdPlayerPool is difference
+// transmitted, to ensure that the synchronous model has the same batches as the
+// master model.
+//
+// When a player creates a thread, that thread gets an ID batch from the
+// IdPlayerPool.  To make this possible, the Lua runtime has been modified so
+// that a thread can contain an ID batch.  When that thread allocates IDs, it
+// uses the batch it was allocated.  In the unlikely event that a thread's batch
+// is used up, the thread falls back to using the IdGlobalPool. Such fallback
+// IDs are not likely to be predicted correctly.
+//
+// When a player creates a thread, he uses up one batch from his IdPlayerPool.
+// In the master model, the player pool is 'refilled' by asking the IdGlobalPool
+// to create a new batch.  In the synchronous model, the IdPlayerPool is not
+// directly refilled, but the difference transmitter effectively refills it.  It
+// is imperative that this difference transmission happen before the player's
+// asynchronous model runs out of batches, otherwise we'll get prediction
+// failures.
+//
+// It is common that a thread will only use 1 or 2 IDs.  If a thread exits
+// without using up most of its IDs, then the batch it contains is still a
+// pretty usable batch.  The batch gets returned to the IdGlobalPool, which will
+// eventually use that salvaged batch to satisfy an IdPlayerPool refill request.
+//
+// THE NUMERIC RANGES
+//
+// The largest integer that can be stored losslessly in a lua_Number is:
+//
+//  * 0x0020000000000000
+//
+// In other words, any 53-bit number can be stored.  We subdivide the range as
+// follows:
+//
+//  * 0x0000+  : manually created IDs.
+//  * 0x0001+  : used by master model's IdGlobalPool to create batches.
+//  * 0x0010+  : used by master model's IdGlobalPool to create individual IDs.
+//  * 0x001E+  : used by sync model's IdGlobalPool to create individual IDs.
+//
+// If you exhaust the Master Model's invididual pool at a rate of 10,000,000 IDs
+// per second, the individual pool will last for 12 years.
+//
+// BATCH REPRESENTATION
+//
+// A batch is represented as a 64-bit integer.  The batch contains all IDs
+// starting with that integer, and incrementing, until the the two lowest digits
+// are 0x00. 
+//
+// For example, consider the batch ID 0x11111111111111FC.  That batch includes
+// these IDs:
+//
+//  * 0x11111111111111FC
+//  * 0x11111111111111FD
+//  * 0x11111111111111FE
+//  * 0x11111111111111FF
+//
+// But it does not include 0x1111111111111200.
+//
+// As a special case, the number 0 is used to indicate "invalid batch".
+//
+// SEQUENCE NUMBERS
+//
+// If you allocate IDs from a player pool rapidly, you will exhaust the fifo in
+// the player pool.  As a result, ID allocation will fall back to the global
+// pool, which will cause prediction failures.
+//
+// Depending on your requirements, you may be able to use sequence numbers
+// instead of IDs.  The player pool contains a monotonically increasing counter
+// for generating sequence numbers.  These can be allocated extremely fast
+// and they'll never run out.  These are highly predictable as long as the
+// only person fetching sequence numbers is the player himself.  The downside
+// is that sequence numbers are not globally unique - they're only locally
+// unique for a single player.  That may be enough for your requirements.
+// 
+// Currently, sequence numbers are used by the random number generator.
+//
+///////////////////////////////////////////////////////////////////
+
+#ifndef IDALLOC_HPP
+#define IDALLOC_HPP
+
+#include "wrap-map.hpp"
+#include "wrap-sstream.hpp"
+#include "wrap-deque.hpp"
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+#include "debugcollector.hpp"
+
+#include <cstdint>
+#include <ostream>
+
+class IdGlobalPool : public eng::nevernew {
+public:
+    // Construct and destroy global pools.  Note that after constructing
+    // a global pool, it is generally also necessary to initialize it
+    // for Master or Synchronous operation using init_master or init_synch.
+    // Any attempt to use a pool that hasn't been init_master or init_synch
+    // will fail.
+    IdGlobalPool();
+    ~IdGlobalPool();
+
+    // Initialize the pool for use in a master model.
+    void init_master();
+
+    // Initialize the pool for use in a synchronous model.
+    void init_synch();
+
+    // Create a single unique ID.  Ids allocated this way are
+    // unlikely to be predicted correctly.
+    int64_t get_one();
+
+    // Obtain a batch of IDs from the global pool.  In a master
+    // model, the batch is guaranteed to contain at least 128 IDs.
+    // In a synchronous model, the batch is always zero (invalid).
+    int64_t get_batch();
+
+    // Try to return the specified batch to the global pool.
+    // The salvage operation quietly does nothing if the batch is
+    // zero, or the batch contains fewer than 128 IDs.
+    void salvage(int64_t batch);
+
+    // Get a global sequence number.  These are not the same as IDs.
+    // Sequence numbers are unlikely to be predicted correctly.
+    int64_t get_seqno() { return next_seqno_++; }
+
+    // Serialize to or deserialize from a streambuffer.
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
+    // Generate a debug string.
+    eng::string debug_string() const;
+
+private:
+    eng::vector<int64_t> salvaged_;
+    int64_t next_batch_;
+    int64_t next_id_;
+    int64_t next_seqno_;
+    friend int unittests_idalloc(lua_State *L);
+};
+
+class IdPlayerPool : public eng::nevernew {
+public:
+    // Construct a player pool.
+    //
+    // The fifo is initially in the disabled state.  In the disabled state, the
+    // fifo is not kept filled.  You can still use the allocator when the fifo is
+    // disabled - doing so just fetches a batch from the global pool.
+    //
+    IdPlayerPool(IdGlobalPool *g);
+    ~IdPlayerPool();
+
+    // Set the fifo capacity.  Max=255.
+    void set_fifo_capacity(int n);
+    int get_fifo_capacity() const { return fifo_capacity_; }
+
+    // Return all batches from the fifo to the global pool.
+    void unqueue();
+
+    // Refill the fifo of batches from the global pool.
+    void refill();
+
+    // Get a single ID from the fifo.  Also refills the fifo.
+    int64_t get_one();
+
+    // Get a player sequence number.  This is not the same as an ID: player
+    // sequence numbers are unique per player, but not globally.
+    int64_t get_seqno() { return next_seqno_++; }
+
+    // Return the size of the queue.
+    int size() { return ranges_.size(); }
+
+    // Return true if the two pools are identical.
+    bool exactly_equal(const IdPlayerPool &other) const;
+    
+    // Check that the pool is valid.
+    bool valid() const;
+
+    // unit testing functions.
+    //
+    // These let you manipulate the deque explicitly.  But that's
+    // only useful for unit testing.
+    //
+    void test_push_back(int64_t range);
+    void test_pop_front();
+    void test_clear_ranges();
+
+    // Serialize to or deserialize from a streambuffer.
+    // Caution: the pointer to the global pool is not serialized or deserialized.
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
+    // Difference transmission
+    //
+    void diff(const IdPlayerPool &auth, StreamBuffer *sb) const;
+    void patch(StreamBuffer *sb, DebugCollector *dbc);
+
+    // Debug string.
+    eng::string debug_string() const;
+
+private:
+    IdGlobalPool *global_;
+    int fifo_capacity_;
+    int64_t next_seqno_;
+    eng::deque<int64_t> ranges_;
+    friend int lfn_unittests_idalloc(lua_State *L);
+};
+
+#endif // IDALLOC_HPP
+

luprex/cpp/core/invocation.cpp

@@ -0,0 +1,84 @@
+
+#include "wrap-string.hpp"
+#include "wrap-map.hpp"
+#include "wrap-deque.hpp"
+#include "wrap-sstream.hpp"
+
+#include "invocation.hpp"
+
+const eng::string &InvocationData::get(const eng::string &key) const {
+    static eng::string blank_;
+    auto iter = find(key);
+    if (iter == end()) {
+        return blank_;
+    } else {
+        return iter->second;
+    }
+}
+
+void InvocationData::serialize(StreamBuffer *sb) const {
+    assert(int(size()) < 65536);
+    sb->write_uint16(size());
+    for (const auto &pair : *this) {
+        sb->write_string(pair.first);
+        sb->write_string(pair.second);
+    }
+}
+
+void InvocationData::deserialize(StreamBuffer *sb) {
+    clear();
+    int size = sb->read_uint16();
+    for (int i = 0; i < size; i++) {
+        eng::string key = sb->read_string();
+        eng::string val = sb->read_string();
+        (*this)[key] = val;
+    }
+}
+
+Invocation::Invocation() : kind_(KIND_INVALID), actor_(0), place_(0) {}
+
+Invocation::Invocation(Kind kind, int64_t actor, int64_t place, const eng::string &action, const InvocationData &data) 
+    : kind_(kind), actor_(actor), place_(place), action_(action), data_(data) {}
+
+Invocation::Invocation(Kind kind, int64_t actor, int64_t place, const eng::string &action) 
+    : kind_(kind), actor_(actor), place_(place), action_(action) {}
+
+void Invocation::serialize(StreamBuffer *sb) const {
+    sb->write_uint8(kind_);
+    sb->write_int64(actor_);
+    sb->write_int64(place_);
+    sb->write_string(action_);
+    data_.serialize(sb);
+}
+
+void Invocation::deserialize(StreamBuffer *sb) {
+    kind_ = Kind(sb->read_uint8());
+    actor_ = sb->read_int64();
+    place_ = sb->read_int64();
+    action_ = sb->read_string();
+    data_.deserialize(sb);
+}
+
+eng::string Invocation::debug_string() {
+    eng::ostringstream oss;
+    oss << "inv[";
+    switch (kind_) {
+    case KIND_INVALID: oss << "invalid"; break;
+    case KIND_PLAN: oss << "plan"; break;
+    case KIND_LUA: oss << "lua"; break;
+    case KIND_FLUSH_PRINTS: oss << "flush_prints"; break;
+    case KIND_TICK: oss << "tick"; break;
+    case KIND_LUA_SOURCE: oss << "lua_source"; break;
+    default: oss << "UNKNOWN"; break;
+    }
+    oss << " a=" << actor_;
+    oss << " p=" << place_;
+    if (kind_ != KIND_LUA_SOURCE) {
+        oss << " " << action_;
+    }
+    for (const auto &pair : data_) {
+        oss << " " << pair.first << "=" << pair.second;
+    }
+    oss << "]";
+    return oss.str();
+}

luprex/cpp/core/invocation.hpp

@@ -0,0 +1,58 @@
+#ifndef INVOCATION_HPP
+#define INVOCATION_HPP
+
+#include "wrap-string.hpp"
+#include "wrap-map.hpp"
+#include "wrap-deque.hpp"
+
+#include "streambuffer.hpp"
+
+
+class InvocationData : public eng::map<eng::string, eng::string> {
+public:
+    const eng::string &get(const eng::string &key) const;
+    
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+};
+
+class Invocation : public eng::nevernew {
+public:
+    enum Kind {
+        KIND_INVALID,
+        KIND_PLAN,
+        KIND_LUA,
+        KIND_FLUSH_PRINTS,
+        KIND_TICK,
+        KIND_LUA_SOURCE,
+    };
+
+private:
+    Kind kind_;
+    int64_t actor_;
+    int64_t place_;
+    eng::string action_;
+    InvocationData data_;
+public:
+    Invocation();
+    Invocation(Kind kind, int64_t actor, int64_t place, const eng::string &action, const InvocationData &data);
+    Invocation(Kind kind, int64_t actor, int64_t place, const eng::string &action);
+    
+    bool valid() const { return kind_ != KIND_INVALID; }
+    Kind kind() const { return kind_; }
+    int64_t actor() const { return actor_; }
+    int64_t place() const { return place_; }
+    const eng::string &action() const { return action_; }
+    const InvocationData &data() const { return data_; }
+
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
+    eng::string debug_string();
+};
+
+class InvocationQueue : public eng::deque<Invocation> {
+};
+
+
+#endif // INVOCATION_HPP

luprex/cpp/core/json.cpp

@@ -0,0 +1,728 @@
+#include "json.hpp"
+#include "luastack.hpp"
+#include "util.hpp"
+#include <string_view>
+#include <ostream>
+#include <cmath>
+#include <iomanip>
+#include <cstdio>
+#include <cstdlib>
+
+
+#define NOINDENT_LEVEL 1000
+
+LuaTokenConstant(json_null, "null", "");
+LuaTokenConstant(json_object, "object", "");
+LuaTokenConstant(json_error, "error", "");
+
+static void indent(eng::ostringstream &oss, int level) {
+    if (level < NOINDENT_LEVEL) {
+        oss << std::endl;
+        for (int i = 0; i < level; i++) {
+            oss << "  ";
+        }
+    }
+}
+
+static bool length_exceeded(eng::ostringstream &oss, int maxlen) {
+    return oss.tellp() > maxlen;
+}
+
+template <class... ARGS>
+inline void store_error(eng::ostringstream &oss, const ARGS & ... args) {
+    oss.str("");
+    util::send_to_stream(oss, args...);
+}
+
+static void store_length_error(eng::ostringstream &oss, int maxlen) {
+    store_error(oss, "maximum json length exceeded: ", maxlen);
+}
+
+static bool use_array_representation(lua_State *L) {
+    int top = lua_gettop(L);
+    int nfound = 0;
+    while (true) {
+        lua_rawgeti(L, top, nfound + 1);
+        bool null = lua_isnil(L, -1);
+        lua_settop(L, top);
+        if (null) break;
+        nfound += 1;
+    }
+    return (nfound == lua_nkeys(L, top));
+}
+
+static bool encode_key(lua_State *L, eng::ostringstream &oss);
+static bool encode_value(lua_State *L, eng::ostringstream &oss, int level, int maxlen);
+
+
+// The goal here is to emit a double in such a way that
+// when we read it back in, we get the *exact* same number.
+//
+// In the worst case, you can accomplish this by using 17
+// digits of precision - that's enough to uniquely identify
+// all double values (see the following URL).  However, 17
+// digits tends to produce unnecessary repeating decimals.
+// So we try 16 digits first, which tends to remove those
+// repeating decimals, but sometimes produces losses.
+// If that doesn't work, we fall back to 17 digits.
+//
+// https://randomascii.wordpress.com/2012/03/08/float-precisionfrom-zero-to-100-digits-2/
+//
+static void encode_double_lossless(double value, eng::ostringstream &oss) {
+    char buffer[80];
+    sprintf(buffer, "%.16g", value);
+    if (strtod(buffer, nullptr) != value) {
+        sprintf(buffer, "%.17g", value);
+        assert(strtod(buffer, nullptr) == value);
+    }
+    oss << buffer;
+}
+
+static bool encode_nil(lua_State *L, eng::ostringstream &oss) {
+    oss << "null";
+    return true;
+}
+
+static bool encode_token(lua_State *L, eng::ostringstream &oss) {
+    LuaToken token(lua_touserdata(L, -1));
+    if (token == LuaToken("jsonnull")) {
+        oss << "null";
+        return true;
+    } else {
+        store_error(oss, "cannot encode token: [", token.str(), "]");
+        return false;
+    }
+}
+
+static bool encode_number(lua_State *L, eng::ostringstream &oss) {
+    lua_Number value = lua_tonumber(L, -1);
+    if (std::isnan(value) || std::isinf(value)) {
+        store_error(oss, "cannot encode infinity or NAN");
+        return false;
+    }
+    int64_t ivalue = int64_t(value);
+    if (double(ivalue) == value) {
+        oss << ivalue;
+    } else {
+        encode_double_lossless(value, oss);
+    }
+    return true;
+}
+
+static bool encode_number_key(lua_State *L, eng::ostringstream &oss) {
+    lua_Number value = lua_tonumber(L, -1);
+    int64_t ivalue = int64_t(value);
+    if (double(ivalue) != value) {
+        store_error(oss, "cannot encode floating point numbers in table keys");
+        return false;
+    }
+    if (ivalue >= 0) {
+        oss << "\"\\uE000+" << ivalue << '"';
+    } else {
+        oss << "\"\\uE000-" << -ivalue << '"';
+    }
+    return true;
+}
+
+static bool encode_boolean(lua_State *L, eng::ostringstream &oss) {
+    int flag = lua_toboolean(L, -1);
+    oss << (flag ? "true" : "false");
+    return true;
+}
+
+static bool encode_string(lua_State *L, eng::ostringstream &oss) {
+    size_t len;
+    const char *s = lua_tolstring(L, -1, &len);
+    std::string_view str(s, len);
+    oss << '"';
+    if (sv::valid_utf8(str) && !sv::has_prefix(str, "")) {
+        // Output the string in the straightforward way,
+        // using traditional json escaping.
+        for (char c : str) {
+            switch (c) {
+                case '\\': oss << "\\\\"; break;
+                case '"' : oss << "\\\""; break;
+                case '\b': oss << "\\b"; break;
+                case '\f': oss << "\\f"; break;
+                case '\r': oss << "\\r"; break;
+                case '\n': oss << "\\n"; break;
+                case '\t': oss << "\\t"; break;
+                default: {
+                    if (c < 32) {
+                        oss << "\\u" << util::hex16.val(c);
+                    } else {
+                        oss << c;
+                    }
+                }
+            }
+        }
+    } else {
+        // Output as a base64-encoded string.
+        oss << "\\uE000=";
+        util::base64_encode(str, &oss);
+    }
+    oss << '"';
+    return true;
+}
+
+static bool encode_array(lua_State *L, eng::ostringstream &oss, int level, int maxlen) {
+    lua_checkstack(L, 20);
+    int top = lua_gettop(L);
+    oss << "[";
+    level ++;
+    int i = 1;
+    while (true) {
+        lua_rawgeti(L, top, i);
+        if (lua_isnil(L, -1)) break;
+        if (i > 1) oss << ",";
+        indent(oss, level);
+        bool ok = encode_value(L, oss, level, maxlen);
+        lua_settop(L, top);
+        if (!ok) return false;
+        if (length_exceeded(oss, maxlen)) {
+            store_length_error(oss, maxlen);
+            return false;
+        }
+        i += 1;
+    }
+    lua_settop(L, top);
+    level --;
+    indent(oss, level);
+    oss << "]";
+    return true;
+}
+
+static bool encode_object(lua_State *L, eng::ostringstream &oss, int level, int maxlen) {
+    lua_checkstack(L, 20);
+    int top = lua_gettop(L);
+    oss << "{";
+    level ++;
+    lua_pushnil(L);
+    int i = 1;
+    while (lua_next(L, top) != 0) {
+        // Check for [json.object]=true, if so skip.
+        if (lua_islightuserdata(L, -2) &&
+            lua_isboolean(L, -1) &&
+            (LuaToken(lua_touserdata(L, -2)) == LuaToken("object")) &&
+            (lua_toboolean(L, -1) == 1)) {
+            lua_pop(L, 1);
+            continue;
+        }
+
+        lua_pushvalue(L, -2);
+        // Stack now has key, value, key
+        assert(lua_gettop(L) == top + 3);
+        if (i > 1) oss << ",";
+        indent(oss, level);
+        bool ok = encode_key(L, oss);
+        if (!ok) {
+            lua_settop(L, top);
+            return false;
+        }
+        if (length_exceeded(oss, maxlen)) {
+            store_length_error(oss, maxlen);
+            lua_settop(L, top);
+            return false;
+        }
+        lua_pop(L, 1);
+        // Stack now has key, value
+        assert(lua_gettop(L) == top + 2);
+        oss << ((level < NOINDENT_LEVEL) ? " : " : ":");
+        ok = encode_value(L, oss, level, maxlen);
+        assert(lua_gettop(L) == top + 2);
+        if (!ok) {
+            lua_settop(L, top);
+            return false;
+        }
+        if (length_exceeded(oss, maxlen)) {
+            store_length_error(oss, maxlen);
+            lua_settop(L, top);
+            return false;
+        }
+        lua_pop(L, 1);
+        // Stack now just has key.
+        assert(lua_gettop(L) == top + 1);
+        i += 1;
+    }
+    // Stack should be back to where we started.
+    assert(lua_gettop(L) == top);
+    level --;
+    indent(oss, level);
+    oss << "}";
+    return true;
+}
+
+static bool encode_key(lua_State *L, eng::ostringstream &oss) {
+    int type = lua_type(L, -1);
+    switch (type) {
+        case LUA_TSTRING: return encode_string(L, oss);
+        case LUA_TNUMBER: return encode_number_key(L, oss);
+        case LUA_TBOOLEAN:
+        case LUA_TTABLE: {
+            store_error(oss, "cannot encode '", lua_typename(L, type), "' in table keys");
+            return false;
+        }
+        default: {
+            store_error(oss, "cannot encode '", lua_typename(L, type), "'");
+            return false;
+        }
+    }
+}
+
+static bool encode_value(lua_State *L, eng::ostringstream &oss, int level, int maxlen) {
+    int type = lua_type(L, -1);
+    switch (type) {
+        case LUA_TNIL: return encode_nil(L, oss);
+        case LUA_TNUMBER: return encode_number(L, oss);
+        case LUA_TBOOLEAN: return encode_boolean(L, oss);
+        case LUA_TSTRING: return encode_string(L, oss);
+        case LUA_TLIGHTUSERDATA: return encode_token(L, oss);
+        case LUA_TTABLE: {
+            if (use_array_representation(L)) {
+                return encode_array(L, oss, level, maxlen);
+            } else {
+                return encode_object(L, oss, level, maxlen);
+            }
+        }
+        default: {
+            store_error(oss, "cannot encode '", lua_typename(L, type), "'");
+            return false;
+        }
+    }
+}
+
+static bool decode_value(lua_State *L, std::string_view &v);
+
+static bool decode_id(lua_State *L, std::string_view &v) {
+    std::string_view id = sv::read_ascii_identifier(v);
+    if (id == "null") lua_pushlightuserdata(L, LuaToken("null").voidvalue());
+    else if (id == "true") lua_pushboolean(L, 1);
+    else if (id == "false") lua_pushboolean(L, 0);
+    else return false;
+    return true;
+}
+
+static bool decode_number(lua_State *L, std::string_view &v) {
+    std::string_view n = sv::read_number(v, true, true, true, true);
+    if (n.empty()) return false;
+
+    // If it's an integer, make sure it fits in a lua double
+    // losslessly.  If it's a double, some loss in precision
+    // is OK.
+    if (sv::valid_number(n, true, true, false, false)) {
+        int64_t i = sv::to_int64(n);
+        if (!LuaStack::int64_storable(i)) return false;
+        lua_pushnumber(L, double(i));
+        return true;
+    } else {
+        double d = sv::to_double(n);
+        if (std::isnan(d) || std::isinf(d)) return false;
+        lua_pushnumber(L, d);
+        return true;
+    }
+}
+
+static bool decode_base64_string(lua_State *L, std::string_view &v) {
+    // We've already read the starting quote and the E000
+    // escape sequence at this point.
+
+    // Skip the equal sign.
+    if (!sv::read_prefix(v, "=")) return false;
+    
+    // Find the end of the quoted string.
+    const char *p = v.data();
+    const char *l = p + v.size();
+    while (true) {
+        if (p == l) return false;
+        if (*p < 32) return false;
+        if (*p == '"') break;
+        p++;
+    }
+    std::string_view b64 = v.substr(0, p - v.data());
+    v.remove_prefix(b64.size() + 1);
+    eng::ostringstream oss;
+    if (!util::base64_decode(b64, &oss)) return false;
+    eng::string str = oss.str();
+    lua_pushlstring(L, str.c_str(), str.size());
+    return true;
+}
+
+static bool decode_int_string(lua_State *L, std::string_view &v) {
+    // We've already read the starting quote and the E000
+    // escape sequence at this point.
+
+    // Parse the number and the closing quote.
+    std::string_view n = sv::read_number(v, true, true, false, false);
+    if (n.empty()) return false;
+    if (!sv::read_prefix(v, "\"")) {
+        return false;
+    }
+
+    // Make sure the number fits in a lua double,
+    // and push it on the stack.
+    int64_t i = sv::to_int64(n);
+    if (!LuaStack::int64_storable(i)) {
+        return false;
+    }
+    lua_pushnumber(L, double(i));
+    return true;
+}
+
+static bool decode_standard_string(lua_State *L, std::string_view &v) {
+    // We've already read the starting quote at this point.
+    eng::ostringstream oss;
+    while (true) {
+        // Get the next codepoint.
+        int32_t c = sv::read_codepoint_utf8(v);
+
+        // If it's a control character or invalid codepoint, reject.
+        if (c < 32) return false;
+
+        // If it is an unescaped quote, that's end of string.
+        if (c == '"') break;
+
+        // If it's a backslash, then deal with the escape sequence.
+        if (c == '\\') {
+            char next = sv::read_ascii_char(v);
+            switch (next) {
+                case '"': oss << '"'; break;
+                case '\\': oss << '\\'; break;
+                case '/': oss << '/'; break;
+                case 'r': oss << '\r'; break;
+                case 'n': oss <<'\n'; break;
+                case 'b': oss << '\b'; break;
+                case 'f': oss << '\f'; break;
+                case 't': oss << '\t'; break;
+                case 'u': {
+                    std::string_view hexdigits = sv::read_nbytes(v, 4);
+                    if (hexdigits.size() != 4) return false;
+                    uint64_t codepoint = sv::to_hex64(hexdigits, 0x10000);
+                    if (codepoint >= 0x10000) return false;
+                    if (!util::write_codepoint_utf8(codepoint, &oss)) return false;
+                    break;
+                }
+                default: return false;
+            }
+            continue;
+        }
+
+        // Any other codepoint should be echoed into stream.
+        util::write_codepoint_utf8(c, &oss);
+    }
+    eng::string result = oss.str();
+    lua_pushlstring(L, result.c_str(), result.size());
+    return true;
+}
+
+static bool decode_string(lua_State *L, std::string_view &v) {
+    if (!sv::read_prefix(v, "\"")) return false;
+
+    // Check for codepoint E000, the escape sequence.
+    if (sv::read_prefix(v, "") ||
+        sv::read_prefix(v, "\\uE000") ||
+        sv::read_prefix(v, "\\ue000")) {
+        char c = sv::zfront(v);
+        if (c == '=') return decode_base64_string(L, v);
+        else if ((c=='-') || (c=='+')) return decode_int_string(L, v);
+        else return false;
+    } else {
+        return decode_standard_string(L, v);
+    }
+}
+
+static bool decode_array(lua_State *L, std::string_view &v) {
+    if (!sv::read_prefix(v, "[")) return false;
+    lua_newtable(L);
+    int tabpos = lua_gettop(L);
+    int next = 1;
+    while (true) {
+        v = sv::ltrim(v);
+        if (sv::zfront(v) == ']') {
+            v.remove_prefix(1);
+            return true;
+        }
+        if (!decode_value(L, v)) {
+            return false;
+        }
+        v = sv::ltrim(v);
+        if (sv::zfront(v) == ',') {
+            v.remove_prefix(1);
+        }
+        lua_rawseti(L, tabpos, next++);
+    }
+}
+
+static bool decode_object(lua_State *L, std::string_view &v) {
+    if (!sv::read_prefix(v, "{")) return false;
+    lua_newtable(L);
+    int tabpos = lua_gettop(L);
+    while (true) {
+        v = sv::ltrim(v);
+        if (sv::zfront(v) == '}') {
+            v.remove_prefix(1);
+            return true;
+        }
+        if (!decode_string(L, v)) {
+            return false;
+        }
+        v = sv::ltrim(v);
+        if (!sv::read_prefix(v, ":")) {
+            return false;
+        }
+        if (!decode_value(L, v)) {
+            return false;
+        }
+        v = sv::ltrim(v);
+        if (sv::zfront(v) == ',') {
+            v.remove_prefix(1);
+        }
+        lua_rawset(L, tabpos);
+    }
+}
+
+// Decode a single value.
+//
+// On success, pushes the value on the stack and returns true.
+// On failure, pushes NIL on the stack and returns false.
+//
+static bool decode_value(lua_State *L, std::string_view &v) {
+    lua_checkstack(L, 20);
+    int top = lua_gettop(L);
+
+    // Skip blanks.
+    v = sv::ltrim(v);
+
+    // Try to read something.
+    char c = sv::zfront(v);
+    bool result;
+    if (c == '"') result = decode_string(L, v);
+    else if (c == '[') result = decode_array(L, v);
+    else if (c == '{') result = decode_object(L, v);
+    else if (sv::ascii_isalpha(c)) result = decode_id(L, v);
+    else result = decode_number(L, v);
+    
+    // On failure, the decode routines may leave junk
+    // on the stack, in which case it's our job to clean up.
+    if (result == false) {
+        lua_settop(L, top);
+        lua_pushnil(L);
+    }
+
+    // Now there should be exactly one new value on the stack.
+    assert(lua_gettop(L) == top + 1);
+    return result;
+}
+
+namespace json {
+
+eng::string encode(LuaStack &LS, LuaSlot in, eng::string &out, bool indent, int maxlen) {
+    eng::ostringstream oss;
+
+    // Call the recursive encoder.  Clean up any crap on the lua stack afterward.
+    int top = lua_gettop(LS.state());
+    lua_pushvalue(LS.state(), in.index());
+    bool ok = encode_value(LS.state(), oss, indent ? 0 : NOINDENT_LEVEL, maxlen);
+    lua_settop(LS.state(), top);
+
+    // One last check for overruns.
+    if (ok && length_exceeded(oss, maxlen)) {
+        store_length_error(oss, maxlen);
+        ok = false;
+    }
+
+    // Produce the return value.
+    if (ok) {
+        out = oss.str();
+        return "";
+    } else {
+        out = "";
+        return oss.str();
+    }
+}
+
+bool decode(LuaStack &LS, LuaSlot out, std::string_view v) {
+    lua_State *L = LS.state();
+
+    // Try to read a single value from the view.
+    int top = lua_gettop(L);
+    bool ok = decode_value(L, v);
+    lua_replace(L, out.index());
+    lua_settop(L, top);
+    if (!ok) {
+        LS.set(out, LuaToken("error"));
+        return false;
+    }
+
+    // Special case: if the top level value is 'null', change
+    // it to 'nil.'
+    if (LS.istoken(out)) {
+        LS.set(out, LuaNil);
+    }
+
+    // There should be nothing left of the input text.
+    if (v.size() > 0) {
+        LS.set(out, LuaToken("error"));
+        return false;
+    }
+    return true;
+}
+
+} // namespace util
+
+
+LuaDefine(json_encode, "data, indent, maxlen",
+    "|Encode a lua data structure returning a json string."
+    "|"
+    "|Data is the value being encoded.  Indent is a flag,"
+    "|if it's true, then the json is indented nicely,"
+    "|otherwise, it is packed tightly.  Maxlen is the maximum"
+    "|length in bytes of the encoded json string."
+    "|"
+    "|Usually, Lua data translates straightforwardly to json."
+    "|However, there are a number of special cases to be"
+    "|aware of:"
+    "|"
+    "|- Closures and threads cannot be encoded.  These will"
+    "|  cause the encoder to abort."
+    "|"
+    "|- The numbers infinity and NAN cannot be encoded."
+    "|  Both of these will cause the encoder to abort."
+    "|"
+    "|- You must specify a size-limit to the encoded"
+    "|  string.  Exceeding the size limit causes the"
+    "|  encoder to abort."
+    "|"
+    "|- Recursive data structures will cause the encoder to"
+    "|  loop infinitely until the size-limit is exceeded,"
+    "|  causing the encoder to abort."
+    "|"
+    "|- There is no way to represent math.huge or math.nan in"
+    "|  json.  Encoding math.nan will cause the encoder to abort,"
+    "|  as expected.  However, encoding math.huge will emit null,"
+    "|  which is probably not what you would expect."
+    "|"
+    "|- Lua tables cannot contain 'nil', but json objects and"
+    "|  arrays can contain null.  If you want the encoder to"
+    "|  emit a json object or array containing null, you must"
+    "|  use token json.null to represent null."
+    "|"
+    "|- Json objects, like lua tables, are key-value stores."
+    "|  However, json objects can only have string keys.  Our"
+    "|  encoder uses a workaround to transparently"
+    "|  allow mixing string and integer keys in json tables."
+    "|  See 'encoding  difficult data' below."
+    "|"
+    "|- Json strings are required to be valid utf-8.  Our encoder"
+    "|  uses a workaround to transparently allow the use of"
+    "|  arbitrary 8-bit-clean strings.  See 'encoding difficult"
+    "|  data' below."
+    "|"
+    "|- Lua tables containing contiguous integer keys from 1-n are"
+    "|  autodetected to be json arrays.  Empty tables are also"
+    "|  emitted as json arrays.  All other tables are emitted"
+    "|  as json objects."
+    "|"
+    "|- You can force a table to be emitted as a json object"
+    "|  by putting the key-value pair table[json.object]=true"
+    "|  into the table.  This special key is not emitted, but"
+    "|  it triggers json object mode.  This is the only way"
+    "|  to emit an empty json object (a truly empty table is"
+    "|  emitted as a json array.)"
+    "|"
+    "|Encoding Difficult Data:"
+    "|"
+    "|Normally, json doesn't allow integer table keys, and it"
+    "|doesn't allow strings that aren't valid utf-8.  Our"
+    "|json encoder and decoder, on the other hand, can"
+    "|encode and decode integer table keys and 8-bit-clean"
+    "|strings transparently.  This is accomplished without"
+    "|violating the json specification, by encoding such"
+    "|values as utf-8 strings:"
+    "|"
+    "|    '123'       (encoded integer 123)"
+    "|    '=aGVsbG8=' (binary string encoded as base64)"
+    "|"
+    "|Those encodings start with utf-8 codepoint E000."
+    "|This codepoint probably shows up in your text editor"
+    "|as a little rectangle.  When the decoder sees codepoint"
+    "|E000 at the beginning of a string, it automatically"
+    "|decodes the string back into its original form."
+    "|"
+    "|The one price for this behavior is that the encoder"
+    "|cannot literally emit strings that start"
+    "|with codepoint E000.  If the encoder detects such a"
+    "|string, it will emit it as a base64-encoded string."
+    "|This should be uncommon, since codepoint E000 is"
+    "|reserved."
+    "|"
+    "|Note that integers are only encoded when they are"
+    "|used as table keys.  Otherwise, numbers are emitted"
+    "|straightforwardly."
+    "|") {
+    LuaArg data, indent, maxlen;
+    LuaRet encoded;
+    LuaStack LS(L, data, indent, maxlen, encoded);
+    eng::string out;
+    eng::string error = json::encode(LS, data, out, LS.ckboolean(indent), LS.ckint(maxlen));
+    if (!error.empty()) {
+        luaL_error(L, "%s", error.c_str());
+        LS.set(encoded, LuaNil);
+        return LS.result();
+    } else {
+        LS.set(encoded, out);
+        return LS.result();
+    }
+}
+
+LuaDefine(json_decode, "data",
+    "|Decode a json expression into a lua data structure."
+    "|"
+    "|Data that was generated by our own encoder is almost"
+    "|8-bit clean.  That includes difficult cases, like"
+    "|binary strings, floating point numbers, and tables"
+    "|with mixed string and integer keys.  The exception"
+    "|are the kinds of data that can't be encoded at all:"
+    "|See doc(json.encode) for details about what"
+    "|can and cannot be encoded."
+    "|"
+    "|Some json may contain 'null' inside objects and"
+    "|arrays.  Lua tables can't store nil, so instead, we"
+    "|store the token json.null.  If that's not what you"
+    "|want, you can use json.stripnulls to strip out"
+    "|the json.null values from a data structure and"
+    "|replace them with nil."
+    "|"
+    "|") {
+    LuaArg encoded;
+    LuaRet data;
+    LuaStack LS(L, encoded, data);
+    std::string_view v = LS.ckstringview(encoded);
+    bool ok = json::decode(LS, data, v);
+    if (!ok) {
+        luaL_error(L, "invalid json string.");
+    }
+    return LS.result();
+}
+
+// LuaDefine(base64_encode, "data", "") {
+//     LuaArg str;
+//     LuaRet ret;
+//     LuaStack LS(L, str, ret);
+//     eng::string cstr = LS.ckstring(str);
+//     eng::ostringstream oss;
+//     util::base64_encode(cstr, &oss);
+//     LS.set(ret, oss.str());
+//     return LS.result();
+// }
+
+// LuaDefine(base64_decode, "data", "") {
+//     LuaArg str;
+//     LuaRet ret;
+//     LuaStack LS(L, str, ret);
+//     eng::string cstr = LS.ckstring(str);
+//     eng::ostringstream oss;
+//     util::base64_decode(cstr, &oss);
+//     LS.set(ret, oss.str());
+//     return LS.result();
+// }
+

luprex/cpp/core/json.hpp

@@ -0,0 +1,34 @@
+// Encode lua data structure into json, and decode them again.
+//
+// See the doc(http.jsonencode) to read about limitations of the encoder.
+//
+#ifndef JSON_HPP
+#define JSON_HPP
+
+#include "luastack.hpp"
+#include "wrap-string.hpp"
+#include <string_view>
+
+namespace json {
+    // Encode json.
+    //
+    // See doc(http.jsonencode) for a lot more information.
+    //
+    // Returns an error message.  If the error message is an
+    // empty string, then the encoding was successful.
+    //
+    eng::string encode(LuaStack &LS, LuaSlot in, eng::string &out, bool indent, int maxlen);
+
+    // Decode json.
+    //
+    // See doc(http.jsondecode) for a lot more information.
+    //
+    // The only error condition is syntactically invalid json.
+    // In that case, we return false and set 'out' to the
+    // token 'error'.
+    //
+    bool decode(LuaStack &LS, LuaSlot out, std::string_view in);
+}
+    
+#endif // JSON_HPP
+

luprex/cpp/core/lpxclient.cpp

@@ -0,0 +1,283 @@
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+
+#include "drivenengine.hpp"
+#include "world.hpp"
+#include "luaconsole.hpp"
+#include "invocation.hpp"
+#include "util.hpp"
+#include "printbuffer.hpp"
+
+#include <memory>
+
+class LpxClient : public DrivenEngine {
+public:
+    using StringVec = LuaConsole::StringVec;
+    UniqueWorld world_;
+    int64_t actor_id_;
+    InvocationQueue unack_;
+    SharedChannel channel_;
+    LuaConsole console_;
+    PrintChanneler print_channeler_;
+    Gui gui_;
+
+public:
+    void set_initial_state() {
+        // Create the world model.
+        world_.reset(new World(util::WORLD_TYPE_C_SYNC));
+
+        // This is a temporary actor that will be used only until the server sends
+        // us the first difference transmission.  We do this only to establish
+        // the invariant that there's always an actor.  When the first difference
+        // transmission arrives, this actor may be deleted, or it may just be
+        // ignored, at the server's discretion.
+        actor_id_ = world_->create_login_actor();
+
+        // Clear the unack command queue.
+        unack_.clear();
+    }
+
+
+    // When the world is in synchronous mode, there's no
+    // snapshot, and the commands in the unack queue are not
+    // reflected in the world.  In asynchronous mode, there is
+    // a snapshot that allows return to the synchronous mode,
+    // and the unack commands are in the world model.
+
+    void world_to_synchronous() {
+        if (!world_->snapshot_empty()) {
+            world_->rollback();
+        }
+    }
+
+    void world_to_asynchronous() {
+        if (world_->snapshot_empty()) {
+            world_->snapshot();
+            for (const Invocation &inv : unack_) {
+                world_->invoke(inv);
+            }
+        }
+    }
+
+    void abandon_server() {
+        stdostream() << "Abandoning server." << std::endl;
+
+        // Put the world model back into a known-good state.
+        set_initial_state();
+
+        // Disconnect from the server.
+        channel_.reset();
+    }
+
+    virtual void event_init(int argc, char *argv[]) {
+        // Put the world into the starting state.
+        set_initial_state();
+
+        // Establish a connection to the server.
+        channel_ = new_outgoing_channel("nocert:localhost:8085");
+
+        // Set the console prompt
+        get_stdio_channel()->set_prompt(console_.get_prompt());
+
+        // The driver loads the lua source automatically.
+        // However, we don't need it.  Throw it out.
+        get_lua_source();
+    }
+
+    void send_invocation(const Invocation &inv) {
+        if (channel_ == nullptr) {
+            stdostream() << "Cannot invoke any actions, not connected." << std::endl;
+            return;
+        }
+        world_to_asynchronous();
+        world_->invoke(inv);
+        unack_.push_back(inv);
+        StreamBuffer *sb = channel_->out();
+        sb->write_uint8(util::MSG_INVOKE);
+        inv.serialize(sb);
+    }
+
+    void send_lua_source(const util::LuaSourceVec &sv) {
+        StreamBuffer serial;
+        SourceDB::serialize_source(sv, &serial);
+        eng::string sstr = serial.read_entire_contents();
+        Invocation inv(Invocation::KIND_LUA_SOURCE, actor_id_, actor_id_, sstr);
+        send_invocation(inv);
+    }
+
+    void do_luainvoke_command(const StringVec &words) {
+        send_invocation(Invocation(Invocation::KIND_LUA, actor_id_, actor_id_, words[1]));
+    }
+
+    void do_luaprobe_command(const StringVec &words) {
+        world_to_asynchronous();
+        stdostream() << world_->probe_lua(actor_id_, words[1]);
+        world_to_synchronous();
+    }
+
+    void do_syntax_command(const StringVec &words) {
+        stdostream() << "Syntax Error: " << words[1] << std::endl;
+    }
+
+    void do_view_command(const StringVec &cmd) {
+        stdostream() << world_->tangibles_near_debug_string(actor_id_, 100);
+    }
+
+    void do_menu_command(const StringVec &cmd) {
+        world_to_asynchronous();
+        int64_t place = sv::to_int64(cmd[1], actor_id_);
+        world_->update_gui(actor_id_, place, &gui_);
+        stdostream() << gui_.menu_debug_string();
+    }
+
+    void do_choose_command(const StringVec &cmd) {
+        eng::string action = gui_.get_action(sv::to_int64(cmd[1]));
+        if (action == "") {
+            stdostream() << "Invalid menu item #" << std::endl;
+            return;
+        }
+        stdostream() << "Invoking plan: " << action << std::endl;
+        Invocation inv(Invocation::KIND_PLAN, actor_id_, gui_.place(), action);
+        send_invocation(inv);
+    }
+
+    void do_tick_command(const util::StringVec &words) {
+        send_invocation(Invocation(Invocation::KIND_TICK, actor_id_, actor_id_, ""));
+    }
+
+    void do_cpl_command(const util::StringVec &words) {
+        rescan_lua_source();
+    }
+
+    void do_work_command(const util::StringVec &words) {
+        int reps = 10000;
+        for (int i = 0; i < reps; i++) {
+            world_to_synchronous();
+            world_to_asynchronous();
+        }
+    }
+
+    void do_quit_command(const util::StringVec &words) {
+        abandon_server();
+        stop_driver();
+    }
+
+    void do_command(const util::StringVec &words) {
+        if (words.empty()) return;
+        else if (words[0] == "luainvoke") do_luainvoke_command(words);
+        else if (words[0] == "luaprobe") do_luaprobe_command(words);
+        else if (words[0] == "syntax") do_syntax_command(words);
+        else if (words[0] == "view") do_view_command(words);
+        else if (words[0] == "menu") do_menu_command(words);
+        else if (words[0] == "choose") do_choose_command(words);
+        else if (words[0] == "tick") do_tick_command(words);
+        else if (words[0] == "cpl") do_cpl_command(words);
+        else if (words[0] == "work") do_work_command(words);
+        else if (words[0] == "quit") do_quit_command(words);
+        else {
+            stdostream() << "Unsupported command: " << words[0] << std::endl;
+        }
+    }
+
+    void change_actor_id(int64_t actor_id) {
+        stdostream() << "Actor ID changing: " << actor_id << std::endl;
+        print_channeler_.reset();
+        actor_id_ = actor_id;
+    }
+
+    void receive_ack_from_server(StreamBuffer *sb) {
+        // An ack is just a single byte, so there's nothing left to read.
+        if (unack_.empty()) {
+            // Invalid acknowledgement when theres' nothing in the unack queue.
+            abandon_server();
+            return;
+        }
+        world_to_synchronous();
+        world_->invoke(unack_.front());
+        unack_.pop_front();
+    }
+
+    void receive_diff_from_server(StreamBuffer *sb) {
+        world_to_synchronous();
+        try {
+  	    DebugCollector dbc("");
+            int64_t nactor = world_->patch_everything(sb, &dbc);
+            if (nactor != actor_id_) change_actor_id(nactor);
+            dbc.dump(stdostream());
+        } catch (const StreamEof &seof) {
+            abandon_server();
+            return;
+        } catch (const StreamCorruption &scorr) {
+            abandon_server();
+            return;
+        }
+    }
+
+    bool receive_message_from_server(StreamBuffer *sb) {
+        if (sb->fill() < 5) return false;
+        int64_t tr_before = sb->total_reads();
+        uint8_t message_type = sb->read_uint8();
+        uint32_t message_body_len = sb->read_uint32();
+        if (sb->fill() < message_body_len) {
+            sb->unread_to(tr_before);
+            return false;
+        }
+        StreamBuffer body(sb->read_bytes(message_body_len), message_body_len);
+        if (message_type == util::MSG_ACK) {
+            receive_ack_from_server(&body);
+        } else if (message_type == util::MSG_DIFF) {
+            receive_diff_from_server(&body);
+        } else {
+            abandon_server();
+            return false;
+        }
+        if (!body.empty()) {
+            abandon_server();
+            return false;
+        }
+        return true;
+    }
+
+    virtual void event_update() {
+        // Check for lua source code.  If this returns non-null,
+        // it is because somebody typed CPL.
+        util::LuaSourcePtr lua_source = get_lua_source();
+        if (lua_source != nullptr) {
+            send_lua_source(*lua_source);
+            lua_source.reset();
+        }
+
+        // Check for keyboard input on stdin.
+        while (true) {
+            eng::string line = get_stdio_channel()->in()->readline();
+            if (line == "") break;
+            console_.add(line);
+            get_stdio_channel()->set_prompt(console_.get_prompt());
+            do_command(console_.get_command());
+        }
+
+        // Check for communication from server..
+        if (channel_ != nullptr) {
+            if (channel_->closed()) {
+                stdostream() << "server closed connection: " << channel_->error() << std::endl;
+                abandon_server();
+            } else {
+                while (true) {
+                    if (!receive_message_from_server(channel_->in())) break;
+                    if (channel_ == nullptr) break;
+                }
+                world_to_asynchronous();
+            }
+        }
+
+        // Channel print statements.
+        if (print_channeler_.channel(world_->get_printbuffer(actor_id_), stdostream())) {
+            if (channel_ != nullptr) {
+                send_invocation(print_channeler_.invocation(actor_id_));
+            }
+        }
+    }
+};
+
+DrivenEngineDefine("lpxclient", LpxClient);
+

luprex/cpp/core/lpxclient.hpp

@@ -0,0 +1,4 @@
+#ifndef LPXCLIENT_HPP
+#define LPXCLIENT_HPP
+
+#endif // LPXCLIENT_HPP

luprex/cpp/core/lpxserver.cpp

@@ -0,0 +1,272 @@
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+
+#include "world.hpp"
+#include "drivenengine.hpp"
+#include "luaconsole.hpp"
+#include "util.hpp"
+#include "printbuffer.hpp"
+
+#include <memory>
+
+class Client : public eng::opnew {
+public:
+    int64_t actor_id_;
+    SharedChannel channel_;
+    UniqueWorld sync_;
+};
+using UniqueClient = std::unique_ptr<Client>;
+using ClientVector = eng::vector<UniqueClient>;
+
+class LpxServer : public DrivenEngine {
+public:
+    using StringVec = LuaConsole::StringVec;
+    UniqueWorld master_;
+    LuaConsole console_;
+    ClientVector clients_;
+    PrintChanneler print_channeler_;
+    HttpChannelMap http_client_channels_;
+    HttpChannelVec http_server_channels_;
+    int64_t admin_id_;
+    Gui gui_;
+
+public:
+    virtual void event_init(int argc, char *argv[]) {
+        // Create the master world model.
+        master_.reset(new World(util::WORLD_TYPE_MASTER));
+
+        // Update the source code of the master model.
+        master_->update_source(get_lua_source());
+
+        // Create an actor for administrative commands.
+        admin_id_ = master_->create_login_actor();
+
+        // Print out admin ID for debugging purposes.
+        stdostream() << "Admin actor id = " << admin_id_ << std::endl;
+
+        // Enable listening on port 8085 (client connections)
+        listen_port(8085);
+
+        // Enable listening on port 8080 (http server connections)
+        listen_port(8080);
+
+        // Set the console prompt.
+        get_stdio_channel()->set_prompt(console_.get_prompt());
+    }
+
+    void do_luainvoke_command(const util::StringVec &words) {
+        master_->invoke(Invocation(Invocation::KIND_LUA, admin_id_, admin_id_, words[1]));
+    }
+
+    void do_luaprobe_command(const util::StringVec &words) {
+        master_->snapshot();
+        stdostream() << master_->probe_lua(admin_id_, words[1]);;
+        master_->rollback();
+    }
+
+    void do_syntax_command(const util::StringVec &words) {
+        stdostream() << "Syntax Error: " << words[1] << std::endl;
+    }
+
+    void do_menu_command(const StringVec &cmd) {
+        int64_t place = sv::to_int64(cmd[1], admin_id_);
+        master_->update_gui(admin_id_, place, &gui_);
+        stdostream() << gui_.menu_debug_string();
+    }
+
+    void do_choose_command(const StringVec &cmd) {
+        eng::string action = gui_.get_action(sv::to_int64(cmd[1]));
+        if (action == "") {
+            stdostream() << "Invalid menu item #" << std::endl;
+            return;
+        }
+        stdostream() << "Invoking plan: " << action << std::endl;
+        master_->invoke(Invocation(Invocation::KIND_PLAN, admin_id_, gui_.place(), action));
+    }
+
+    void do_tick_command(const util::StringVec &words) {
+        master_->invoke(Invocation(Invocation::KIND_TICK, admin_id_, admin_id_, ""));
+    }
+
+    void do_cpl_command(const util::StringVec &words) {
+        rescan_lua_source();
+    }
+
+    void do_quit_command(const util::StringVec &words) {
+        stop_driver();
+    }
+
+    void do_aborthttp_command(const util::StringVec &words) {
+        master_->abort_all_http_requests(425, "http requests aborted from the server command line");
+    }
+
+    void do_command(const util::StringVec &words) {
+        if (words.empty()) return;
+        else if (words[0] == "luainvoke") do_luainvoke_command(words);
+        else if (words[0] == "luaprobe") do_luaprobe_command(words);
+        else if (words[0] == "syntax") do_syntax_command(words);
+        else if (words[0] == "menu") do_menu_command(words);
+        else if (words[0] == "choose") do_choose_command(words);
+        else if (words[0] == "tick") do_tick_command(words);
+        else if (words[0] == "cpl") do_cpl_command(words);
+        else if (words[0] == "quit") do_quit_command(words);
+        else if (words[0] == "aborthttp") do_aborthttp_command(words);
+        else {
+            stdostream() << "Unsupported command: " << words[0] << std::endl;
+        }
+    }
+
+    void delete_client(UniqueClient &client) {
+        stdostream() << "Client closed: actor id=" << client->actor_id_ << std::endl;
+        client.reset();
+    }
+
+    void send_diffs(UniqueClient &client) {
+        StreamBuffer *sb = client->channel_->out();
+        sb->write_uint8(util::MSG_DIFF);
+        sb->write_uint32(0);
+        int64_t tw_1 = sb->total_writes();
+        stdostream() << "Sending diffs to client " << client->actor_id_ << std::endl;
+        client->sync_->diff_everything(client->actor_id_, master_.get(), sb);
+        int64_t tw_2 = sb->total_writes();
+        sb->overwrite_int32(tw_1, tw_2 - tw_1);
+    }
+
+    bool handle_invocation(UniqueClient &client) {
+        StreamBuffer *sb = client->channel_->in();
+        int64_t tr_before = sb->total_reads();
+        Invocation inv;
+        try {
+            uint8_t msg_type = sb->read_uint8();
+            if (msg_type != util::MSG_INVOKE) {
+                delete_client(client);
+                return false;
+            }
+            inv.deserialize(sb);
+        } catch (const StreamEof &seof) {
+            sb->unread_to(tr_before);
+            return false;
+        } catch (const StreamCorruption &scorr) {
+            delete_client(client);
+            return false;
+        }
+        if (inv.actor() != client->actor_id_) {
+            stdostream() << "Ignoring invoke with wrong actor ID " << inv.actor() << std::endl;
+            return true;
+        }
+        stdostream() << "Invoking: " << inv.debug_string() << std::endl;
+        master_->invoke(inv);
+        client->channel_->out()->write_uint8(util::MSG_ACK);
+        client->channel_->out()->write_uint32(0);
+        client->sync_->invoke(inv);
+        send_diffs(client);
+        return true;
+    }
+    
+    virtual void event_update() {
+        // If the driver has reloaded the source, put it into master model.
+        master_->update_source(get_lua_source());
+
+        // Check for keyboard input on stdin.
+        while (true) {
+            eng::string line = get_stdio_channel()->in()->readline();
+            if (line == "") break;
+            console_.add(line);
+            get_stdio_channel()->set_prompt(console_.get_prompt());
+            do_command(console_.get_command());
+        }
+
+        // Anything in the administrator printbuffer should go to stdostream.
+        if (print_channeler_.channel(master_->get_printbuffer(admin_id_), stdostream())) {
+            master_->invoke(print_channeler_.invocation(admin_id_));
+        }
+
+        // Check for new incoming channels, set up client structures.
+        while (true) {
+            SharedChannel chan = new_incoming_channel();
+            if (chan == nullptr) break;
+            if (chan->port() == 8085) {
+                Client *client = new Client;
+                client->actor_id_ = master_->create_login_actor();
+                client->channel_ = std::move(chan);
+                client->sync_.reset(new World(util::WORLD_TYPE_S_SYNC));
+                client->sync_->create_login_actor();
+                clients_.emplace_back(client);
+                stdostream() << "New client: actor id=" << client->actor_id_ << std::endl;
+                send_diffs(clients_.back());
+            } else if (chan->port() == 8080) {
+                HttpChannel htchan;
+                htchan.channel_ = chan;
+                http_server_channels_.push_back(htchan);
+            }
+        }
+
+        // Traverse all existing channels, process any communication.
+        for (UniqueClient &client : clients_) {
+            if (client->channel_->closed()) {
+                delete_client(client);
+                continue;
+            }
+            
+            // Check for received invocations.
+            while (handle_invocation(client));
+        }
+        util::remove_nullptrs(clients_);
+
+        // Look for new outgoing HTTP client requests.
+        for (const auto &pair : master_->http_requests()) {
+            const HttpClientRequest &request = pair.second;
+            HttpChannel &channel = http_client_channels_[request.request_id()];
+            if (channel.channel_ == nullptr) {
+                channel.channel_ = new_outgoing_channel(request.target());
+                channel.method_ = request.method();
+                channel.parsed_bytes_ = 0;
+                request.send(channel.channel_->out());
+            }
+        }
+
+        // Maintain existing outgoing HTTP client requests.
+        HttpParserVec http_responses;
+        for (auto &pair : http_client_channels_) {
+            HttpChannel &htchan = pair.second;
+            Channel &channel = *htchan.channel_;
+            if (channel.closed() || (channel.in()->fill() > htchan.parsed_bytes_)) {
+                HttpParser response;
+                if (!channel.error().empty()) {
+                    response.fail(503, util::ss("Service Unavailable: ", channel.error()));
+                } else {
+                    htchan.parsed_bytes_ = channel.in()->fill();
+                    response.parse_response(channel.in()->view(), channel.closed(), htchan.method_);
+                }
+                if (response.complete()) {
+                    response.set_request_id(pair.first);
+                    http_responses.push_back(response);
+                }
+            }
+        }
+        for (const HttpParser &response : http_responses) {
+            http_client_channels_.erase(response.request_id());
+        }
+        master_->http_responses(http_responses);
+
+        // Maintain incoming HTTP server channels.
+        for (HttpChannel &htchan : http_server_channels_) {
+            SharedChannel &chan = htchan.channel_;
+            if (chan->in()->fill() > htchan.parsed_bytes_) {
+                HttpParser parser;
+                parser.parse_request(chan->in()->view(), chan->closed());
+                htchan.parsed_bytes_ = chan->in()->fill();
+                if (parser.complete()) {
+                    StreamBuffer *sb = chan->out();
+                    HttpServerResponse resp = master_->http_serve(parser);
+                    resp.send(sb);
+                    htchan.channel_ = nullptr;
+                }
+            }
+        }
+        util::remove_marked_items(http_server_channels_);
+    }
+};
+
+DrivenEngineDefine("lpxserver", LpxServer);
+

luprex/cpp/core/lpxserver.hpp

@@ -0,0 +1,5 @@
+#ifndef LPXSERVER_HPP
+#define LPXSERVER_HPP
+
+#endif // LPXSERVER_HPP
+

luprex/cpp/core/luaconsole.cpp

@@ -0,0 +1,168 @@
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+#include "eng-malloc.hpp"
+#include "luastack.hpp"
+#include "luaconsole.hpp"
+#include "util.hpp"
+
+#include <cstring>
+#include <iostream>
+
+LuaConsole::LuaConsole() {
+    lua_state_ = LuaStack::newstate(eng::l_alloc);
+    clear_raw_input();
+}
+
+LuaConsole::~LuaConsole() {
+    lua_close(lua_state_);
+}
+
+static LuaConsole::StringVec split_words(const eng::string &raw) {
+    LuaConsole::StringVec result;
+    eng::string acc;
+    for (char c : raw) {
+        if ((c == ' ')||(c == '\n')||(c == '\r')) {
+            if (!acc.empty()) {
+                result.push_back(acc);
+                acc = "";
+            }
+        } else {
+            acc += c;
+        }
+    }
+    if (!acc.empty()) {
+        result.push_back(acc);
+    }
+    return result;
+}
+
+LuaConsole::StringVec LuaConsole::get_command() {
+    StringVec result = words_;
+    words_.clear();
+    return result;
+}
+
+void LuaConsole::synerr(const eng::string &msg) {
+    words_.clear();
+    words_.push_back("syntax");
+    words_.push_back(msg);
+}
+
+void LuaConsole::simplify(const StringVec &words) {
+    words_ = words;
+    if (words.size() == 0) {
+        return;
+    } else if (sv::valid_int64(words[0])) {
+        if (words.size() == 1) {
+            words_.clear();
+            words_.push_back("choose");
+            words_.push_back(words[0]);
+        } else {
+            synerr("/choose command takes no arguments");
+        }
+    } else if (words[0] == "choose") {
+        if ((words.size() == 2)&&(sv::valid_int64(words[1]))) {
+            // OK
+        } else {
+            synerr("/choose [menu-line-number]");
+        }
+    } else if (words[0] == "view") {
+        if (words.size() != 1) {
+            synerr("/view takes no arguments");
+        }
+    } else if (words[0] == "menu") {
+        if (words.size() == 1) {
+            words_.push_back("-");
+        } else if ((words.size() == 2)&&(sv::valid_int64(words[1]))) {
+            // OK
+        } else {
+            synerr("/menu [optional-tangible-id]");
+        }
+    } else if (words[0] == "quit") {
+        if (words.size() != 1) {
+            synerr("/quit takes no arguments");
+        }
+    } else if (words[0] == "tick") {
+        if (words.size() != 1) {
+            synerr("/tick takes no arguments");
+        }
+    } else if (words[0] == "cpl") {
+        if (words.size() != 1) {
+            synerr("/cpl takes no arguments");
+        }
+    } else if (words[0] == "work") {
+        if (words.size() != 1) {
+            synerr("/work takes no arguments");
+        }
+    } else if (words[0] == "aborthttp") {
+        if (words.size() != 1) {
+            synerr("/aborthttp takes no arguments");
+        }
+    } else {
+        synerr("unrecognized command");
+    }
+}
+
+void LuaConsole::clear_raw_input() {
+    raw_input_ = "";
+    lines_ = 0;
+    prompt_ = "> ";
+}
+
+void LuaConsole::add(eng::string line) {
+    for (int i = 0; i < int(line.size()); i++) {
+        if (line[i] == '\n') line[i] = ' ';
+    }
+    raw_input_ += line;
+    raw_input_ += '\n';
+    lines_ += 1;
+    prompt_ = ">> ";
+    words_.clear();
+
+    // Try to interpret it as a slash-command.
+    if ((lines_ == 1)&&(raw_input_[0] == '/')) {
+        simplify(split_words(raw_input_.substr(1)));
+        clear_raw_input();
+        return;
+    }
+
+    // Translate lua expression, deal with initial prefix.
+    eng::string partial;
+    eng::string lua_mode;
+    if (sv::has_prefix(raw_input_, "?=")) {
+        lua_mode = "luaprobe";
+        partial = eng::string("return ") + raw_input_.substr(2);
+    } else if (sv::has_prefix(raw_input_, "?")) {
+        lua_mode = "luaprobe";
+        partial = raw_input_.substr(1);
+    } else if (sv::has_prefix(raw_input_, "=")) {
+        lua_mode = "luainvoke";
+        partial = eng::string("return ") + raw_input_.substr(1);
+    } else {
+        lua_mode = "luainvoke";
+        partial = raw_input_;
+    }
+
+    // Try to parse the lua expression
+    int top = lua_gettop(lua_state_);
+    int status = luaL_loadbuffer(lua_state_, partial.c_str(), partial.size(), "=stdin");
+    if (status == LUA_ERRSYNTAX)
+    {
+        const char *eof = "<eof>";
+        int leof = strlen(eof);
+        size_t lmsg;
+        const char *msg = lua_tolstring(lua_state_, -1, &lmsg);
+        const char *tp = msg + lmsg - leof;
+        if (strstr(msg, eof) != tp) {
+            words_.push_back("syntax");
+            words_.push_back(msg);
+            clear_raw_input();
+        }
+    } else {
+        words_.push_back(lua_mode);
+        words_.emplace_back(sv::rtrim(partial));
+        clear_raw_input();
+    }
+    lua_settop(lua_state_, top);
+}
+

luprex/cpp/core/luaconsole.hpp

@@ -0,0 +1,89 @@
+//////////////////////////////////////////////////////
+//
+// LuaConsole:
+//
+// Used to parse commands that are being interactively typed
+// in by a user.  The common usage pattern is:
+//
+//    1. Print the prompt suggested by 'get_prompt'.
+//    2. Read a line of text from stdin.
+//    3. Add the line to the LuaConsole using 'add'.
+//    4. Get the command word using get_command.
+//    5. If the command is empty, do nothing.
+//    6. If the command is nonempty, get the args and execute it.
+//
+// The LuaConsole expects you to type a lua command, or one
+// of the following slash-commands:
+//
+//    /quit               - exit the program
+//    /view               - display the nearby tangibles
+//    /menu [tanid]       - display the menu for tangible 
+//    /tick [timevalue]   - advance the simulation clock
+//    /choose [number]    - choose menu item number
+//    /1234               - choose menu item 1234
+//
+// If you type anything else, the LuaConsole will generate a
+// syntax error.  Note that not all of the commands above are
+// supported in all interpreters.
+//
+// Lua commands can be one of the following:
+//
+// <expression>       - invoke "<expression>""
+// = <expression>     - invoke "return <expression>"
+// ? <expression>     - probe "<expression>"
+// ?= <expression>    - probe "return <expression>"
+//
+// Once a command has been typed (or a syntax error has been
+// typed), the LuaConsole will return the command.  The command
+// can be fetched using command(), int_arg(), and str_arg()
+//
+//////////////////////////////////////////////////////
+
+#ifndef LUACONSOLE_HPP
+#define LUACONSOLE_HPP
+
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+
+#include "luastack.hpp"
+
+class LuaConsole : public eng::nevernew {
+public:
+    using StringVec = eng::vector<eng::string>;
+
+private:
+    lua_State *lua_state_;
+    eng::string raw_input_;
+    int lines_;
+    eng::string prompt_;
+    StringVec words_;
+
+    void clear_raw_input();
+    void simplify(const StringVec &words);
+    void synerr(const eng::string &msg);
+
+public:
+    LuaConsole();
+    ~LuaConsole();
+
+    // Fetch the recommended prompt.
+    //
+    // You should update the prompt immediately after 'add'.
+    //
+    const eng::string &get_prompt() { return prompt_; }
+    
+    // Get the command words.
+    //
+    // Note that the command words may not be exactly what
+    // the user typed.  Typically, the LuaConsole simplifies
+    // the command before returning it to the caller.
+    //
+    StringVec get_command();
+    
+    // Add a line of text that was just read from the console.
+    //
+    void add(eng::string line);
+
+};
+
+#endif // LUACONSOLE_HPP

luprex/cpp/core/luasnap.cpp

@@ -0,0 +1,129 @@
+
+#include "wrap-sstream.hpp"
+#include "wrap-string.hpp"
+
+#include "luasnap.hpp"
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+
+#include <cassert>
+
+
+LuaSnap::LuaSnap() {
+    state_ = LuaStack::newstate(eng::l_alloc);
+    LuaStack LS(state_);
+
+    // Create the persist table and the unpersist table.
+    //
+    // These tables need to contain all C functions.  Whenever
+    // the source module inserts a C function into the lua environment,
+    // it also needs to register the C function in these tables.
+    //
+    // I don't think anything else needs to go in these tables.
+    //
+    LS.rawset(LuaRegistry, "persist", LuaNewTable);
+    LS.rawset(LuaRegistry, "unpersist", LuaNewTable);
+}
+
+LuaSnap::~LuaSnap() {
+    lua_close(state_);
+}
+
+void LuaSnap::serialize(StreamBuffer *sb) {
+    // Lua stack should be empty.
+    assert(lua_gettop(state_) == 0);
+
+    // lua variables that we'll need.
+    LuaVar key, value;
+    LuaRet permstable, regcopy;
+    LuaStack LS(state_, permstable, regcopy, key, value);
+
+    // Construct a copy of the registry table.
+    LS.set(regcopy, LuaNewTable);
+    LS.set(key, LuaNil);
+    while (LS.next(LuaRegistry, key, value) != 0) {
+        LS.rawset(regcopy, key, value);
+    }
+
+    // Remove certain things from the copy.  These items
+    // don't get included in the snapshot.
+    //
+    // From a modularity perspective, this is bad.
+    //
+    LS.rawset(regcopy, LUA_RIDX_MAINTHREAD, LuaNil);
+    LS.rawset(regcopy, "persist", LuaNil);
+    LS.rawset(regcopy, "unpersist", LuaNil);
+    LS.rawset(regcopy, "world", LuaNil);
+    LS.rawset(regcopy, "gui", LuaNil);
+    LS.rawset(regcopy, "tnmap", LuaNil);
+    LS.rawset(regcopy, "ntmap", LuaNil);
+
+    // Get the eris permanents table from the registry.
+    LS.rawget(permstable, LuaRegistry, "persist");
+    assert(LS.istable(permstable));
+
+    // When we call 'LS.result', this should leave the
+    // permstable and the regcopy on the stack.
+    LS.result();
+    assert(lua_gettop(state_) == 2);
+
+    // Write dummy length, use eris to write data, then overwrite length.
+    sb->write_int64(0);
+    int64_t pos1 = sb->total_writes();
+    eris_dump(state_, sb->lua_writer, sb->lua_writer_ud());
+    int64_t pos2 = sb->total_writes();
+    sb->overwrite_int64(pos1, pos2 - pos1);
+    lua_settop(state_, 0);
+}
+
+void LuaSnap::deserialize(StreamBuffer *sb) {
+    // Lua stack should be empty.
+    assert(lua_gettop(state_) == 0);
+
+    // Get the length of the eris dump.
+    int64_t len = sb->read_int64();
+    void *ud = sb->lua_reader_ud(len);
+
+    // Call eris with the permanents table and passing the snapshot as a lua_Reader.
+    lua_getfield(state_, LUA_REGISTRYINDEX, "unpersist");
+    eris_undump(state_, sb->lua_reader, ud);
+    assert(lua_gettop(state_) == 2);
+
+    // Set up a stack frame.
+    LuaArg permstable, regcopy;
+    LuaVar key, value;
+    LuaStack LS(state_, permstable, regcopy, key, value);
+
+    assert(LS.istable(regcopy));
+
+    // Copy the contents of the snapshot over to the registry.
+    LS.set(key, LuaNil);
+    while (LS.next(regcopy, key, value) != 0) {
+        LS.rawset(LuaRegistry, key, value);
+    }
+    LS.result();
+    assert(lua_gettop(state_) == 0);
+}
+
+// Snapshot and rollback can trivially be implemented on top of serialize and
+// deserialize.  However, it's also possible to implement snapshot and rollback
+// using an alternative technique:
+//
+// 1. When constructing the lua interpreter, use a custom memory allocator that
+//    keeps track of all the memory blocks used by lua.
+//
+// 2. Snapshot simply copies all the memory blocks used by lua into a buffer.
+//
+// 3. Rollback restores lua's memory blocks back to their previous state.  This
+//    has the effect of restoring lua's state.
+//
+// A proof-of-concept implementation of the memory-snapshotting design was
+// created, and it worked. It is probably faster than using serialize and
+// deserialize.
+//
+// Note: even if we implement this alternative design, we still need to keep
+// serialize and deserialize around in order to implement the save-game
+// functionality. So for now, we're sticking with this design, which doesn't
+// require us to maintain any additional code.
+
+

luprex/cpp/core/luasnap.hpp

@@ -0,0 +1,42 @@
+/////////////////////////////////////////////////////////////////////////////
+//
+// LUASNAP
+//
+// A lua interpreter that can be checkpointed (snapshotted). This makes it
+// possible to roll the entire interpreter back to a previously-snapshotted
+// state.
+//
+// To accomplish this, we use eris serialization.  This is messy and not
+// very modular, but it does work.
+//
+/////////////////////////////////////////////////////////////////////////////
+
+#ifndef LUASNAP_HPP
+#define LUASNAP_HPP
+
+#include "streambuffer.hpp"
+#include "luastack.hpp"
+
+class LuaSnap : public eng::nevernew {
+private:
+    lua_State *state_;
+    
+public:
+    LuaSnap();
+    ~LuaSnap();
+
+    // Get the lua intepreter.
+    //
+    lua_State *state() const { return state_; }
+
+    // Serialize the state of the lua interpreter.
+    //
+    void serialize(StreamBuffer *sb);
+    
+    // Restore the the lua interpreter given a serialized state.
+    //
+    void deserialize(StreamBuffer *sb);
+};
+
+
+#endif // LUASNAP_HPP

luprex/cpp/core/luastack.cpp

@@ -0,0 +1,519 @@
+#include "luastack.hpp"
+#include <iostream>
+#include <cassert>
+#include <cstdio>
+#include <climits>
+#include "wrap-string.hpp"
+#include "wrap-set.hpp"
+#include "wrap-sstream.hpp"
+
+LuaSpecial LuaRegistry(LUA_REGISTRYINDEX);
+LuaNilMarker LuaNil;
+LuaNewTableMarker LuaNewTable;
+
+LuaFunctionReg::LuaFunctionReg(const char *n, const char *a, const char *d, bool s, lua_CFunction f) {
+    name_ = n;
+    args_ = a;
+    docs_ = d;
+    func_ = f;
+    sandbox_ = s;
+    next_ = All;
+    All = this;
+}
+
+LuaConstantReg::LuaConstantReg(const char *n, const char *d, LuaToken tokenvalue, lua_Number numbervalue) {
+    name_ = n;
+    docs_ = d;
+    tokenvalue_ = tokenvalue;
+    numbervalue_ = numbervalue;
+    next_ = All;
+    All = this;
+}
+
+const LuaFunctionReg *LuaFunctionReg::lookup(lua_CFunction fn) {
+    for (const LuaFunctionReg *r = All; r != 0; r = r->next_) {
+        if (r->func_ == fn) {
+            return r;
+        }
+    }
+    return nullptr;
+}
+
+LuaFunctionReg *LuaFunctionReg::All;
+LuaConstantReg *LuaConstantReg::All;
+
+
+eng::string LuaToken::str() const {
+    uint64_t token = (uint64_t)value;
+    char buffer[9];
+    for (int i = 0; i < 8; i++) {
+        unsigned char c = token;
+        buffer[7-i] = c;
+        token >>= 8;
+    }
+    buffer[8] = 0;
+    return eng::string(buffer);
+}
+
+static int panicf(lua_State *L) {
+    const char *p = lua_tostring(L, -1);
+    fprintf(stderr, "PANIC: unprotected error in call to Lua API (%s)\n", p);
+    fflush(stderr);
+    exit(1);
+}
+
+// An allocator for lua states that uses system malloc and system free.
+static void *l_alloc(void *ud, void *ptr, size_t osize, size_t nsize) {
+    if (nsize == 0) {
+        free(ptr);
+        return NULL;
+    } else {
+        return realloc(ptr, nsize);
+    }
+}
+
+lua_State *LuaStack::newstate (lua_Alloc allocf) {
+    if (allocf == nullptr) allocf = l_alloc;
+    lua_State *L = lua_newstate(allocf, NULL);
+    if (L) lua_atpanic(L, &panicf);
+    return L;
+}
+
+void LuaStack::argerr(const char *nm, const char *tp) const {
+    luaL_error(L_, "'%s' should be %s", nm, tp);
+}
+
+bool LuaStack::isinteger(LuaSlot s) const {
+    if (lua_type(L_, s) == LUA_TNUMBER) {
+        lua_Number result = lua_tonumber(L_, s);
+        if (lua_Integer(result) == result) return true;
+    }
+    return false;
+}
+
+bool LuaStack::isint(LuaSlot s) const {
+    if (lua_type(L_, s) == LUA_TNUMBER) {
+        lua_Number result = lua_tonumber(L_, s);
+        if (int(result) == result) return true;
+    }
+    return false;
+}
+
+bool LuaStack::ckboolean(LuaSlot s) const {
+    checkboolean(s, "value");
+    return lua_toboolean(L_, s) ? true:false;
+}
+
+lua_Integer LuaStack::ckinteger(LuaSlot s) const {
+    checknumber(s, "value");
+    luaL_checktype(L_, s, LUA_TNUMBER);
+    lua_Number result = lua_tonumber(L_, s);
+    lua_Integer iresult(result);
+    if (iresult != result) {
+        luaL_error(L_, "not a valid integer");
+        return 0;
+    }
+    return iresult;
+}
+
+int LuaStack::ckint(LuaSlot s) const {
+    checknumber(s, "value");
+    lua_Number result = lua_tonumber(L_, s);
+    int iresult(result);
+    if (iresult != result) {
+        luaL_error(L_, "not a valid int");
+        return 0;
+    }
+    return iresult;
+}
+
+lua_Number LuaStack::cknumber(LuaSlot s) const {
+    checknumber(s, "value");
+    return lua_tonumber(L_, s);
+}
+
+eng::string LuaStack::ckstring(LuaSlot s) const {
+    checkstring(s, "value");
+    size_t len;
+    const char *str = lua_tolstring(L_, s, &len);
+    return eng::string(str, len);
+}
+
+std::string_view LuaStack::ckstringview(LuaSlot s) const {
+    checkstring(s, "value");
+    size_t len;
+    const char *str = lua_tolstring(L_, s, &len);
+    return std::string_view(str, len);
+}
+
+lua_State *LuaStack::ckthread(LuaSlot s) const {
+    checkthread(s, "value");
+    return lua_tothread(L_, s);
+}
+
+LuaToken LuaStack::cktoken(LuaSlot s) const {
+    checktoken(s, "value");
+    return LuaToken(lua_touserdata(L_, s));
+}
+
+void LuaStack::count_slots_finalize(int narg, int nvar, int nret) {
+    narg_ = narg;
+    nret_ = nret;
+    nvar_ = nvar;
+    ngap_ = nret - nvar - narg;
+    if (ngap_ < 0) ngap_ = 0;
+
+    int argtop = lua_gettop(L_);
+    argpos_ = argtop + 1 - narg_;
+    gappos_ = argpos_ + narg_;
+    varpos_ = gappos_ + ngap_;
+    retpos_ = varpos_ + nvar_;
+
+    rettop_ = retpos_ + nret_ - 1;
+    finaltop_ = argpos_ + nret_ - 1;
+}
+
+void LuaStack::clear_frame() {
+    lua_settop(L_, varpos_ - 1);
+    for (int i = 0; i < nvar_ + nret_; i++) {
+        lua_pushnil(L_);
+    }
+}
+
+int LuaStack::result() {
+    lua_settop(L_, rettop_);
+    int i = finaltop_;
+    for (int j = 0; j < nret_; j++) {
+        lua_replace(L_, i);
+        i -= 1;
+    }
+    lua_settop(L_, finaltop_);
+    return nret_;
+}
+
+void LuaStack::clearmetatable(LuaSlot tab) const {
+    lua_pushnil(L_);
+    lua_setmetatable(L_, tab);
+}
+
+void LuaStack::setmetatable(LuaSlot tab, LuaSlot mt) const {
+    lua_pushvalue(L_, mt);
+    lua_setmetatable(L_, tab);
+}
+
+bool LuaStack::getmetatable(LuaSlot mt, LuaSlot tab) const {
+    if (lua_getmetatable(L_, tab)) {
+        lua_replace(L_, mt);
+        return true;
+    } else {
+        lua_pushnil(L_);
+        lua_replace(L_, mt);
+        return false;
+    }
+}
+
+int LuaStack::next(LuaSlot tab, LuaSlot key, LuaSlot value) const {
+    lua_pushvalue(L_, key);
+    int ret = lua_next(L_, tab);
+    if (ret != 0) {
+        lua_replace(L_, value);
+        lua_replace(L_, key);
+    }
+    return ret;
+}
+
+void LuaStack::getglobaltable(LuaSlot target) const {
+    lua_pushglobaltable(L_);
+    lua_replace(L_, target);
+}
+
+void LuaStack::newtable(LuaSlot target) const {
+    lua_newtable(L_);
+    lua_replace(L_, target);
+}
+
+void LuaStack::createtable(LuaSlot target, int narr, int nrec) const {
+    lua_createtable(L_, narr, nrec);
+    lua_replace(L_, target);
+}
+
+lua_State *LuaStack::newthread(LuaSlot target) const {
+    lua_State *result = lua_newthread(L_);
+    lua_replace(L_, target);
+    return result;
+}
+
+bool LuaStack::validclassname(std::string_view cname) {
+    if (cname.empty()) return false;
+    if (cname == "_G") return false;
+    return true;
+}
+
+bool LuaStack::validclassname(LuaSlot slot) const {
+    if (!isstring(slot)) return false;
+    return validclassname(ckstring(slot));
+}
+
+eng::string LuaStack::classname(LuaSlot tab) const {
+    eng::string result;
+    if (istable(tab)) {
+        lua_pushstring(L_, "__class");
+        lua_rawget(L_, tab);
+        if (lua_type(L_, -1) == LUA_TSTRING) {
+            lua_pushglobaltable(L_); // cname table
+            lua_pushvalue(L_, -2); // cname table cname
+            lua_rawget(L_, -2); // cname table ctab
+            if (lua_rawequal(L_, -1, tab)) {
+                size_t len;
+                const char *s = lua_tolstring(L_, -3, &len);
+                result = eng::string(s, len);
+                if (!validclassname(result)) {
+                    result = "";
+                }
+            }
+            lua_pop(L_, 3);
+        } else {
+            lua_pop(L_, 1);
+        }
+    }
+    return result;
+}
+
+eng::string LuaStack::getclass(LuaSlot classtab, LuaSlot classname) const {
+    lua_checkstack(L_, 20);
+    LuaVar globtab, cname;
+    LuaStack LS(L_, globtab, cname);
+    LS.getglobaltable(globtab);
+
+    if (LS.isstring(classname)) {
+        if (!validclassname(LS.ckstring(classname))) {
+            eng::string err = "invalid class name: " + LS.ckstring(classname);
+            LS.result();
+            return err;
+        }
+        LS.rawget(classtab, globtab, classname);
+        if (!LS.istable(classtab)) {
+            eng::string err = "not a class: " + LS.ckstring(classname);
+            LS.result();
+            return err;
+        }
+        LS.rawget(cname, classtab, "__class");
+        if (!LS.rawequal(cname, classname)) {
+            eng::string err = "not a valid class: " + LS.ckstring(classname);
+            LS.result();
+            return err;
+        }
+        LS.result();
+        return "";
+    } else if (LS.istable(classname)) {
+        LS.rawget(cname, classname, "__class");
+        if (!LS.isstring(cname)) {
+            eng::string err = "table is not a class.";
+            LS.result();
+            return err;
+        }
+        if (!validclassname(LS.ckstring(cname))) {
+            eng::string err = "invalid class name: " + LS.ckstring(cname);
+            LS.result();
+            return err;
+        }
+        LS.rawget(classtab, globtab, cname);
+        if (!LS.rawequal(classtab, classname)) {
+            eng::string err = "not a valid class: " + LS.ckstring(cname);
+            LS.result();
+            return err;
+        }
+        LS.result();
+        return "";
+    } else {
+        eng::string err = "getclass expects a string or a classtab";
+        LS.result();
+        return err;
+    }
+}
+
+eng::string LuaStack::getclass(LuaSlot tab, std::string_view name) const {
+    push_any_value(name);
+    LuaSpecial classname(lua_gettop(L_));
+    eng::string err = getclass(tab, classname);
+    lua_pop(L_, 1);
+    return err;
+}
+
+void LuaStack::makeclass(LuaSlot classtab, LuaSlot classname) const {
+    lua_checkstack(L_, 20);
+    LuaVar globtab, cname;
+    LuaStack LS(L_, globtab, cname);
+
+    // Validate the class name.
+    assert(LS.validclassname(classname));
+    
+    // Fetch the global environment from the registry.
+    LS.getglobaltable(globtab);
+    
+    // Get the classtab from the global environment.
+    LS.rawget(classtab, globtab, classname);
+
+    // Make a new table if necessary.
+    if (!LS.istable(classtab)) {
+        LS.newtable(classtab);
+        LS.rawset(globtab, classname, classtab);
+    }
+    
+    // Repair the special fields.
+    LS.settabletype(classtab, LUA_TT_CLASS);
+    LS.rawset(classtab, "__class", classname);
+    LS.rawset(classtab, "__index", classtab);
+
+    // Put the stack back.
+    LS.result();
+}
+
+void LuaStack::makeclass(LuaSlot tab, std::string_view name) const {
+    push_any_value(name);
+    LuaSpecial classname(lua_gettop(L_));
+    makeclass(tab, classname);
+    lua_pop(L_, 1);
+}
+
+
+int64_t LuaStack::tanid(LuaSlot tab) const {
+    int64_t result = 0;
+    if (istable(tab) && gettabletype(tab) == LUA_TT_TANGIBLE) {
+        if (lua_getmetatable(L_, tab.index())) {
+            lua_pushstring(L_, "id");
+            lua_rawget(L_, -2);
+            if (lua_type(L_, -1) == LUA_TNUMBER) {
+                result = int64_t(lua_tonumber(L_, -1));
+            }
+            lua_pop(L_, 2);
+        }
+    }
+    return result;
+}
+
+bool LuaStack::issortablekey(LuaSlot s) const {
+    int type = lua_type(L_, s);
+    return (type == LUA_TBOOLEAN) || (type == LUA_TNUMBER) || (type == LUA_TSTRING);
+}
+
+void LuaStack::movesortablekey(LuaSlot key, LuaStack &otherstack, LuaSlot otherslot) {
+    int type = lua_type(L_, key);
+    switch (type) {
+    case LUA_TBOOLEAN:
+        lua_pushboolean(otherstack.L_, lua_toboolean(L_, key));
+        lua_replace(otherstack.L_, otherslot);
+        break;
+    case LUA_TNUMBER:
+        lua_pushnumber(otherstack.L_, lua_tonumber(L_, key));
+        lua_replace(otherstack.L_, otherslot);
+        break;
+    case LUA_TSTRING: {
+        size_t len;
+        const char *str = lua_tolstring(L_, key, &len);
+        lua_pushlstring(otherstack.L_, str, len);
+        lua_replace(otherstack.L_, otherslot);
+        break;
+    }
+    default:
+        assert(false && "movesortablekey: not a sortable key");
+    }
+}
+
+void LuaStack::cleartable(LuaSlot tab, bool clearmeta) const {
+    assert(istable(tab));
+    lua_pushnil(L_);
+    while (lua_next(L_, tab.index()) != 0) {
+        lua_pop(L_, 1); // Pop the old value.
+        lua_pushvalue(L_, -1); // Clone the key
+        lua_pushnil(L_); // Push the new value.
+        lua_rawset(L_, tab.index());
+    }
+    if (clearmeta) {
+        lua_pushnil(L_);
+        lua_setmetatable(L_, tab.index());
+    }
+}
+
+int LuaStack::rawlen(LuaSlot obj) const {
+    return lua_rawlen(L_, obj.index());
+}
+
+int LuaStack::gettabletype(LuaSlot tab) const {
+    uint16_t bits = lua_getflagbits(L_, tab.index());
+    return LUA_TT_GENERAL + (bits & 0x000F);
+}
+
+void LuaStack::settabletype(LuaSlot tab, int t) const {
+    assert((t >= LUA_TT_GENERAL) && (t <= LUA_TT_CLASS));
+    int offset = (t - LUA_TT_GENERAL);
+    lua_modflagbits(L_, tab.index(), 0x000F, offset);
+}
+
+int LuaStack::xtype(LuaSlot slot) const {
+    int t = lua_type(L_, slot);
+    if (t != LUA_TTABLE) return t;
+    uint16_t bits = lua_getflagbits(L_, slot);
+    return LUA_TT_GENERAL + (bits & 0x000F);
+}
+
+bool LuaStack::getvisited(LuaSlot tab) const {
+    uint16_t bits = lua_getflagbits(L_, tab.index());
+    return (bits & 0x0010);
+}
+
+void LuaStack::setvisited(LuaSlot tab, bool visited) const {
+    lua_modflagbits(L_, tab.index(), 0x0010, visited ? 0x0010 : 0);
+}
+
+LuaKeywordParser::LuaKeywordParser(lua_State *L, int slot) {
+    L_ = L;
+    slot_ = slot;
+    not_table_ = !lua_istable(L_, slot_);
+    if (not_table_) {
+        lua_newtable(L_);
+        lua_replace(L_, slot_);
+    }
+}
+
+bool LuaKeywordParser::parse(LuaSlot out, const char *kw) {
+    lua_pushstring(L_, kw);
+    lua_rawget(L_, slot_);
+    lua_replace(L_, out.index());
+    if (!lua_isnil(L_, out.index())) {
+        parsed_.insert(kw);
+        return true;
+    } else {
+        return false;
+    }
+};
+
+eng::string LuaKeywordParser::final_check() {
+    if (not_table_) {
+        return "expected a keyword table";
+    }
+    if (lua_nkeys(L_, slot_) != int(parsed_.size())) {
+        lua_pushnil(L_);
+        while (lua_next(L_, slot_) != 0) {
+            lua_pop(L_, 1); // Don't need the value.
+            if (!lua_isstring(L_, -1)) {
+                return "keyword table contains non-string key";
+            }
+            const char *kw = lua_tostring(L_, -1);
+            if (parsed_.find(kw) == parsed_.end()) {
+                eng::ostringstream oss;
+                oss << "keyword " << kw << " not known";
+                return oss.str();
+            }
+        }
+        assert(false && "should never get here in check_unparsed_keywords");
+    }
+    return "";
+}
+
+void LuaKeywordParser::final_check_throw() {
+    eng::string err = final_check();
+    if (!err.empty()) {
+        luaL_error(L_, "%s", err.c_str());
+    }
+}

luprex/cpp/core/luastack.hpp

@@ -0,0 +1,629 @@
+/////////////////////////////////////////////////////////
+//
+//
+// LUASTACK
+//
+// The standard lua C API asks you to work with a stack machine. You're supposed
+// to manually push and pop values on the lua stack. I find this difficult, I
+// find it hard to remember what stack position contains what value.
+//
+// To make it easier, I've created this module, "LuaStack."  This module
+// creates the illusion that you're working with local variables that contain
+// lua values.
+//
+// Of course, this is all using the lua stack under the covers.  Lua
+// local variables are actually just lua stack addresses.  But that's
+// all kept fairly well hidden.  When you use Lua local variables, and
+// the accessors inside class LuaStack, it appears that you're
+// manipulating data using local variables instead of using a stack.
+// For people like me, that's easier to think about.
+//
+// Here's an example.
+//
+// let's say you have a function that takes two arguments
+// ARG1 and ARG2, has a single return value RET1, and needs two local
+// variables LOC1 and LOC2.  We would declare it like this:
+//
+// int myfunc(lua_State *L) {
+//
+//     LuaArg arg1, arg2;         // Declare local variables to hold the arguments.
+//     LuaRet ret1;               // Declare local variables to hold the return values.
+//     LuaVar loc1, loc2, loc3;   // Declare local variables for other purposes.
+//
+//     // Assign every local var a stack index.
+//     LuaStack LS(L, arg1, arg2, ret1, loc1, loc2, loc3);
+//
+//     // manipulate the data in the lua local variables...
+//     LS.rawget(loc1, arg1, arg2);
+//     ... etc ...
+// }
+//
+// Class LuaArg, LuaRet, and LuaVar are all lua local variables.
+// The luastack constructor assigns each one of them a position on
+// the lua stack.  It also makes sure that the arguments are in
+// the LuaArg variables, and it makes sure that the LuaRet values
+// are the only thing left on the stack at return time.
+//
+// Class LuaStack provides a complete catalog of accessors
+// like 'rawget' - roughly speaking, it provides equivalents to
+// every major accessor in the lua API.  However, the accessors
+// provided by LuaStack take input and output from lua locals, not
+// from the stack.  For example, consider this:
+//
+//    LS.rawget(value, tab, key);
+//
+// In the above, value, tab, and key should be lua local variables.
+// This does a rawget on 'table', with the specified 'key', and
+// stores the result in 'value'.  Nothing is added to or removed
+// from the lua stack.  In general, none of the accessors in class
+// LuaStack add anything to the stack, or pop anything from the
+// stack.
+//
+// Class LuaStack can also do automatic type conversions.  For
+// example, suppose you do this:
+//
+//    LS.rawget(value, tab, key);
+//
+// Nominally, you would expect value, tab, and key to be lua local
+// variables. But if you pass a eng::string for key, then LuaStack will
+// automatically convert it.  In general, class LuaStack can
+// convert lua_Integer, lua_Number, eng::string, bool, and LuaNil.
+//
+// On output, LuaStack can convert lua_Integers, lua_Numbers, and
+// eng::strings.  In this case, strict type checking is done.  If
+// there is a type mismatch, a lua error is thrown.
+//
+// You can use the operator 'set' to assign a value to a lua local
+// variable:
+//
+//   LS.set(val1, val2);
+//
+// This is actually a copy operation that copies from one lua local
+// variable to another.  But using type conversions, it can also be
+// used to assign arbitrary values to lua local variables, or to
+// get values from lua local variables.
+// 
+// Passing LuaNewTable as an input will cause a new table to be
+// created before calling the specified operation.
+//
+//
+/////////////////////////////////////////////////////////
+//
+//
+// LuaStack type checking
+//
+// LuaStack contains accessors for type checking.  These include:
+//
+//     bool LuaStack::isnumber(LuaSlot s)
+//     bool LuaStack::isinteger(LuaSlot s)
+//     bool LuaStack::isstring(LuaSlot s)
+//     etc...
+//
+// And it also contains operations that throw errors:
+//
+//     void LuaStack::checknumber(LuaSlot s)
+//     void LuaStack::checkinteger(LuaSlot s)
+//     void LuaStack::checkstring(LuaSlot s)
+//     etc...
+//
+// These are different from the lua builtins in that they are strict.
+// For example, 'isnumber' only returns true if the value in the
+// lua local variable is already a number.  No conversions are done.
+//
+// These functions do checking and also conversion at the same time:
+//
+//     lua_Integer  LuaStack::ckinteger(LuaSlot s)
+//     lua_Number   LuaStack::cknumber(LuaSlot s)
+//     eng::string  LuaStack::ckstring(LuaSlot s)
+//     lua_State   *LuaStack::ckthread(LuaSlot s)
+//
+// Like the other operations, they are strict.
+//
+//
+// LUADEFINE
+//
+// LuaDefine is a macro that defines a C function which is
+// exposed to lua.  It creates a global registry of functions
+// created with LuaDefine. You use it like so:
+//
+//    LuaDefine(function_name, "arguments", "documentation") {
+//        ...
+//    }
+//
+// This macroexpands into a function definition and a function
+// registration.  The function definition looks like this:
+//
+//    int function_name(lua_State *L) {
+//        ...
+//    }
+//
+// The macro expansion generates this function definition, but it
+// also generates a "registration object" whose constructor puts
+// this function into a global registry of lua-callable C functions.
+// This global registry is later used to inject these C functions
+// into the lua intepreter.  The mode is a string that contain
+// the following characters:
+//
+//    c - create a class, and put a function into it.
+//    f - create a global function not inside a class.
+//
+//
+/////////////////////////////////////////////////////////
+
+
+#ifndef LUASTACK_HPP
+#define LUASTACK_HPP
+
+#include "wrap-string.hpp"
+#include "wrap-set.hpp"
+#include <cstring>
+
+extern "C" {
+#include "lua.h"
+#include "lauxlib.h"
+#include "lualib.h"
+#include "eris.h"
+}
+
+class LuaSlot : public eng::nevernew {
+protected:
+    int index_;
+private:
+    inline operator int() const {
+        return index_;
+    }
+public:
+    LuaSlot() {
+        index_ = 0;
+    }
+
+    int index() const {
+        return index_;
+    }
+    
+    friend class LuaStack;
+};
+
+class LuaArg : public LuaSlot {};
+class LuaRet : public LuaSlot {};
+class LuaVar : public LuaSlot {};
+
+class LuaSpecial : public LuaSlot {
+public:
+    LuaSpecial(int n) {
+        index_ = n;
+    }
+};
+
+extern LuaSpecial LuaRegistry;
+
+class LuaUpvalue : public LuaSlot {
+    public:
+        LuaUpvalue(int n) {
+            index_ = lua_upvalueindex(n);
+        }
+};
+
+class LuaNilMarker {};
+extern LuaNilMarker LuaNil;
+
+class LuaNewTableMarker {};
+extern LuaNewTableMarker LuaNewTable;
+
+using LuaDeleterFn = void (*)(void *);
+
+using LuaTypeTag = lua_CFunction;
+template<typename T>
+int LuaTypeTagValue(lua_State *L) { return 0; }
+
+// Lua table types.  These deliberately do not overlap
+// with lua type values.
+//
+#define LUA_TT_GENERAL       16
+#define LUA_TT_REGISTRY      17
+#define LUA_TT_GLOBALENV     18
+#define LUA_TT_TANGIBLE      19
+#define LUA_TT_TANGIBLEMETA  20
+#define LUA_TT_DEADTANGIBLE  21
+#define LUA_TT_GLOBALDB      22
+#define LUA_TT_CLASS         23
+
+// We use lightuserdata to store 'tokens': short
+// strings of 8 characters or less.  These tokens
+// are useful as unique markers.  The 8 characters
+// are packed into a uint64.
+
+struct LuaToken {
+private:
+    static constexpr uint64_t literal_to_token(const char *str) {
+        uint64_t result = 0;
+        for (int i = 0; i < 8; i++) {
+            unsigned char c = *str;
+            result = (result << 8) + c;
+            if (*str) str++;
+        }
+        return result;
+    }
+public:
+    uint64_t value;
+
+    template<class T>
+    LuaToken(T arg) = delete;
+
+    constexpr LuaToken(const char *str) : value(literal_to_token(str)) {}
+    LuaToken(uint64_t v) : value(v) {}
+    LuaToken(void *v) : value((uint64_t)v) {}
+    LuaToken() : value(0) {}
+
+    bool empty() const { return value == 0; }
+    bool operator ==(const LuaToken &other) const { return value == other.value; }
+    void *voidvalue() const { return (void*)value; }
+
+    eng::string str() const;
+};
+
+class LuaStack : public eng::nevernew {
+private:
+    int narg_;
+    int ngap_;
+    int nvar_;
+    int nret_;
+
+    int argpos_;
+    int gappos_;
+    int varpos_;
+    int retpos_;
+
+    int rettop_;
+    int finaltop_;
+    
+    lua_State *L_;
+
+    template<int NARG, int NVAR, int NRET, class... SS>
+    void count_slots(LuaArg &v, SS & ... stackslots)
+    {
+        count_slots<NARG+1, NVAR, NRET>(stackslots...);
+    }
+    template<int NARG, int NVAR, int NRET, class... SS>
+    void count_slots(LuaVar &v, SS & ... stackslots)
+    {
+        count_slots<NARG, NVAR+1, NRET>(stackslots...);
+    }
+    template<int NARG, int NVAR, int NRET, class... SS>
+    void count_slots(LuaRet &v, SS & ... stackslots)
+    {
+        count_slots<NARG, NVAR, NRET+1>(stackslots...);
+    }
+
+    template<int NARG, int NVAR, int NRET>
+    void count_slots() {
+        count_slots_finalize(NARG, NVAR, NRET);
+    }
+
+    void count_slots_finalize(int narg, int nvar, int nret);
+
+    template<class... SS>
+    void assign_slots(int argp, int varp, int retp, LuaArg &v, SS & ... stackslots) {
+        v.index_ = argp;
+        assign_slots(argp + 1, varp, retp, stackslots...);
+    }
+
+    template<class... SS>
+    void assign_slots(int argp, int varp, int retp, LuaVar &v, SS & ... stackslots) {
+        v.index_ = varp;
+        assign_slots(argp, varp+1, retp, stackslots...);
+    }
+    
+    template<class... SS>
+    void assign_slots(int argp, int varp, int retp, LuaRet &v, SS & ... stackslots) {
+        v.index_ = retp;
+        assign_slots(argp, varp, retp+1, stackslots...);
+    }
+
+    void assign_slots(int argp, int varp, int retp) {}
+
+    void clear_frame();
+    
+private:
+    // Push any value on the stack, by type.
+    void push_any_value(LuaNewTableMarker s) const { lua_newtable(L_); }
+    void push_any_value(LuaNilMarker s) const { lua_pushnil(L_); }
+    void push_any_value(LuaSlot s) const { lua_pushvalue(L_, s); }
+    void push_any_value(const eng::string &s) const { lua_pushlstring(L_, s.c_str(), s.size()); }
+    void push_any_value(std::string_view s) const { lua_pushlstring(L_, s.data(), s.size()); }
+    void push_any_value(const char *s) const { lua_pushstring(L_, s); }
+    void push_any_value(float s) const { lua_pushnumber(L_, s); }
+    void push_any_value(double s) const { lua_pushnumber(L_, s); }
+    void push_any_value(int s) const { lua_pushinteger(L_, s); }
+    void push_any_value(lua_Integer s) const { lua_pushinteger(L_, s); }
+    void push_any_value(lua_CFunction s) const { lua_pushcfunction(L_, s); }
+    void push_any_value(bool b) const { lua_pushboolean(L_, b ? 1:0); }
+    void push_any_value(LuaToken token) const { lua_pushlightuserdata(L_, (void*)(token.value)); }
+
+    // Push multiple values on the stack, in order, by type.
+    template<typename T0, typename... T>
+    void push_any_values(T0 arg0, T... args) {
+        push_any_value(arg0);
+        push_any_values(args...);
+    }
+    void push_any_values() {
+    }
+
+    template<typename T>
+    static void delete_pointer(void *p) { delete (T*)p; }
+    static void do_not_delete(void *p) { }
+
+    void argerr(const char *arg, const char *tp) const;
+
+public:
+    template<class... SS>
+    LuaStack(lua_State *L, SS & ... stackslots) {
+        L_ = L;
+        count_slots<0, 0, 0>(stackslots...);
+        if (lua_gettop(L) < narg_) {
+            luaL_error(L, "not enough arguments to function");
+        }
+        assign_slots(argpos_, varpos_, retpos_, stackslots...);
+        clear_frame();
+    }
+
+    ~LuaStack() {};
+
+    int result();
+
+public:
+    // This is the largest integer that can be stored in a lua_Number.
+    // In other words, any 53-bit number can be stored.
+    static const int64_t MAXINT = 0x001FFFFFFFFFFFFF;
+
+    static lua_State *newstate (lua_Alloc allocf);
+
+    lua_State *state() const { return L_; }
+
+    int type(LuaSlot s) const { return lua_type(L_, s); }
+    void checktype(LuaSlot s, int type) const { luaL_checktype(L_, s, type); }
+
+    bool istable(LuaSlot s) const { return lua_type(L_, s) == LUA_TTABLE; }
+    bool isstring(LuaSlot s) const { return lua_type(L_, s) == LUA_TSTRING; }
+    bool isnumber(LuaSlot s) const { return lua_type(L_, s) == LUA_TNUMBER; }
+    bool isinteger(LuaSlot s) const;
+    bool isint(LuaSlot s) const;
+    bool isthread(LuaSlot s) const { return lua_type(L_, s) == LUA_TTHREAD; }
+    bool isfunction(LuaSlot s) const { return lua_type(L_, s) == LUA_TFUNCTION; }
+    bool iscfunction(LuaSlot s) const { return lua_iscfunction(L_, s) != 0; }
+    bool isboolean(LuaSlot s) const { return lua_type(L_, s) == LUA_TBOOLEAN; }
+    bool isnil(LuaSlot s) const { return lua_type(L_, s) == LUA_TNIL; }
+    bool istoken(LuaSlot s) const { return lua_islightuserdata(L_, s) != 0; }
+
+    void checktable(LuaSlot s, const char *n) const { if (!istable(s)) argerr(n, "table"); }
+    void checkstring(LuaSlot s, const char *n) const { if (!isstring(s)) argerr(n, "string"); }
+    void checknumber(LuaSlot s, const char *n) const { if (!isnumber(s)) argerr(n, "number"); }
+    void checkinteger(LuaSlot s, const char *n) const { if (!isinteger(s)) argerr(n, "integer"); }
+    void checkint(LuaSlot s, const char *n) const { if (!isint(s)) argerr(n, "int"); }
+    void checkthread(LuaSlot s, const char *n) const { if (!isthread(s)) argerr(n, "thread"); }
+    void checkfunction(LuaSlot s, const char *n) const { if (!isfunction(s)) argerr(n, "function"); }
+    void checkcfunction(LuaSlot s, const char *n) const { if (!iscfunction(s)) argerr(n, "cfunction"); }
+    void checkboolean(LuaSlot s, const char *n) const { if (!isboolean(s)) argerr(n, "boolean"); }
+    void checknil(LuaSlot s, const char *n) const { if (!isnil(s)) argerr(n, "nil"); }
+    void checktoken(LuaSlot s, const char *n) const { if (!istoken(s)) argerr(n, "token"); }
+    
+    bool ckboolean(LuaSlot s) const;
+    lua_Integer ckinteger(LuaSlot s) const;
+    int ckint(LuaSlot s) const;
+    lua_Number cknumber(LuaSlot s) const;
+    eng::string ckstring(LuaSlot s) const;
+    std::string_view ckstringview(LuaSlot s) const;
+    lua_State *ckthread(LuaSlot s) const;
+    LuaToken cktoken(LuaSlot s) const;
+    
+    void clearmetatable(LuaSlot tab) const;
+    void setmetatable(LuaSlot tab, LuaSlot mt) const;
+    bool getmetatable(LuaSlot mt, LuaSlot tab) const;
+
+    void newtable(LuaSlot target) const;
+    void createtable(LuaSlot target, int narr, int nrec) const;
+    lua_State *newthread(LuaSlot target) const;
+    void getglobaltable(LuaSlot gltab) const;
+    void cleartable(LuaSlot tab, bool clearmeta) const;
+
+    int rawlen(LuaSlot val) const;
+    
+    int next(LuaSlot tab, LuaSlot key, LuaSlot value) const;
+
+    // Return true if the classname is legal.
+    bool validclassname(LuaSlot value) const;
+    static bool validclassname(std::string_view cname);
+
+    // Return the class name if x is a valid classtab.
+    // Otherwise, returns empty string.  If nonempty, the
+    // result is guaranteed to be a validclassname.
+    // This can also function as an "isclass" operator.
+    eng::string classname(LuaSlot x) const;
+
+    // Look up a class.
+    // If there is a problem, returns an error message.
+    // There are lots of error conditions, including such things
+    // as no such class, corrupted class, classname invalid, etc.
+    eng::string getclass(LuaSlot tab, LuaSlot name) const;
+    eng::string getclass(LuaSlot tab, std::string_view name) const;
+
+    // Create a class, or look up an existing class.
+    // WARNING: this routine assert-fails if the parameter is not
+    // a valid classname.  Check the classname before calling this!
+    void makeclass(LuaSlot tab, LuaSlot name) const;
+    void makeclass(LuaSlot tab, std::string_view name) const;
+
+    // Get the ID of a tangible.  It's a little weird to put this in
+    // this module.
+    int64_t tanid(LuaSlot tab) const;
+
+    // Return true if the value is a sortable key (string, number, or boolean).
+    bool issortablekey(LuaSlot s) const;
+
+    // Move a sortable key (string, number, or boolean) from one lua
+    // environment to another lua environment.  WARNING: this assert-fails
+    // if the value is not a sortable key.
+    void movesortablekey(LuaSlot val, LuaStack &other, LuaSlot otherslot);
+
+    bool rawequal(LuaSlot v1, LuaSlot v2) const {
+        return lua_rawequal(L_, v1, v2);
+    }
+
+    bool rawequal(LuaSlot v1, const char *name) const {
+        push_any_value(name);
+        bool result = lua_rawequal(L_, v1, -1);
+        lua_pop(L_, 1);
+        return result;
+    }
+    
+    template<typename VT>
+    void set(LuaSlot target, VT value) const {
+        push_any_value(value);
+        lua_replace(L_, target);
+    }
+
+    template<typename KT>
+    void rawget(LuaSlot target, LuaSlot tab, KT key) const {
+        push_any_value(key);
+        lua_rawget(L_, tab);
+        lua_replace(L_, target);
+    }
+
+    void rawget(LuaSlot target, LuaSlot tab, int key) const {
+         lua_rawgeti(L_, tab, key);
+         lua_replace(L_, target);
+    }
+    
+    template<typename KT, typename VT>
+    void rawset(LuaSlot tab, KT key, VT value) const {
+        push_any_value(key);
+        push_any_value(value);
+        lua_rawset(L_, tab);
+    }
+    
+    template<typename VT>
+    void rawset(LuaSlot tab, int key, VT value) const {
+        push_any_value(value);
+        lua_rawseti(L_, tab, key);
+    }
+    
+    // Lua flagbits manipulation: Table types.
+    int gettabletype(LuaSlot tab) const;
+    void settabletype(LuaSlot tab, int t) const;
+
+    // If slot is a table, returns the LUA_TT_XXX table type.
+    // If slot is not a table, returns the LUA_TXXX general type.
+    int xtype(LuaSlot slot) const;
+
+    // Lua flagbits manipulation: visited bit.
+    bool getvisited(LuaSlot tab) const;
+    void setvisited(LuaSlot tab, bool visited) const;
+
+    // Return true if the int64 value can be stored as a lua number.
+    static bool int64_storable(int64_t v) { return (v <= MAXINT) && (v >= -MAXINT); }
+};
+
+// This is a helper class to help parse tables full of keywords.
+class LuaKeywordParser {
+    struct cmp_char {
+        bool operator () (const char *s1, const char *s2) const {
+            return strcmp(s1, s2) < 0;
+        };
+    };
+private:
+    bool not_table_;
+    lua_State *L_;
+    int slot_;
+    eng::set<const char *, cmp_char> parsed_;
+
+    void init(const lua_State *L, int slot);
+public:
+    // If the slot is not a table, sets the not_table
+    // flag and creates a dummy table in the slot.
+    LuaKeywordParser(lua_State *L, int slot);
+    LuaKeywordParser(const LuaStack &LS, LuaSlot slot) : LuaKeywordParser(LS.state(), slot.index()) {}
+
+    // Fetch a value from the table.  This never throws.
+    // Return true if the value is non-nil.
+    bool parse(LuaSlot slot, const char *kw);
+
+    // Check if there were any errors.  If so, return an
+    // error message.
+    eng::string final_check();
+
+    // Check if there are any errors.  If so, throw a lua error.
+    void final_check_throw();
+
+    // Fetch the state pointer. 
+    lua_State *state() const { return L_; }
+};
+
+class LuaConstantReg : public eng::nevernew {
+private:
+    const char *name_;
+    const char *docs_;
+    LuaToken tokenvalue_;
+    lua_Number numbervalue_;
+    LuaConstantReg *next_;
+
+public:
+    static LuaConstantReg *All;
+    LuaConstantReg(const char *name, const char *docs, LuaToken tokenvalue, lua_Number numbervalue);
+
+    const char *get_name() const { return name_; }
+    const char *get_docs() const { return docs_; }
+    LuaToken get_tokenvalue() const { return tokenvalue_; }
+    lua_Number get_numbervalue() const { return numbervalue_; }
+    LuaConstantReg *next() const { return next_; }
+};
+
+class LuaFunctionReg : public eng::nevernew {
+private:
+    const char *name_;
+    const char *args_;
+    const char *docs_;
+    bool sandbox_;
+    lua_CFunction func_;
+    LuaFunctionReg *next_;
+
+public:
+    static LuaFunctionReg *All;
+    LuaFunctionReg(const char *name, const char *args, const char *docs, bool sand, lua_CFunction f);
+    static const LuaFunctionReg *lookup(lua_CFunction fn);
+
+    const char *get_name() const { return name_; }
+    const char *get_args() const { return args_; }
+    const char *get_docs() const { return docs_; }
+    lua_CFunction get_func() const { return func_; }
+    bool get_sandbox() const { return sandbox_; }
+    LuaFunctionReg *next() const { return next_; }
+    void set_func(lua_CFunction fn) { func_ = fn; }
+};
+
+#define LuaTokenConstant(name, tvalue, docs) \
+    LuaConstantReg reg_##name(#name, docs, LuaToken(tvalue), 0);
+
+#define LuaNumberConstant(name, nvalue, docs) \
+    LuaConstantReg reg_##name(#name, docs, LuaToken(), nvalue);
+
+#define LuaDefine(name, args, docs) \
+    int lfn_##name(lua_State *L); \
+    LuaFunctionReg reg_##name(#name, args, docs, false, lfn_##name); \
+    int lfn_##name(lua_State *L)
+
+#define LuaSandbox(name, args, docs) \
+    int lfn_##name(lua_State *L); \
+    LuaFunctionReg reg_##name(#name, args, docs, true, lfn_##name); \
+    int lfn_##name(lua_State *L)
+
+#define LuaDefineBuiltin(name, args, docs) \
+    LuaFunctionReg reg_##name(#name, args, docs, false, nullptr);
+
+#define LuaSandboxBuiltin(name, args, docs) \
+    LuaFunctionReg reg_##name(#name, args, docs, true, nullptr);
+
+#define LuaStringify(x) #x
+#define LuaAssert(L, x) if (!(x)) { luaL_error((L), "Assert failed: %s (file %s line %d)", LuaStringify(x), __FILE__, __LINE__); }
+#define LuaAssertStrEq(L, x, y) { eng::string _s1_(x); eng::string _s2_(y); if (_s1_ != _s2_) luaL_error((L), "Assert failed: value=%s (file %s line %d)", _s1_.c_str(), __FILE__, __LINE__); }
+
+#endif // LUASTACK_HPP

luprex/cpp/core/planemap.hpp

@@ -0,0 +1,247 @@
+//////////////////////////////////////////////////////////////
+//
+// PLANEMAP
+//
+// This module defines two classes: PlaneMap, and PlaneItem.  A
+// PlaneItem is an object that has a plane (a string) and an
+// XYZ position.  A PlaneMap keeps track of potentially thousands
+// of PlaneItem objects, allowing you to search for PlaneItems
+// by scanning a geographic region.
+//
+// CLASS SPRITE DERIVES FROM PLANEITEM
+//
+// A PlaneMap records the positions of PlaneItems.  The intent is
+// that class Sprite should derive from PlaneItem.  That way, the
+// PlaneMap can record the positions of sprites.
+//
+// When you put derived types like Sprite into a PlaneMap, the
+// PlaneMap will work fine.  However, when you scan the PlaneMap,
+// it will give return PlaneItem pointers, not Sprite pointers.
+//
+// If you're sure that the PlaneMap contains only sprites, then
+// you can use static_cast to convert those PlaneItem pointers to
+// Sprite pointers.  Sadly, there's no simple way to avoid the casting.
+//
+// THE SCANNABLE REGION IS FINITE
+//
+// A plane is a rectangle, with a finite size: if you stray more
+// than 80,000,000 meters from the origin, then that PlaneItem
+// is beyond the scannable region.
+//
+// There's nothing stopping you from moving a PlaneItem outside
+// the scannable region.  It is not an error to do so.
+// However, if you do move a PlaneItem outside the scannable
+// region, then that PlaneItem will not show up for scan_radius.
+//
+// PLANES CANNOT BE DESTROYED BUT THEY CAN BE UNINHABITED
+//
+// You can't "create" or "destroy" planes.  Instead, we say
+// that a plane is "uninhabited" until you put a PlaneItem
+// there.  Initially, all possible planes exist, but they're
+// all uninhabited until you put a PlaneItem there.  Planes
+// that are uninhabited take no memory.
+//
+// THE NOWHERE PLANE
+//
+// If you use the literal "nowhere" as a plane name, a special case is
+// triggered: you're "nowhere."  Radius scans can never find items
+// whose plane is "nowhere."  The same also applies when you use the
+// empty string as a plane name.
+//
+// THE Z COORDINATE IS IGNORED
+//
+// Class PlaneItem stores X, Y, and Z.  The Z coordinate is
+// ignored for all plane-scanning operations.  In other words,
+// planes are 2D.  The only reason we store a Z coordinate
+// is for the consistency of storing the model's X, Y, and Z
+// all in the same place.
+//
+// MEMORY MANAGEMENT
+//
+// PlaneMaps do not own PlaneItems.  This is a deliberate choice.
+// We assume that sprites will be owned by the sprite ID table.
+// So therefore, the PlaneMaps shouldn't own the sprites.
+//
+// So instead, we use this rule: a PlaneMap has a function 'track'
+// that causes it to start tracking the location of a PlaneItem.
+// However, the PlaneMap still doesn't own the PlaneItem.
+// If you destroy a PlaneMap, all the PlaneItems
+// will automatically be untracked, but they won't be deleted.
+//
+//////////////////////////////////////////////////////////////
+
+#ifndef PLANEMAP_HPP
+#define PLANEMAP_HPP
+
+#include "wrap-vector.hpp"
+#include "wrap-map.hpp"
+#include "wrap-string.hpp"
+#include "wrap-bytell-hash-map.hpp"
+#include "util.hpp"
+#include "luastack.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <string_view>
+
+class PlaneMap;
+class PlaneTree;
+
+class PlaneScan : public eng::nevernew {
+public:
+    friend class PlaneMap;
+    friend class PlaneTree;
+    enum Shape { BOX, CYLINDER, SPHERE };
+private:
+    // The plane to scan.
+    eng::string plane_;
+
+    // The bounding box of the scan.
+    util::XYZ center_, radius_;
+
+    // If you scan a cylinder or SPHERE, it actually
+    // scans the bounding box first, then clips out
+    // the parts that aren't correct.
+    Shape shape_;
+
+    // When true, the items are sorted by ID.
+    // WARNING: setting this to false can create
+    // nondeterminism.  Scans by lua should always be sorted.
+    bool sorted_;
+
+    // The near ID, if nonzero, is either PREPENDED to the
+    // results, or OMITTED from the results, depending on include_near_.
+    int64_t near_;
+    bool include_near_;
+    
+    // If this is true, items on the nowhere plane are not scanned.
+    bool omit_nowhere_;
+
+public:
+    void clear() {
+        plane_ = "";
+        shape_ = BOX;
+        sorted_ = true;
+        near_ = 0;
+        include_near_ = false;
+        omit_nowhere_ = false;
+        radius_ = center_ = util::XYZ();
+    }
+    PlaneScan() { clear(); }
+    
+    // Convert a lua table into a scan configuration.
+    //
+    // If there is an error in the configuration, 
+    // throws a lua error message.
+    //
+    // Caution: if this detects the configuration flag 'near',
+    // then it stores the near ID.  However, it doesn't fetch
+    // the center and plane.  It is the caller's responsibility
+    // to check if 'near' has been set, and if so, store a center
+    // and plane.  This is admittedly ugly, but it eliminates
+    // a dependency on the world module.
+    //
+    void configure(LuaKeywordParser &kw);
+
+    void set_bbox_given_center_radius(const util::XYZ &center, float r) {
+        set_center(center);
+        set_radius(r);
+    }
+
+    void set_whole_plane() {
+        shape_ = BOX;
+        center_ = util::XYZ();
+        radius_.x = std::numeric_limits<float>::infinity();
+        radius_.y = radius_.z = radius_.x;
+    }
+
+    void set_center(const util::XYZ &center) { center_ = center; }
+    void set_radius(const util::XYZ &radius) { radius_ = radius; }
+    void set_radius(float f) { radius_.x = radius_.y = radius_.z = f; }
+    void set_plane(std::string_view p) { plane_ = p; }
+    void set_shape(Shape s) { shape_ = s; }
+    void set_sorted(bool s) { sorted_ = s; }
+    void set_near(int64_t id, bool inc) { near_ = id; include_near_ = inc; }
+    void set_omit_nowhere(bool b) { omit_nowhere_ = b; }
+
+    int64_t near() const { return near_; }
+
+    // Reveal the contents of the PlaneScan as a string.
+    eng::string debug_string() const;
+};
+
+class PlaneItem : public eng::nevernew {
+    friend class PlaneTree;
+    friend class PlaneMap;    
+private:
+    PlaneTree *tree_;
+    PlaneItem *prev_;
+    PlaneItem *next_;
+    int64_t id_;
+    eng::string plane_;
+    float x_, y_, z_;
+
+public:
+    PlaneItem();
+    ~PlaneItem();
+
+    // You may modify the ID at any time.  
+    void set_id(int64_t id) { id_ = id; }
+
+    int64_t id() const { return id_; }
+    const eng::string &plane() const { return plane_; }
+    const float x() const { return x_; }
+    const float y() const { return y_; }
+    const float z() const { return z_; }
+
+    void track(PlaneMap *pmap);
+    void untrack() { track(nullptr); }
+
+    void set_pos(const eng::string &plane, float x, float y, float z);
+    void set_xyz(float x, float y, float z);
+};
+
+class PlaneMap : public eng::nevernew  {
+    friend class PlaneItem;
+    friend class PlaneTree;
+public:
+    using IdVector = util::IdVector;
+
+private:
+    float default_radius_;
+    eng::map<eng::string, std::unique_ptr<PlaneTree>> planes_;
+
+public:
+    // No special code is needed for construction or destruction.
+    PlaneMap();
+    ~PlaneMap();
+
+    // The 'insert' and 'remove' operators are inside class
+    // PlaneItem.  See PlaneItem::track and PlaneItem::untrack.
+    
+    // Scan the PlaneMap for items, return their IDs.
+    IdVector scan(const PlaneScan &s) const;
+    
+    // Set the default radius for all planes.
+    // Maybe we'll make it adaptive some day.
+    void set_default_radius(float r) { default_radius_ = r; }
+
+    // Return a debug string for the specified plane.
+    // This is for unit testing.
+    eng::string tree_debug_string(const eng::string &plane);
+    eng::string outliers_debug_string(const eng::string &plane);
+    eng::string search_bboxes_debug_string(const PlaneScan &scan);
+    eng::string scan_steps_debug_string(const PlaneScan &scan);
+
+    // Untrack all planeitems.  This is for unit testing.
+    void untrack_all();
+
+private:
+    // unit testing stuff.
+    friend int lfn_unittests_planemap(lua_State *L);
+};
+
+
+#endif // PLANEMAP_HPP
+
+

luprex/cpp/core/pprint.cpp

@@ -0,0 +1,303 @@
+
+#include <ostream>
+#include "pprint.hpp"
+#include "util.hpp"
+#include "table.hpp"
+
+#include <iostream>
+#include <cmath>
+
+
+void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os) {
+    int tt = LS.type(val);
+    switch (tt) {
+    case LUA_TNIL:
+        (*os) << "nil";
+        return;
+    case LUA_TSTRING:
+        if (quote) {
+            util::quote_string(LS.ckstring(val), os);
+        } else {
+            // TODO: this could be more efficient.
+            (*os) << LS.ckstring(val);
+        }   
+        return;
+    case LUA_TNUMBER: {
+        double value = LS.cknumber(val);
+        if (std::isnan(value)) {
+            (*os) << "nan";
+        } else {
+            int64_t ivalue = int64_t(value);
+            if (double(ivalue) == value) {
+                (*os) << ivalue;
+            } else {
+                (*os) << value;
+            }
+        }
+        return;
+    }
+    case LUA_TBOOLEAN:
+        (*os) << (LS.ckboolean(val) ? "true" : "false");
+        return;
+    case LUA_TFUNCTION: {
+        (*os) << "<function>";
+        return;
+    }
+    case LUA_TLIGHTUSERDATA: {
+        LuaToken token = LS.cktoken(val);
+        (*os) << "[" << token.str() << "]";
+        return;
+    }
+    default:
+        (*os) << "<" << lua_typename(LS.state(), tt) << ">";
+        return;
+    }
+}
+
+
+// Find tables recursively.
+//
+// Builds a table (tabcount) whose keys are tables.  If a table
+// is visited exactly once, then tabcount[t]=0.  If a table is
+// visited more than once, then tabcount[t]=-1.
+//
+static void findtables(LuaStack &LS0, LuaSlot root, LuaSlot tabcount) {
+    lua_State *L = LS0.state();
+    LuaVar key, val, tab, count;
+    LuaStack LS(L, key, val, tab, count);
+
+    LS.newtable(tabcount);
+    int top = lua_gettop(L);
+    if (LS.istable(root)) {
+        lua_pushvalue(L, root.index());
+    }
+    while (lua_gettop(L) > top) {
+        lua_checkstack(L, 20);
+        lua_replace(L, tab.index());
+        LS.rawget(count, tabcount, tab);
+        if (LS.isnil(count)) {
+            LS.rawset(tabcount, tab, 0);
+            LS.set(key, LuaNil);
+            while (LS.next(tab, key, val)) {
+                lua_checkstack(L, 20);
+                if (LS.istable(key)) {
+                    lua_pushvalue(L, key.index());
+                }
+                if (LS.istable(val)) {
+                    lua_pushvalue(L, val.index());
+                }
+            }
+            LS.getmetatable(val, tab);
+            if (LS.istable(val)) {
+                lua_pushvalue(L, val.index());
+            }
+        } else {
+            LS.rawset(tabcount, tab, -1);
+        }
+    }
+    LS.result();
+}
+
+LuaDefine(table_findtables, "root", "recursively find tables (debugging only)") {
+    LuaArg root;
+    LuaRet tabcount;
+    LuaStack LS(L, root, tabcount);
+    findtables(LS, root, tabcount);
+    return LS.result();
+}
+
+struct Inspector {
+    lua_State *L;
+    LuaVar ids;
+    int nextid;
+    bool indent;
+    int maxlen;
+    bool anyindent;
+    std::ostream *stream;
+};
+
+static void tabify(Inspector &insp, int level) {
+    if (insp.indent) {
+        (*insp.stream) << std::endl;
+        for (int i = 0; i < level; i++) {
+            (*insp.stream) << "  ";
+        }
+        insp.anyindent = true;
+    } else {
+        (*insp.stream) << " ";
+    }
+}
+
+static void pprint_r(Inspector &insp, int level, LuaSlot root) {
+    lua_checkstack(insp.L, 20);
+    LuaVar idv, pairs, key, val, nextseq;
+    LuaStack LS(insp.L, idv, pairs, key, val, nextseq);
+
+    // If it's anything but a table, use 'atomic_print'.
+    if (!LS.istable(root)) {
+        atomic_print(LS, root, true, insp.stream);
+        LS.result();
+        return;
+    }
+
+    // Determine the table's ID, allocating an ID if necessary.
+    LS.rawget(idv, insp.ids, root);
+    int id = LS.ckint(idv);
+    bool new_id = false;
+    if (id < 0) {
+        new_id = true;
+        id = insp.nextid++;
+        LS.rawset(insp.ids, root, id);
+    }
+    
+    // Print the table's name, if any.
+    bool is_class = false;
+    bool is_tangible = false;
+    eng::string cname = LS.classname(root);
+    if (cname != "") {
+        is_class = true;
+        (*insp.stream) << "<class " << cname << ">";
+    } else {
+        int64_t tid = LS.tanid(root);
+        if (tid > 0) {
+            is_tangible = true;
+            (*insp.stream) << "<tangible " << tid << ">";
+        } else if (id > 0) {
+            (*insp.stream) << "<table " << id << ">";
+        }
+    }
+
+    // If this is a class, and we're not at the top level, truncate.
+    if ((is_class || is_tangible) && (level > 0)) {
+        LS.result();
+        return;
+    }
+
+    // If this is a table we've already printed, truncate it.
+    if ((id > 0) && (!new_id)) {
+        if (lua_nkeys(insp.L, root.index())==0) {
+            (*insp.stream) << "{}";
+        } else {
+            (*insp.stream) << "{...}";
+        }
+        LS.result();
+        return;
+    }
+
+    // State variables.
+    bool needcomma = false;
+    bool multiline = false;
+    LS.set(nextseq, 1);
+
+    // Open the brackets.
+    (*insp.stream) << "{";
+
+    // Output the table keys.
+    table_getpairs(LS, root, pairs, true);
+    for (int i = 2; ; i+=2) {
+        LS.rawget(key, pairs, i);
+        if (LS.isnil(key)) break;
+        LS.rawget(val, pairs, i+1);
+        if (needcomma) (*insp.stream) << ",";
+        needcomma = true;
+        if (LS.rawequal(key, nextseq)) {
+            (*insp.stream) << " ";
+            pprint_r(insp, level + 1, val);
+            LS.set(nextseq, LS.ckinteger(nextseq) + 1);
+        } else {
+            multiline = true;
+            tabify(insp, level + 1);
+            if (LS.isstring(key) && sv::is_lua_id(LS.ckstring(key))) {
+                (*insp.stream) << LS.ckstring(key);
+            } else {
+                (*insp.stream) << "[";
+                pprint_r(insp, level + 1, key);
+                (*insp.stream) << "]";
+            }
+            if (insp.indent) {
+                (*insp.stream) << " = ";
+            } else {
+                (*insp.stream) << "=";
+            }
+            pprint_r(insp, level + 1, val);
+        }
+    }
+
+    // Output the metatable.
+    LS.getmetatable(val, root);
+    if (LS.istable(val)) {
+        multiline = true;
+        if (needcomma) (*insp.stream) << ",";
+        needcomma = true;
+        tabify(insp, level + 1);
+        (*insp.stream) << "<meta> = ";
+        pprint_r(insp, level + 1, val);
+    }
+    
+    // Close the brackets.
+    if (multiline) {
+        tabify(insp, level);
+    } else if (LS.ckinteger(nextseq) > 1) {
+        (*insp.stream) << " ";
+    }
+    (*insp.stream) << "}";
+    
+    LS.result();
+}
+
+void pprint(LuaStack &LS0, LuaSlot root, bool indent, std::ostream *os) {
+    Inspector insp;
+    LuaStack LS(LS0.state(), insp.ids);
+    findtables(LS, root, insp.ids);
+    insp.L = LS0.state();
+    insp.nextid = 1;
+    insp.indent = indent;
+    insp.anyindent = false;
+    insp.stream = os;
+    pprint_r(insp, 0, root);
+    LS.result();
+}
+
+LuaDefine(string_isidentifier, "str", "return true if the string is a valid lua identifier") {
+    LuaArg str;
+    LuaRet result;
+    LuaStack LS(L, str, result);
+    if (LS.isstring(str)) {
+        eng::string s = LS.ckstring(str);
+        LS.set(result, sv::is_lua_id(s));
+    } else {
+        LS.set(result, false);
+    }
+    return LS.result();
+}
+
+LuaDefine(string_print, "obj", "print the specified object into a string") {
+    LuaArg val;
+    LuaRet result;
+    LuaStack LS(L, val, result);
+    eng::ostringstream oss;
+    atomic_print(LS, val, false, &oss);
+    LS.set(result, oss.str());
+    return LS.result();
+}
+
+LuaDefine(string_pprint, "obj,indent", "pretty-print the specified object into a string") {
+    LuaArg root, indent;
+    LuaRet result;
+    LuaStack LS(L, root, indent, result);
+    bool ind = LS.ckboolean(indent);
+    eng::ostringstream oss;
+    pprint(LS, root, ind, &oss);
+    LS.set(result, oss.str());
+    return LS.result();
+}
+
+LuaDefine(tostring, "obj", "print the specified object into a string") {
+    LuaArg val;
+    LuaRet result;
+    LuaStack LS(L, val, result);
+    eng::ostringstream oss;
+    atomic_print(LS, val, false, &oss);
+    LS.set(result, oss.str());
+    return LS.result();
+}

luprex/cpp/core/pprint.hpp

@@ -0,0 +1,37 @@
+//////////////////////////////////////////////////////////////////////////////////
+//
+// print, pprint, and tostring
+//
+// This module implements the heart of the lua 'print', lua 'pprint', and lua
+// 'tostring' functions. Note that we have to override the lua builtins 'print'
+// and 'tostring' for two reasons:
+//
+//   * We need to suppress the printing of table addresses, for determinism.
+//   * We need to channel the output to a PrintBuffer in the world model.
+//
+// Note that the actual lua 'print' and 'pprint' routines aren't defined in this
+// module, they're in the World module, because they send their output into the
+// PrintBuffer of the world model.  But all the tricky code to implement 'print'
+// and 'pprint' are in this module.
+//
+//////////////////////////////////////////////////////////////////////////////////
+
+#ifndef PPRINT_HPP
+#define PPRINT_HPP
+
+#include "luastack.hpp"
+#include <ostream>
+
+// Atomic print to a stream.
+//
+// This prints an atomic value to a stream.  If you give it a table,
+// it just prints "<table>".  This routine is the heart of the lua
+// primitives 'print' and 'tostring'.
+//
+void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os);
+
+// Pretty print to a stream.
+//
+void pprint(LuaStack &LS, LuaSlot val, bool indent, std::ostream *os);
+
+#endif // PPRINT_HPP

luprex/cpp/core/printbuffer.cpp

@@ -0,0 +1,275 @@
+#include "wrap-sstream.hpp"
+
+#include "printbuffer.hpp"
+
+#include <algorithm>
+#include <cstdio>
+#include <cinttypes>
+
+struct PrintBufferCore  : public eng::opnew {
+    // The most recent lines printed.
+    eng::deque<eng::string> lines_;
+
+    // Line number of the first line in the buffer.  From this, all other
+    // line numbers can be inferred.
+    int first_line_;
+
+    // Line number of the first unchecked line in the buffer.  All line
+    // numbers including this one and beyond are unchecked.
+    int first_unchecked_;
+
+    // Constructor.
+    PrintBufferCore() : first_line_(0), first_unchecked_(0) {}
+};
+
+static PrintBufferCore shared_core;
+
+PrintBuffer::PrintBuffer() {
+    core_ = &shared_core;
+}
+
+PrintBuffer::~PrintBuffer() {
+    if (core_ != &shared_core) delete core_;
+}
+
+int PrintBuffer::first_line() const {
+    return core_->first_line_;
+}
+
+int PrintBuffer::last_line() const {
+    return core_->first_line_ + core_->lines_.size();
+}
+
+int PrintBuffer::first_unchecked() const {
+    return core_->first_unchecked_;
+}
+
+bool PrintBuffer::never_printed() const {
+    return (core_->first_line_==0) && (core_->first_unchecked_==0) && (core_->lines_.size()==0);
+}
+
+const eng::string &PrintBuffer::nth(int n) const {
+    return core_->lines_[n - core_->first_line_];
+}
+
+eng::string PrintBuffer::debug_string() const {
+    eng::ostringstream oss;
+    oss << core_->first_line_ << "," << core_->first_unchecked_ << ":";
+    for (int i = 0; i < int(core_->lines_.size()); i++) {
+        oss << core_->lines_[i] << ";";
+    }
+    return oss.str();
+}
+
+void PrintBuffer::clear() {
+    if (core_ != &shared_core) {
+        delete core_;
+        core_ = &shared_core;
+    }
+}
+
+static int first_line_len(const char *text, int len) {
+    for (int i = 0; i < len; i++) {
+        if (text[i] == '\n') return i;
+    }
+    return len;
+}
+
+static void add_line(PrintBufferCore *core, const char *text, int len, bool auth) {
+    assert(core != &shared_core);
+    core->lines_.emplace_back(text, len);
+    if (auth) {
+        core->first_unchecked_ = core->first_line_ + int(core->lines_.size());
+    }
+}
+
+void PrintBuffer::add_string(const eng::string &s, bool auth) {
+    const char *text = s.c_str();
+    int len = s.size();
+    if (core_ == &shared_core) core_ = new PrintBufferCore;
+    while (true) {
+        int fll = first_line_len(text, len);
+        if (fll == len) {
+            if (len > 0) {
+                add_line(core_, text, len, auth);
+            }
+            return;
+        } else {
+            add_line(core_, text, fll, auth);
+            text += (fll + 1);
+            len -= (fll + 1);
+        }
+    }
+}
+
+void PrintBuffer::discard_upto(int n) {
+    if (core_ == &shared_core) return;
+    while ((!core_->lines_.empty()) && (core_->first_line_ < n)) {
+        core_->lines_.pop_front();
+        core_->first_line_ += 1;
+    }
+    if (core_->first_unchecked_ < core_->first_line_) {
+        core_->first_unchecked_ = core_->first_line_;
+    }
+}
+
+void PrintBuffer::serialize(StreamBuffer *sb) const {
+    if (never_printed()) {
+        sb->write_bool(true);
+    } else {
+        sb->write_bool(false);
+        sb->write_uint32(core_->first_line_);
+        sb->write_uint32(core_->first_unchecked_);
+        sb->write_uint32(core_->lines_.size());
+        for (const eng::string &s : core_->lines_) {
+            sb->write_string(s);
+        }
+    }
+}
+
+void PrintBuffer::deserialize(StreamBuffer *sb) {
+    bool never_printed = sb->read_bool();
+    if (never_printed) {
+        clear();
+    } else {
+        if (core_ == &shared_core) core_ = new PrintBufferCore;
+        core_->first_line_ = sb->read_uint32();
+        core_->first_unchecked_ = sb->read_uint32();
+        int nlines = sb->read_uint32();
+        core_->lines_.clear();
+        for (int i = 0; i < nlines; i++) {
+            core_->lines_.push_back(sb->read_string());
+        }
+    }
+}
+
+void PrintBuffer::diff(const PrintBuffer &auth, StreamBuffer *sb) const {
+    // Currently, the implementation is simple.  The synchronous model discards
+    // all prediction lines from its buffer, then the server retransmits all
+    // those lines.  So this barely counts as difference transmission - it's
+    // just transmission, regardless of what was in the synchronous model.  I
+    // think that's okay for the text console.
+    // Ask the client to discard anything that precedes auth_first.
+    sb->write_int32(auth.first_line());
+    sb->write_int32(auth.last_line());
+    for (int i = core_->first_unchecked_; i < auth.last_line(); i++) {
+        eng::string line;
+        if (i >= auth.first_line()) line = auth.nth(i);
+        sb->write_string(line);
+    }
+}
+
+void PrintBuffer::patch(StreamBuffer *sb, DebugCollector *dbc) {
+    DebugBlock dbb(dbc, "PrintBuffer::patch");
+    if (core_ == &shared_core) core_ = new PrintBufferCore;
+    int auth_first = sb->read_int32();
+    int auth_last = sb->read_int32();
+    core_->lines_.resize(core_->first_unchecked_ - core_->first_line_);
+    int nlines = auth_last - core_->first_unchecked_;
+    if (nlines > 0) DebugLine(dbc) << "PrintBuffer received " << nlines << " lines.";
+    for (int i = core_->first_unchecked_; i < auth_last; i++) {
+        core_->lines_.emplace_back(sb->read_string());
+    }
+    core_->first_unchecked_ = core_->first_line_ + core_->lines_.size();
+    discard_upto(auth_first);
+    if (core_->first_line_ < auth_first) {
+        core_->first_line_ = auth_first;
+    }
+}
+
+bool PrintChanneler::channel(const PrintBuffer *printbuffer, std::ostream &ostream) {
+    if (printbuffer == nullptr) return false;
+    if (printbuffer->first_line() > line_) {
+        line_ = printbuffer->first_line();
+    }
+    while (line_ < printbuffer->first_unchecked()) {
+        ostream << "|" << printbuffer->nth(line_) << std::endl;
+        line_ += 1;
+    }
+    return line_ > printbuffer->first_line();
+}
+
+Invocation PrintChanneler::invocation(int64_t actor_id) {
+    char buf[80];
+    sprintf(buf, "%" PRId64, line_);
+    return Invocation(Invocation::KIND_FLUSH_PRINTS, actor_id, actor_id, buf, InvocationData());
+}
+
+LuaDefine(unittests_printbuffer, "", "some unit tests") {
+    PrintBuffer pbm;
+    PrintBuffer pbs;
+    StreamBuffer sb;
+
+    // Test authoritative add_string and discard_upto.
+    LuaAssertStrEq(L, pbm.debug_string(), "0,0:");
+    pbm.add_string("foo\nbar\nbaz\n", true);
+    LuaAssertStrEq(L, pbm.debug_string(), "0,3:foo;bar;baz;");
+    pbm.add_string("a\nb\nc", true);
+    LuaAssertStrEq(L, pbm.debug_string(), "0,6:foo;bar;baz;a;b;c;");
+    pbm.discard_upto(2);
+    LuaAssertStrEq(L, pbm.debug_string(), "2,6:baz;a;b;c;");
+    pbm.discard_upto(8);
+    LuaAssertStrEq(L, pbm.debug_string(), "6,6:");
+
+    // Test nonauthoritative add_string and discard_upto.
+    LuaAssertStrEq(L, pbs.debug_string(), "0,0:");
+    pbs.add_string("foo\nbar\nbaz\n", false);
+    LuaAssertStrEq(L, pbs.debug_string(), "0,0:foo;bar;baz;");
+    pbs.add_string("a\nb\nc", false);
+    LuaAssertStrEq(L, pbs.debug_string(), "0,0:foo;bar;baz;a;b;c;");
+    pbs.discard_upto(2);
+    LuaAssertStrEq(L, pbs.debug_string(), "2,2:baz;a;b;c;");
+    pbs.discard_upto(8);
+    LuaAssertStrEq(L, pbs.debug_string(), "6,6:");
+
+    // Test serialization and deserialization of a nonempty printbuffer.
+    pbm.clear();
+    sb.clear();
+    pbm.add_string("boy\nhowdy\nyeah\n", true);
+    pbm.add_string("baby\nget\ndown\n", false);
+    LuaAssertStrEq(L, pbm.debug_string(), "0,3:boy;howdy;yeah;baby;get;down;");
+    pbm.serialize(&sb);
+    pbs.deserialize(&sb);
+    LuaAssertStrEq(L, pbs.debug_string(), "0,3:boy;howdy;yeah;baby;get;down;");
+
+    // Test serialization and deserialization of an empty printbuffer.
+    pbm.clear();
+    sb.clear();
+    LuaAssert(L, pbm.never_printed());
+    pbm.serialize(&sb);
+    pbs.deserialize(&sb);
+    LuaAssert(L, pbs.never_printed());
+    
+    pbm.clear();
+    pbs.clear();
+    sb.clear();
+    pbm.add_string("foo\nbar\n", true);
+    pbs.diff(pbm, &sb);
+    pbs.patch(&sb, nullptr);
+    LuaAssertStrEq(L, pbs.debug_string(), "0,2:foo;bar;");
+    pbm.clear();
+    pbm.add_string("foo\nyow\nding\ndong\n", true);
+    pbs.diff(pbm, &sb);
+    pbs.patch(&sb, nullptr);
+    LuaAssertStrEq(L, pbs.debug_string(), "0,4:foo;bar;ding;dong;");
+    pbs.discard_upto(2);
+    LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;");
+    pbs.diff(pbm, &sb);
+    pbs.patch(&sb, nullptr);
+    LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;");
+    pbs.add_string("boy\nhowdy\n", false);
+    LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;boy;howdy;");
+    pbs.diff(pbm, &sb);
+    pbs.patch(&sb, nullptr);
+    LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;");
+    pbs.add_string("boy\nhowdy\nyeah\nbaby\nget\ndown\n", false);
+    LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;boy;howdy;yeah;baby;get;down;");
+    pbs.discard_upto(5);
+    LuaAssertStrEq(L, pbs.debug_string(), "5,5:howdy;yeah;baby;get;down;");
+    pbs.diff(pbm, &sb);
+    pbs.patch(&sb, nullptr);
+    LuaAssertStrEq(L, pbs.debug_string(), "5,5:");
+    
+
+    return 0;
+}

luprex/cpp/core/printbuffer.hpp

@@ -0,0 +1,162 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// PRINTBUFFER
+//
+// Lua has a 'print' statement - in client-server mode, where does the output of
+// the 'print' statement go?  We need to be able to handle print-statements on
+// the server (and forward them to the client), and we need for predictive
+// prints to be handled gracefully.
+//
+// Class PrintBuffer is a buffer for storing the output of the print statement.
+// It is part of the actor tangible.  When a lua thread calls 'print', the print
+// goes into stringstream lthread_prints.  When the thread stops or yields, the
+// contents of lthread_prints are converted to lines and transferred to the
+// PrintBuffer of the actor. From there, it gets difference transmitted, and the
+// client can probe it.
+//
+// Suppose, for example, that the player logs in and invokes a plan that prints
+// six lines from a Robert Frost poem.  That plan is executed by both the master
+// model and the synchronous model.  When the thread finishes, the PrintBuffer
+// in the actor in the master model will contain this:
+//
+//    * Line 0: Whose woods these are I think I know.  [authoritative]
+//    * Line 1: His house is in the village though;    [authoritative]
+//    * Line 2: He will not see me stopping here       [authoritative]
+//    * Line 3: To watch his woods fill up with snow.  [authoritative]
+//    * Line 4: My little horse must think it queer    [authoritative]
+//    * Line 5: To stop without a farmhouse near.      [authoritative]
+//
+// Note that the buffer stores line numbers, which start from zero the moment
+// the player logs in.  In the master model, all lines are always authoritative
+// because everything in the master model is authoritative.  In the synchronous
+// model, the PrintBuffer is likely to contain the same strings, but the lines
+// will all be marked as [prediction] instead of [authoritative].
+//
+// When the server difference transmits, the difference transmission will fix up
+// any mistakes in the PrintBuffer in the synchronous model.  In the process, it
+// will also fix up any [prediction] changing it to [authoritative].
+//
+// Periodically, the oldest lines in the buffer will get discarded.  More on
+// this later.  When lines get discarded, the line numbers don't change.  For
+// example, if we were to discard three lines from the buffer above, this is
+// what would remain:
+//
+//    * Line 3: To watch his woods fill up with snow.  [authoritative]
+//    * Line 4: My little horse must think it queer    [authoritative]
+//    * Line 5: To stop without a farmhouse near.      [authoritative]
+//
+// The client will periodically probe the PrintBuffer.  Suppose it sees all six
+// lines of the Robert Frost poem.  The next time it probes the buffer, it will
+// still see those same six lines.  The client will continue to see those six
+// lines until it takes explicit steps.
+//
+// Here's how we keep the buffer from growing forever.  The client probes the
+// PrintBuffer, and sees some authoritative lines.  The client downloads and
+// stores those lines locally. Once the lines are stored locally in the client,
+// the client injects a command into the command stream: "delete from
+// PrintBuffer up to line N" into the command stream.  This will cause the
+// engine to discard the lines that the client no longer needs.
+//
+// Note that when the client injects a "delete from PrintBuffer" into the
+// command stream, that's an invoke-command that has to follow the same process
+// as any other client invoke command.  It can take time for it to get
+// processed. Therefore, the client must be prepared that it might see some
+// redundant lines for a little while.
+//
+// MEMORY EFFICIENCY.
+//
+// Every tangible will contain a printbuffer.  To avoid memory waste, the
+// implementation of PrintBuffer stores everything inside a dynamically
+// allocated "core" struct.  All printbuffers that contain nothing share a
+// common empty core.  That way, the empty printbuffers that will exist in 99% of
+// all tangibles will take up very little space.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+
+#ifndef PRINTBUFFER_HPP
+#define PRINTBUFFER_HPP
+
+#include "wrap-deque.hpp"
+#include "wrap-string.hpp"
+#include <ostream>
+#include "streambuffer.hpp"
+#include "util.hpp"
+#include "invocation.hpp"
+#include "debugcollector.hpp"
+
+#include <memory>
+
+struct PrintBufferCore;
+
+class PrintBuffer : public eng::nevernew {
+private:
+    PrintBufferCore *core_;
+
+public:
+    PrintBuffer();
+    ~PrintBuffer();
+
+    // Get the current first line.
+    int first_line() const;
+    
+    // Return the line number beyond the end of the buffer.
+    int last_line() const;
+    
+    // Get the current first unchecked line.
+    // Guaranteed to be between first_line and last_line inclusive.
+    int first_unchecked() const;
+    
+    // Return true if the printbuffer is in the initial state.
+    // Note: if you clear the buffer, it's back in the initial state.
+    bool never_printed() const;
+
+    // Get the specified line number.
+    const eng::string &nth(int n) const;
+    
+    // Print the entire contents of the buffer to a string (for unit testing).
+    eng::string debug_string() const;
+
+    // Clear the buffer
+    void clear();
+
+    // Add a string.  If the string doesn't end in a newline, a newline
+    // is added.  The string is broken into lines, and the lines are added
+    // to the PrintBuffer.
+    void add_string(const eng::string &s, bool auth);
+
+    // Discard lines up to but not including line N.
+    void discard_upto(int n);
+
+    // Serialization and deserialization
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
+    // Difference transmission
+    void diff(const PrintBuffer &auth, StreamBuffer *sb) const;
+    void patch(StreamBuffer *sb, DebugCollector *dbc);
+};
+
+
+class PrintChanneler : public eng::nevernew {
+private:
+    int64_t line_;
+public:
+    PrintChanneler() { line_ = 0; }
+
+    // Reset the print channeler.
+    void reset() { line_ = 0; }
+    
+    // Copy any new lines from the printbuffer to the stdostream.
+    // Update the current line number.  Return true if the printbuffer
+    // contains any lines that have already been channeled.
+    bool channel(const PrintBuffer *pb, std::ostream &ostream);
+
+    // Generate an invocation that removes unnecessary lines from the
+    // printbuffer.
+    Invocation invocation(int64_t actor_id);
+};
+
+
+#endif // PRINTBUFFER_HPP
+

luprex/cpp/core/sched.cpp

@@ -0,0 +1,107 @@
+
+#include "wrap-sstream.hpp"
+
+#include "sched.hpp"
+#include "streambuffer.hpp"
+#include "luastack.hpp"
+
+#include <ostream>
+
+bool SchedEntry::operator < (const SchedEntry &other) const {
+    if (clock_ < other.clock_) return true;
+    if (clock_ > other.clock_) return false;
+    if (thread_id_ < other.thread_id_) return true;
+    if (thread_id_ > other.thread_id_) return false;
+    if (place_id_ < other.place_id_) return true;
+    if (place_id_ > other.place_id_) return false;
+    return false;
+}
+
+eng::string SchedEntry::debug_string() const {
+    eng::ostringstream oss;
+    oss << "(" << clock_ << "," << thread_id_ << "," << place_id_ << ")";
+    return oss.str();
+}
+
+void Schedule::add(int64_t clk, int64_t thid, int64_t plid) {
+    assert(plid != 0);
+    assert(thid != 0);
+    schedule_.insert(SchedEntry(clk, thid, plid));
+}
+
+bool Schedule::ready(int64_t clk) const {
+    return (!schedule_.empty()) && (schedule_.begin()->clock() <= clk);
+}
+
+SchedEntry Schedule::pop() {
+    assert(!schedule_.empty());
+    SchedEntry result = *schedule_.begin();
+    schedule_.erase(schedule_.begin());
+    return result;
+}
+
+eng::string Schedule::debug_string() {
+    eng::ostringstream oss;
+    for (const SchedEntry &se : schedule_) {
+        oss << se.debug_string();
+    }
+    return oss.str();
+}
+
+void Schedule::serialize(StreamBuffer *sb) {
+    sb->write_uint32(schedule_.size());
+    for (const SchedEntry &entry : schedule_) {
+        sb->write_int64(entry.clock_);
+        sb->write_int64(entry.thread_id_);
+        sb->write_int64(entry.place_id_);
+    }
+}
+
+void Schedule::deserialize(StreamBuffer *sb) {
+    schedule_.clear();
+    size_t nentry = sb->read_uint32();
+    for (size_t i = 0; i < nentry; i++) {
+        int64_t clock = sb->read_int64();
+        int64_t thread_id = sb->read_int64();
+        int64_t place_id = sb->read_int64();
+        add(clock, thread_id, place_id);
+    }
+}
+
+LuaDefine(unittests_scheduler, "", "some unit tests") {
+    Schedule s, xs;
+    StreamBuffer sb;
+
+    // Put some stuff into a scheduler.
+    s.add(1, 5, 3);
+    s.add(4, 3, 2);
+    s.add(1, 3, 2);
+    s.add(4, 2, 3);
+
+    // Test serialize and deserialize.
+    LuaAssert(L, s.debug_string() == "(1,3,2)(1,5,3)(4,2,3)(4,3,2)");
+    s.serialize(&sb);
+    xs.deserialize(&sb);
+    LuaAssert(L, xs.debug_string() == "(1,3,2)(1,5,3)(4,2,3)(4,3,2)");
+
+    // Verify that pop, ready, and empty do the right thing.
+    LuaAssert(L, s.ready(0) == false)
+    LuaAssert(L, s.ready(1) == true)
+    LuaAssert(L, s.pop().debug_string() == "(1,3,2)");
+    LuaAssert(L, s.ready(0) == false)
+    LuaAssert(L, s.ready(1) == true)
+    LuaAssert(L, s.pop().debug_string() == "(1,5,3)");
+    LuaAssert(L, s.debug_string() == "(4,2,3)(4,3,2)");
+    LuaAssert(L, s.pop().debug_string() == "(4,2,3)");
+    LuaAssert(L, s.pop().debug_string() == "(4,3,2)");
+    LuaAssert(L, s.ready(10000) == false);
+    LuaAssert(L, s.empty());
+    LuaAssert(L, s.debug_string() == "");
+
+    // Test serialization of an empty streambuffer.
+    s.serialize(&sb);
+    xs.deserialize(&sb);
+    LuaAssert(L, xs.debug_string() == "");
+
+    return 0;
+}

luprex/cpp/core/sched.hpp

@@ -0,0 +1,48 @@
+#ifndef SCHED_HPP
+#define SCHED_HPP
+
+#include "wrap-set.hpp"
+
+#include "streambuffer.hpp"
+
+#include <cstdint>
+
+class SchedEntry : public eng::nevernew  {
+private:
+    friend class Schedule;
+    int64_t clock_;
+    int64_t thread_id_;
+    int64_t place_id_;
+
+public:
+    int64_t clock() const { return clock_; }
+    int64_t thread_id() const { return thread_id_; }
+    int64_t place_id() const { return place_id_; }
+
+    SchedEntry(int64_t clk, int64_t thid, int64_t plid) {
+        clock_ = clk;
+        thread_id_ = thid;
+        place_id_ = plid;
+    }
+
+    bool operator < (const SchedEntry &other) const;
+
+    eng::string debug_string() const;
+};
+
+class Schedule : public eng::nevernew  {
+private:
+    eng::set<SchedEntry> schedule_;
+public:
+    void add(int64_t clk, int64_t thid, int64_t plid);
+    bool ready(int64_t clk) const;
+    bool empty() const { return schedule_.empty(); }
+    SchedEntry pop();
+    eng::string debug_string();
+
+    void serialize(StreamBuffer *sb);
+    void deserialize(StreamBuffer *sb);
+};
+
+#endif // SCHED_HPP
+

luprex/cpp/core/source.cpp

@@ -0,0 +1,862 @@
+
+#define _USE_MATH_DEFINES
+#include <cmath>
+
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+#include "wrap-map.hpp"
+#include "wrap-set.hpp"
+#include "wrap-sstream.hpp"
+#include "util.hpp"
+#include "luastack.hpp"
+#include "traceback.hpp"
+#include "table.hpp"
+#include "source.hpp"
+#include "luasnap.hpp"
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+
+LuaDefine(makeclass, "classname", "create a class if it doesn't already exist") {
+    LuaArg classname;
+    LuaRet classtab;
+    LuaStack LS(L, classname, classtab);
+    if (!LS.isstring(classname)) {
+        luaL_error(L, "class name must be a string");
+    }
+    if (!LS.validclassname(classname)) {
+        luaL_error(L, "invalid class name: %s", LS.ckstring(classname).c_str());
+    };
+    LS.makeclass(classtab, classname);
+    return LS.result();
+}
+
+LuaDefine(getclass, "classname", "get the classtab with the specified name") {
+    LuaArg classname;
+    LuaRet classtab;
+    LuaStack LS(L, classname, classtab);
+    eng::string err = LS.getclass(classtab, classname);
+    if (err != "") {
+        luaL_error(L, "%s", err.c_str());
+    }
+    return LS.result();
+}
+
+LuaDefine(classname, "classtable", "get the class name from a class table") {
+    LuaArg table;
+    LuaRet result;
+    LuaStack LS(L, table, result);
+    eng::string rstr = LS.classname(table);
+    if (rstr == "") {
+        LS.set(result, LuaNil);
+    } else {
+        LS.set(result, rstr);
+    }
+    return LS.result();
+}
+
+static void get_reg_name(std::string_view name, std::string_view &classname, std::string_view &funcname) {
+    size_t upos = name.find('_');
+    if (upos == std::string_view::npos) {
+        funcname = name;
+        classname = "";
+    } else {
+        funcname = name.substr(upos + 1);
+        classname = name.substr(0, upos);
+    }
+}
+
+static void get_info_table(LuaStack &LS, LuaSlot db, LuaSlot info, const eng::string &fn) {
+    LS.rawget(info, db, fn);
+    if (!LS.istable(info)) {
+        LS.set(info, LuaNewTable);
+        LS.rawset(db, fn, info);
+    }
+    LS.rawset(info, "name", fn);
+}
+
+static void calculate_loadresult(LuaStack &LS0, LuaSlot info, const eng::string &fn, const eng::string &code) {
+    LuaVar loadresult;
+    LuaStack LS(LS0.state(), loadresult);
+    if (code == "") {
+        LS.rawset(info, "loadresult", "missing or empty source file");
+    } else {
+        eng::string chunk = "=" + fn;
+        luaL_loadbuffer(LS.state(), code.c_str(), code.size(), chunk.c_str());
+        lua_replace(LS.state(), loadresult.index());
+        LS.rawset(info, "loadresult", loadresult);
+    }
+}
+
+
+void SourceDB::diff(const SourceDB &auth, StreamBuffer *sb) {
+    LuaVar sdb, sfn, sinfo, shash, sseq;
+    LuaVar mdb, mfn, minfo, mhash, mseq, mcode;
+    LuaStack SLS(lua_state_, sdb, sfn, sinfo, shash, sseq);
+    LuaStack MLS(auth.lua_state_, mdb, mfn, minfo, mhash, mseq, mcode);
+    sb->write_int32(0);
+    int wc_after = sb->total_writes();
+    int nupdates = 0;
+    // Fetch the two source databases.
+    SLS.rawget(sdb, LuaRegistry, "sourcedb");
+    MLS.rawget(mdb, LuaRegistry, "sourcedb");
+    // Loop over the master database.
+    MLS.set(mfn, LuaNil);
+    while (MLS.next(mdb, mfn, minfo) != 0) {
+        eng::string fn = MLS.ckstring(mfn);
+        assert(MLS.istable(minfo));
+        MLS.rawget(mseq, minfo, "sequence");
+        MLS.rawget(mhash, minfo, "hash");
+        bool diffhash = true;
+        bool diffseq = true;
+        SLS.set(sfn, fn);
+        SLS.rawget(sinfo, sdb, sfn);
+        if (SLS.istable(sinfo)) {
+            SLS.rawget(shash, sinfo, "hash");
+            SLS.rawget(sseq, sinfo, "sequence");
+            diffhash = MLS.ckstring(mhash) != SLS.ckstring(shash);
+            diffseq = MLS.ckinteger(mseq) != SLS.ckinteger(sseq);
+        }
+        if (diffhash || diffseq) {
+            sb->write_string(MLS.ckstring(mfn));
+            sb->write_int32(MLS.ckinteger(mseq));
+            if (diffhash) {
+                MLS.rawget(mcode, minfo, "code");
+                sb->write_string(MLS.ckstring(mcode));
+            } else {
+                sb->write_string("\001");
+            }
+            nupdates += 1;
+        }
+    }
+    // Loop over synch database.
+    SLS.set(sfn, LuaNil);
+    while (SLS.next(sdb, sfn, sinfo) != 0) {
+        eng::string fn = SLS.ckstring(sfn);
+        assert(SLS.istable(sinfo));
+        MLS.set(mfn, fn);
+        MLS.rawget(minfo, mdb, mfn);
+        if (!MLS.istable(minfo)) {
+            sb->write_string(SLS.ckstring(sfn));
+            sb->write_int32(-1);
+            sb->write_string("");
+            nupdates += 1;
+        }
+    }
+    sb->overwrite_int32(wc_after, nupdates);
+    MLS.result();
+    SLS.result();
+}
+
+bool SourceDB::patch(StreamBuffer *sb, DebugCollector *dbc) {
+    lua_State *L = lua_state_;
+    LuaVar db, info;
+    LuaStack LS(L, db, info);
+    LS.rawget(db, LuaRegistry, "sourcedb");
+    int nupdates = sb->read_int32();
+    for (int i = 0; i < nupdates; i++) {
+        eng::string fn = sb->read_string();
+        int sequence = sb->read_int32();
+        eng::string code = sb->read_string();
+        DebugLine(dbc) << "Source file " << fn << " updated";
+        if (sequence < 0) {
+            LS.rawset(db, fn, LuaNil);
+        } else {
+            get_info_table(LS, db, info, fn);
+            LS.rawset(info, "sequence", sequence);
+            if (code != "\001") {
+                LS.rawset(info, "code", code);
+                LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
+                calculate_loadresult(LS, info, fn, code);
+            }
+        }
+    }
+    LS.result();
+    return (nupdates > 0);
+}
+
+void SourceDB::set(const eng::string &fn, const eng::string &code, int sequence) {
+    lua_State *L = lua_state_;
+    LuaVar db, info;
+    LuaStack LS(L, db, info);
+    LS.rawget(db, LuaRegistry, "sourcedb");
+    get_info_table(LS, db, info, fn);
+    LS.rawset(info, "sequence", sequence);
+    LS.rawset(info, "code", code);
+    LS.rawset(info, "fingerprint", "");
+    LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
+    calculate_loadresult(LS, info, fn, code);
+    LS.result();
+}
+
+eng::string SourceDB::get(const eng::string &fn) {
+    lua_State *L = lua_state_;
+    LuaVar db, info, code, sequence, loadresult;
+    LuaStack LS(L, db, info, code, sequence, loadresult);
+    LS.rawget(db, LuaRegistry, "sourcedb");
+    LS.rawget(info, db, fn);
+    if (!LS.istable(info)) {
+        LS.result();
+        return "<nonexistent>";
+    }
+    LS.rawget(code, info, "code");
+    LS.rawget(sequence, info, "sequence");
+    LS.rawget(loadresult, info, "loadresult");
+    eng::string ccode = LS.ckstring(code);
+    int seqno = LS.ckint(sequence);
+    eng::string cloadresult;
+    if (LS.isfunction(loadresult)) {
+        cloadresult = "<function>";
+    } else {
+        cloadresult = LS.ckstring(loadresult);
+    }
+    eng::ostringstream oss;
+    oss << seqno << ":" << ccode << ":" << cloadresult;
+    LS.result();
+    return oss.str(); 
+}
+
+void SourceDB::update(const util::LuaSourceVec &source) {
+    lua_State *L = lua_state_;
+    LuaVar sourcedb, info;
+    LuaStack LS(L, sourcedb, info);
+
+    // Get and clear the source database.
+    LS.rawget(sourcedb, LuaRegistry, "sourcedb");
+    if (!LS.istable(sourcedb)) {
+        LS.newtable(sourcedb);
+        LS.rawset(LuaRegistry, "sourcedb", sourcedb);
+    }
+    LS.cleartable(sourcedb, true);
+
+    for (int i = 0; i < int(source.size()); i++) {
+        const eng::string &file = source[i].first;
+        const eng::string &code = source[i].second;
+        std::cerr << "Compiling " << file << std::endl;
+        LS.newtable(info);
+        LS.rawset(info, "name", file);
+        LS.rawset(info, "code", code);
+        LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
+        LS.rawset(info, "sequence", i + 1);
+        calculate_loadresult(LS, info, file, code);
+        LS.rawset(sourcedb, file, info);
+    }
+    LS.result();
+}
+
+// Delete everything from the global environment except
+// the class tables.
+//
+static void source_clear_globals(lua_State *L) {
+    LuaVar classname, classtab, key, globtab;
+    LuaStack LS(L, classname, classtab, key, globtab);
+
+    LS.getglobaltable(globtab);
+    LS.rawset(globtab, "_G", LuaNil);
+    LS.set(classname, LuaNil);
+    while (LS.next(globtab, classname, classtab) != 0) {
+        if (LS.rawequal(globtab, classtab)) {
+            LS.rawset(globtab, classname, LuaNil);
+        } else if (LS.istable(classtab)) {
+            LS.cleartable(classtab, true);
+        } else {
+            LS.rawset(globtab, classname, LuaNil);
+        }
+    }
+    LS.rawset(globtab, "_G", globtab);
+    LS.result();
+}
+
+
+// Load all the 'LuaDefine' C functions into the lua state.
+//
+static void source_load_cfunctions(lua_State *L) {
+    LuaVar classobj;
+    LuaStack LS(L, classobj);
+    for (auto r = LuaFunctionReg::All; r != nullptr; r=r->next()) {
+        lua_CFunction func = r->get_func();
+        if ((func != nullptr) && (!r->get_sandbox())) {
+            std::string_view classname;
+            std::string_view funcname;
+            get_reg_name(r->get_name(), classname, funcname);
+            if (classname.empty()) {
+                LS.getglobaltable(classobj);
+                LS.rawset(classobj, funcname, func);
+            } else {
+                LS.makeclass(classobj, classname);
+                LS.rawset(classobj, funcname, func);
+            }
+        }
+    }
+    LS.result();
+}
+
+// Load all the 'LuaConstant' constants into the lua state.
+//
+static void source_load_cconstants(lua_State *L) {
+    LuaVar classobj, value;
+    LuaStack LS(L, classobj, value);
+    for (auto r = LuaConstantReg::All; r != nullptr; r=r->next()) {
+        if (r->get_tokenvalue().empty()) {
+            LS.set(value, r->get_numbervalue());
+        } else {
+            LS.set(value, r->get_tokenvalue());
+        }
+        std::string_view classname;
+        std::string_view funcname;
+        get_reg_name(r->get_name(), classname, funcname);
+        if (classname.empty()) {
+            LS.getglobaltable(classobj);
+            LS.rawset(classobj, funcname, value);
+        } else {
+            LS.makeclass(classobj, classname);
+            LS.rawset(classobj, funcname, value);
+        }
+    }
+    LS.result();
+}
+
+// Run all the closures from the source database.
+//
+static eng::string source_load_lfunctions(lua_State *L) {
+    LuaVar sourcedb, key, info, seq, closure, err;
+    LuaStack LS(L, sourcedb, key, info, seq, closure, err);
+
+    // Get the source database.
+    LS.rawget(sourcedb, LuaRegistry, "sourcedb");
+    if (LS.type(sourcedb) != LUA_TTABLE) {
+        LS.newtable(sourcedb);
+        LS.rawset(LuaRegistry, "sourcedb", sourcedb);
+    }
+
+    // Sort the keys by sequence number.
+    eng::map<int, eng::string> indices;
+    LS.set(key, LuaNil);
+    while (LS.next(sourcedb, key, info) != 0) {
+        LS.rawget(seq, info, "sequence");
+        indices[LS.ckinteger(seq)] = LS.ckstring(key);
+    }
+
+    // Now call the closures in the proper order.
+    eng::ostringstream errss;
+    for (const auto &p : indices) {
+        LS.rawget(info, sourcedb, p.second);
+        LS.rawget(closure, info, "loadresult");
+
+        // If there's already an error in the sourcedb, collect it.
+        if (!LS.isfunction(closure)) {
+            errss << LS.ckstring(closure) << "\n";
+            continue;
+        }
+        
+        // Call the closure.  If there's an error, collect it.
+        lua_pushvalue(L, closure.index());
+        eng::string msg = traceback_pcall(L, 0, 0);
+        if (!msg.empty()) {
+            LS.set(err, msg);
+            errss << msg << std::endl;
+        }
+    }
+    LS.result();
+    return errss.str();
+}
+
+
+eng::string SourceDB::rebuild() {
+    lua_State *L = lua_state_;
+    LuaVar mathclass, httpclass, jsonnull;
+    LuaStack LS(L, mathclass, httpclass, jsonnull);
+    source_clear_globals(L);
+    source_load_cfunctions(L);
+    source_load_cconstants(L);    
+    eng::string errs = source_load_lfunctions(L);
+    LS.result();
+    return errs;
+}
+
+void SourceDB::run_unittests() {
+    lua_State *L = lua_state_;
+    LuaVar unittests, name, func, err, globtab;
+    LuaStack LS(L, unittests, name, func, err, globtab);
+
+    LS.getglobaltable(globtab);
+    LS.rawget(unittests, globtab, "unittests");
+
+    // Sort the unit test names.
+    eng::set<eng::string> names;
+    LS.set(name, LuaNil);
+    while (LS.next(unittests, name, func) != 0) {
+        if (LS.isfunction(func) && LS.isstring(name)) {
+            names.insert(LS.ckstring(name));
+        }
+    }
+
+    // Run the functions in order
+    bool any = false;
+    for (const eng::string &name : names) {
+        std::cerr << "Running unittests." << name << std::endl;
+        LS.rawget(func, unittests, name);
+
+        lua_pushvalue(L, func.index());
+        eng::string msg = traceback_pcall(L, 0, 0);
+        if (!msg.empty()) {
+            LS.set(err, msg);
+            std::cerr << msg << std::endl;
+            any = true;
+        }
+    }
+
+    if (any) {
+        exit(1);
+    }
+    LS.result();
+}
+
+void SourceDB::init(lua_State *L) {
+    lua_state_ = L;
+    LuaVar globtab, persist, unpersist, classname, classtab, funcname, funcp, rawfunc, nullstring;
+    LuaStack LS(L, globtab, persist, unpersist, classname, classtab, funcname, funcp, rawfunc, nullstring);
+    LS.getglobaltable(globtab);
+    LS.rawset(LuaRegistry, "sourcedb", LuaNewTable);
+
+    // Set the metatable for strings.
+    LS.makeclass(classtab, "string");
+    LS.set(nullstring, "");
+    LS.setmetatable(nullstring, classtab);
+
+    // Rebuild the global environment.
+    rebuild();
+
+    // We need to register all C functions with the eris permanents tables.
+    LS.rawget(persist, LuaRegistry, "persist");
+    LS.rawget(unpersist, LuaRegistry, "unpersist");
+    LS.set(classname, LuaNil);
+    while (LS.next(globtab, classname, classtab) != 0) {
+        if (LS.isstring(classname) && LS.istable(classtab)) {
+            LS.set(funcname, LuaNil);
+            while (LS.next(classtab, funcname, funcp) != 0) {
+                if (LS.isstring(funcname) && LS.iscfunction(funcp)) {
+                    eng::string full = "cfunc:";
+                    full += LS.ckstring(classname);
+                    full += ".";
+                    full += LS.ckstring(funcname);
+                    lua_pushcfunction(L, lua_tocfunction(L, funcp.index()));
+                    lua_replace(L, rawfunc.index());
+                    LS.rawset(persist, rawfunc, full);
+                    LS.rawset(unpersist, full, rawfunc);
+                }
+            }
+        }
+    }
+}
+
+void SourceDB::serialize_source(const util::LuaSourceVec &sv, StreamBuffer *sb) {
+    sb->write_int32(sv.size());
+    for (const auto &pair : sv) {
+        sb->write_string(pair.first);
+        sb->write_string(pair.second);
+    }
+}
+
+void SourceDB::deserialize_source(util::LuaSourceVec *sv, StreamBuffer *sb) {
+    sv->clear();
+    int count = sb->read_int32();
+    if ((count < 0) || (count > 10000)) throw StreamCorruption();
+    for (int i = 0; i < count; i++) {
+        eng::string fn = sb->read_string();
+        eng::string code = sb->read_string();
+        sv->emplace_back(fn, code);
+    }
+}
+
+// This function should not touch the dlmalloc heap.
+void SourceDB::register_lua_builtins() {
+    lua_State *L = LuaStack::newstate(nullptr);
+    luaL_openlibs(L);
+    LuaVar globals, lclassname, lfuncname, classtab, func;
+    LuaStack LS(L, globals, lclassname, lfuncname, classtab, func);
+    LS.getglobaltable(globals);
+
+    // Iterate over the function registry, copying function pointers from
+    // the prototype lua state into the registry, then remove the closure
+    // from the prototype.
+    for (auto reg = LuaFunctionReg::All; reg != nullptr; reg=reg->next()) {
+        std::string_view funcname;
+        std::string_view classname;
+        get_reg_name(reg->get_name(), classname, funcname);
+        if (classname.empty()) {
+            LS.getglobaltable(classtab);
+        } else {
+            LS.rawget(classtab, globals, classname);
+        }
+        lua_CFunction builtin = nullptr;
+        if (LS.istable(classtab)) {
+            LS.rawget(func, classtab, funcname);
+            if (LS.iscfunction(func)) {
+                builtin = lua_tocfunction(L, func.index());
+                LS.rawset(classtab, funcname, LuaNil);
+            }
+        }
+        if (reg->get_func() == nullptr) {
+            if (builtin == nullptr) {
+                if (!reg->get_sandbox()) {
+                    std::cerr << "No such builtin function: " << classname << " " << funcname << std::endl;
+                }
+            } else {
+                reg->set_func(builtin);
+            }
+        }
+    }
+
+    // Iterate over the prototype.  All cfunctions should have been removed.
+    LS.set(lclassname, LuaNil);
+    while (LS.next(globals, lclassname, classtab)) {
+        if (LS.isstring(lclassname)) {
+            if (LS.istable(classtab)) {
+                LS.set(lfuncname, LuaNil);
+                while (LS.next(classtab, lfuncname, func)) {
+                    if (LS.iscfunction(func)) {
+                        std::cerr << "Failed to declare builtin: " << LS.ckstring(lclassname) << "."
+                            << LS.ckstring(lfuncname) << std::endl;
+                    }
+                }
+            }
+        }
+    }
+
+    lua_close(L);
+}
+
+
+eng::string SourceDB::function_docs(const LuaStack &LS0, LuaSlot fn) {
+    lua_State *L = LS0.state();
+    LuaVar sourcedb, fname, finfo, code;
+    LuaStack LS(L, sourcedb, fname, finfo, code);
+
+    if (LS.iscfunction(fn)) {
+        lua_CFunction cfn = lua_tocfunction(L, fn.index());
+        const LuaFunctionReg *reg = LuaFunctionReg::lookup(cfn);
+        if (reg == nullptr) {
+            return "";
+        }
+        std::string_view classname;
+        std::string_view funcname;
+        get_reg_name(reg->get_name(), classname, funcname);
+        eng::ostringstream oss;
+        util::StringVec docs = util::split_docstring(reg->get_docs());
+        oss << "function ";
+        if (!classname.empty()) {
+            oss << classname << ".";
+        }
+        oss << funcname << "(" << reg->get_args() << ")" << std::endl;
+        oss << "--" << std::endl;
+        for (const eng::string &line : docs) {
+            oss << "-- " << line << std::endl;
+        }
+        oss << "--" << std::endl;
+        return oss.str();
+    } else if (LS.isfunction(fn)) {
+        lua_Debug ar;
+        lua_pushvalue(L, fn.index());
+        int status = lua_getinfo(L, ">S", &ar);
+        if (status == 0) return "";
+
+        // Get the source database.
+        LS.rawget(sourcedb, LuaRegistry, "sourcedb");
+        if (!LS.istable(sourcedb)) {
+            return "";
+        }
+
+        // Get the finfo table from the source db.
+        LS.set(fname, eng::string(ar.short_src));
+        LS.rawget(finfo, sourcedb, fname);
+        if (!LS.istable(finfo)) {
+            return "";
+        }
+
+        // Get the code from the finfo table.
+        LS.rawget(code, finfo, "code");
+        if (!LS.isstring(code)) {
+            return "";
+        }
+        
+        // Split the code into lines.
+        util::StringVec lines = util::split_lines(LS.ckstring(code));
+        int linehi = ar.linedefined - 1;
+        if ((linehi < 0) || (linehi >= int(lines.size()))) {
+            return "";
+        }
+
+        // Incorporate the function comment.
+        int linelo = linehi;
+        while ((linelo > 0) && (sv::is_lua_comment(lines[linelo-1]))) linelo -= 1;
+        
+        // Output the docs.
+        eng::ostringstream result;
+        result << lines[linehi] << std::endl;
+        result << "--" << std::endl;
+        for (int i = linelo; i < linehi; i++) {
+            result << lines[i] << std::endl;
+        }
+        result << "--" << std::endl;
+        return result.str();
+    } else {
+        return "";
+    }
+}
+
+// These should go away eventually.  They're for debugging.
+LuaDefine(coroutine_setnextid, "thread,id", "set the next id of a thread (debugging only)") {
+    LuaArg co, lid;
+    LuaStack LS(L, co, lid);
+    lua_State *CO = LS.ckthread(co);
+    lua_Number id = LS.ckinteger(lid);
+    lua_setnextid(CO, id);
+    return LS.result();
+}
+
+LuaDefine(coroutine_getnextid, "thread", "get the next id of a thread (debugging only)") {
+    LuaArg co;
+    LuaRet lid;
+    LuaStack LS(L, co, lid);
+    lua_State *CO = LS.ckthread(co);
+    LS.set(lid, lua_getnextid(CO));
+    return LS.result();
+}
+
+LuaDefine(unittests_sourcedb, "", "some unit tests") {
+    LuaSnap msnap;
+    LuaSnap ssnap;
+    SourceDB mdb;
+    SourceDB sdb;
+    mdb.init(msnap.state());
+    sdb.init(ssnap.state());
+    StreamBuffer sb;
+
+    // Create a code database using 'set'.
+    mdb.set("foo", "function foo() print('foo') end", 1);
+    mdb.set("bar", "function bar() print('bar') end", 2);
+    mdb.set("zoo", "funcjdshja mxooso yowza!", 3);
+    
+    // Verify that get works.
+    LuaAssertStrEq(L, mdb.get("foo"), "1:function foo() print('foo') end:<function>");
+    LuaAssertStrEq(L, mdb.get("bar"), "2:function bar() print('bar') end:<function>");
+    LuaAssertStrEq(L, mdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
+    LuaAssertStrEq(L, mdb.get("baz"), "<nonexistent>");
+
+    // Difference transmit.
+    sdb.diff(mdb, &sb);
+
+    // There should still be nothing in the sdb.
+    LuaAssertStrEq(L, sdb.get("foo"), "<nonexistent>");
+    LuaAssertStrEq(L, sdb.get("bar"), "<nonexistent>");
+    LuaAssertStrEq(L, sdb.get("zoo"), "<nonexistent>");
+    LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
+
+    // Apply the diffs.
+    sdb.patch(&sb, nullptr);
+
+    // Everything should now be copied to sdb.
+    LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:<function>");
+    LuaAssertStrEq(L, sdb.get("bar"), "2:function bar() print('bar') end:<function>");
+    LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
+    LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
+
+    // Make a single change to the code.
+    mdb.set("bar", "function bar1() print('bar1') end", 2);
+
+    // Diff and patch
+    sdb.diff(mdb, &sb);
+    sdb.patch(&sb, nullptr);
+
+    // Verify that it's been updated.
+    LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:<function>");
+    LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:<function>");
+    LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
+    LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
+
+    // Make a single change to just a sequence number.
+    mdb.set("foo", "function foo() print('foo') end", 6);
+
+    // Diff
+    sdb.diff(mdb, &sb);
+
+    // The only thing we've updated is a single sequence number.  Make
+    // sure the number of bytes transmitted is reasonable.
+    //    4 bytes for the count of changes
+    //    4 bytes for "foo"
+    //    4 bytes for the sequence number
+    //    2 bytes for unmodified code
+    // This verifies that the hash comparisons are working.
+    LuaAssert(L, sb.fill() == 14);
+
+    // Patch.
+    sdb.patch(&sb, nullptr);
+
+    // Verify that it's been updated.
+    LuaAssertStrEq(L, sdb.get("foo"), "6:function foo() print('foo') end:<function>");
+    LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:<function>");
+    LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
+    LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
+
+    return 0;
+}
+
+LuaDefineBuiltin(table_concat, "vector1, vector2", "concatenate two vectors");
+LuaDefineBuiltin(table_insert, "vector, pos, value", "insert an element into a vector");
+LuaDefineBuiltin(table_remove, "vector, pos", "remove an element from a vector");
+LuaDefineBuiltin(table_sort, "vector [,comparefn]", "sort a vector");
+LuaDefineBuiltin(table_pack, "v1, v2, v3...", "turn a sequence of arguments into a vector");
+LuaDefineBuiltin(table_unpack, "vector", "turn a vector into a sequence of return values");
+LuaSandboxBuiltin(table_maxn, "", "");
+
+LuaDefineBuiltin(string_byte, "str [,index]", "get a single byte from a string");
+LuaDefineBuiltin(string_char, "byte, byte,...", "convert sequence of bytes to a string");
+LuaDefineBuiltin(string_find, "str, pattern [,index]", "return start and end of pattern in a string");
+LuaDefineBuiltin(string_len, "str", "return the length of the string, in bytes");
+LuaDefineBuiltin(string_rep, "str, count", "repeat the string some number of times");
+LuaDefineBuiltin(string_reverse, "str", "reverse the bytes of the string");
+LuaDefineBuiltin(string_lower, "str", "convert string to lowercase");
+LuaDefineBuiltin(string_upper, "str", "convert string to uppercase");
+LuaDefineBuiltin(string_format, "formatstr, v1,v2,v3...", "generate formatted output string");
+LuaDefineBuiltin(string_gmatch, "str, pattern", "iterate over pattern-matched substrings");
+LuaDefineBuiltin(string_gsub, "str, pattern, replace", "global replace pattern in string");
+LuaDefineBuiltin(string_match, "str, pattern", "return start and end of pattern in string");
+LuaDefineBuiltin(string_sub, "str, pos1, pos2", "return substring of str from pos1 to pos2");
+LuaSandboxBuiltin(string_dump, "func", "convert a function to a string");
+
+LuaDefineBuiltin(bit32_arshift, "n, shift", "shift 32-bit number to the right, keeping high bit unchanged");
+LuaDefineBuiltin(bit32_band, "n, n, n...", "return the bitwise and of all 32-bit numbers");
+LuaDefineBuiltin(bit32_bnot, "n", "return the bitwise negation of n, ie, (-1 - n) % 2^32");
+LuaDefineBuiltin(bit32_bor, "n, n, n...", "return the bitwise or of all 32-bit arguments");
+LuaDefineBuiltin(bit32_bxor, "n, n, n...", "return the bitwise exclusive or of all 32-bit arguments");
+LuaDefineBuiltin(bit32_btest, "n, n, n...", "compute bitwise and of all 32-bit arguments, return true if nonzero");
+LuaDefineBuiltin(bit32_extract, "n, field, width", "return value from extracted bitfield of 32-bit number");
+LuaDefineBuiltin(bit32_lrotate, "n, shift", "rotate 32-bit number to the left");
+LuaDefineBuiltin(bit32_lshift, "n, shift", "shift 32-bit number to the left, padding with zeros");
+LuaDefineBuiltin(bit32_replace, "n, v, field, width", "change value of extracted bitfield in 32-bit number");
+LuaDefineBuiltin(bit32_rrotate, "n, shift", "rotate 32-bit number to the right");
+LuaDefineBuiltin(bit32_rshift, "n, shift", "shift 32-bit number to the right, padding with zeros");
+
+LuaDefineBuiltin(math_abs, "x", "return absolute value of x");
+LuaDefineBuiltin(math_acos, "x", "return arc-cosine of x in radians");
+LuaDefineBuiltin(math_asin, "x", "return arc-sine of x in radians");
+LuaDefineBuiltin(math_atan, "x", "return the arc-tangent of x in radians");
+LuaDefineBuiltin(math_atan2, "y, x", "return arc-tangent of y/x, using signs to compute quadrant");
+LuaDefineBuiltin(math_ceil, "x", "return the smallest integer larger than or equal to x");
+LuaDefineBuiltin(math_cos, "x", "return the cosine of x in radians");
+LuaDefineBuiltin(math_cosh, "x", "return the hyperbolic cosine of x in radians");
+LuaDefineBuiltin(math_deg, "rad", "convert radians to degrees");
+LuaDefineBuiltin(math_exp, "x", "returns the value e^x");
+LuaDefineBuiltin(math_floor, "x", "returns the smallest integer less than or equal to x");
+LuaDefineBuiltin(math_fmod, "x, y", "return the remainder of x/y that rounds the quotient towards zero");
+LuaDefineBuiltin(math_frexp, "x", "given x, returns mantissa and exponent");
+LuaDefineBuiltin(math_ldexp, "x", "return unnormalized mantissa and exponent");
+LuaDefineBuiltin(math_log, "x [, base]", "return the log of x in base, default base is e");
+LuaDefineBuiltin(math_max, "x, x, x...", "return the largest argument");
+LuaDefineBuiltin(math_min, "x, x, x...", "return the smallest argument");
+LuaDefineBuiltin(math_modf, "x", "returns the integral and fractional part of x");
+LuaDefineBuiltin(math_pow, "x, y", "returns x ^ y, equivalent to the operator");
+LuaDefineBuiltin(math_rad, "deg", "convert degrees to radians");
+LuaDefineBuiltin(math_sin, "x", "return the sine of x in radians");
+LuaDefineBuiltin(math_sinh, "x", "return the hyperbolic sine of x in radians");
+LuaDefineBuiltin(math_sqrt, "x", "return the square root of x");
+LuaDefineBuiltin(math_tan, "x", "return the tangent of x in radians");
+LuaDefineBuiltin(math_tanh, "x", "return the hyperbolic tangent of x in radians");
+LuaSandboxBuiltin(math_log10, "", "");
+LuaNumberConstant(math_pi, M_PI, "");
+LuaNumberConstant(math_huge, HUGE_VAL, "");
+LuaNumberConstant(math_nan, NAN, "");
+LuaNumberConstant(math_maxint, LuaStack::MAXINT, "");
+
+// math.random and math.randomseed are in world-accessor.cpp, because
+// generating random numbers must manipulate global state which is
+// stored in the world model.
+
+LuaDefineBuiltin(assert, "flag [,message]", "assert that flag is true, if not, raise error");
+LuaDefineBuiltin(error, "message", "raise an error");
+LuaDefineBuiltin(getmetatable, "table", "get the metatable of a table");
+LuaDefineBuiltin(ipairs, "vector", "meant to be used in loops, iterates over a vector");
+LuaDefineBuiltin(next, "table, k", "returns the key after k and the corresponding value");
+LuaDefineBuiltin(pairs, "table", "meant to be used in loops, iterates over a table");
+LuaDefineBuiltin(rawpairs, "table", "meant to be used in loops, iterates over a table");
+LuaDefineBuiltin(pcall, "function, arg1, arg2, ...", "call the function, catching any errors");
+LuaDefineBuiltin(rawequal, "val1, val2", "return true if the two values are the same object");
+LuaDefineBuiltin(rawlen, "obj", "return the length of a vector or string");
+LuaDefineBuiltin(rawget, "table, key", "get a table entry, ignoring metamethods");
+LuaDefineBuiltin(rawset, "table, key, value", "set a table entry, ignoring metamethods");
+LuaDefineBuiltin(select, "n, arg1, arg2, ...", "return the nth argument");
+LuaDefineBuiltin(setmetatable, "table, meta", "set the metatable of the specified table");
+LuaDefineBuiltin(tonumber, "str", "convert a string to a number");
+LuaDefineBuiltin(type, "obj", "return the type of obj as a string");
+// print is redefined in world.cpp (because it prints into the world model)
+// tostring is redefined in pprint.cpp
+
+LuaSandboxBuiltin(collectgarbage, "", "");
+LuaSandboxBuiltin(dofile, "", "");
+LuaSandboxBuiltin(xpcall, "", "");
+LuaSandboxBuiltin(loadfile, "", "");
+LuaSandboxBuiltin(load, "", "");
+LuaSandboxBuiltin(require, "", "");
+LuaSandboxBuiltin(module, "", "");
+LuaSandboxBuiltin(loadstring, "", "");
+LuaSandboxBuiltin(unpack, "", "");
+
+
+LuaSandboxBuiltin(debug_debug, "", "");
+LuaSandboxBuiltin(debug_getuservalue, "", "");
+LuaSandboxBuiltin(debug_gethook, "", "");
+LuaSandboxBuiltin(debug_getinfo, "", "");
+LuaSandboxBuiltin(debug_getlocal, "", "");
+LuaSandboxBuiltin(debug_getregistry, "", "");
+LuaSandboxBuiltin(debug_getmetatable, "", "");
+LuaSandboxBuiltin(debug_getupvalue, "", "");
+LuaSandboxBuiltin(debug_upvaluejoin, "", "");
+LuaSandboxBuiltin(debug_upvalueid, "", "");
+LuaSandboxBuiltin(debug_setuservalue, "", "");
+LuaSandboxBuiltin(debug_sethook, "", "");
+LuaSandboxBuiltin(debug_setlocal, "", "");
+LuaSandboxBuiltin(debug_setmetatable, "", "");
+LuaSandboxBuiltin(debug_setupvalue, "", "");
+LuaSandboxBuiltin(debug_traceback, "", "");
+
+LuaSandboxBuiltin(eris_persist, "", "");
+LuaSandboxBuiltin(eris_unpersist, "", "");
+LuaSandboxBuiltin(eris_settings, "", "");
+
+LuaSandboxBuiltin(package_loadlib, "", "");
+LuaSandboxBuiltin(package_searchpath, "", "");
+LuaSandboxBuiltin(package_seeall, "", "");
+
+LuaSandboxBuiltin(coroutine_create, "", "");
+LuaSandboxBuiltin(coroutine_resume, "", "");
+LuaSandboxBuiltin(coroutine_running, "", "");
+LuaSandboxBuiltin(coroutine_status, "", "");
+LuaSandboxBuiltin(coroutine_wrap, "", "");
+LuaSandboxBuiltin(coroutine_yield, "", "");
+
+LuaSandboxBuiltin(io_close, "", "");
+LuaSandboxBuiltin(io_flush, "", "");
+LuaSandboxBuiltin(io_input, "", "");
+LuaSandboxBuiltin(io_lines, "", "");
+LuaSandboxBuiltin(io_open, "", "");
+LuaSandboxBuiltin(io_output, "", "");
+LuaSandboxBuiltin(io_popen, "", "");
+LuaSandboxBuiltin(io_read, "", "");
+LuaSandboxBuiltin(io_tmpfile, "", "");
+LuaSandboxBuiltin(io_type, "", "");
+LuaSandboxBuiltin(io_write, "", "");
+
+LuaSandboxBuiltin(os_clock, "", "");
+LuaSandboxBuiltin(os_date, "", "");
+LuaSandboxBuiltin(os_difftime, "", "");
+LuaSandboxBuiltin(os_execute, "", "");
+LuaSandboxBuiltin(os_exit, "", "");
+LuaSandboxBuiltin(os_getenv, "", "");
+LuaSandboxBuiltin(os_remove, "", "");
+LuaSandboxBuiltin(os_rename, "", "");
+LuaSandboxBuiltin(os_setlocale, "", "");
+LuaSandboxBuiltin(os_time, "", "");
+LuaSandboxBuiltin(os_tmpname, "", "");
+

luprex/cpp/core/source.hpp

@@ -0,0 +1,193 @@
+////////////////////////////////////////////////////////////
+//
+// SOURCEDB
+//
+// This module manages the loading of lua source files into the Lua environment.
+// Since the source files can be reloaded over and over, this module doesn't
+// just load the source files.  Instead, it does "source rebuilds" in which it
+// purges everything from the global environment, then it reinstalls everything
+// that should be there.  That way, if you delete something from a lua source
+// file, it gets removed from the lua global environment.
+//
+// THE MAKECLASS OPERATOR
+//
+// This module provides a new lua 'builtin' operator: "makeclass". This creates
+// a table and stores it in the global environment. The new table is meant to be
+// used as a class.  Three fields are initially created:
+//
+//    __class --> the name of the class as a string
+//
+//    __index --> points back to the class.  Makes it convenient to use the
+//                class as a metatable.
+//
+//    action  --> a subtable of additional methods.
+//
+// If you invoke 'makeclass' on a class that already exists, the existing table
+// is "repaired" - the __class and __index fields are restored and the action
+// subtable, if not present, is recreated.  If there are already functions or
+// constants inside the class, they are not affected.
+//
+//
+// THE LUA SOURCE DATABASE
+//
+// Class SourceDB only contains a single pointer to the lua environment. That's
+// because all the data for SourceDB is stored in the lua registry.
+// Specifically, the registry contains these keys:
+//
+//     1. The source database proper (registry key "sourcedb")
+//     2. The snapshot of builtins (registry key "source_snapshot_builtins")
+//
+// The source database proper is a table where the keys are filenames, and the
+// values are records containing information about that file:
+//
+//     "foo.lua" : {
+//         "name":        "foo.lua",
+//         "fingerprint": "12893129385854",
+//         "code":        "function xyz ...",
+//         "loadresult":  <function-23>,
+//         "sequence":    13
+//     }
+//
+// Let's break that down a little.  The "name" field is just the filename.  The
+// "fingerprint" is an encoding of the file modification date and the file
+// length, it is used to detect whether the file has been altered on disk
+// without having to reread the file.  The "code" is the entire content of the
+// source file.  The "loadresult" is the closure that results from calling the
+// lua 'load' function on the code.  If load fails, then the "loadresult" is an
+// error message. Finally, "sequence" indicates the order in which the source
+// files are meant to be loaded.
+//
+// The operation SourceDB::update refreshes the source database from disk.  It
+// doesn't reread files whose fingerprints have not changed. In a synchronous
+// model, we don't call SourceDB::update - instead, we update the source
+// database by difference transmission.
+//
+// Note that updating the source database has *no effect* on the lua global
+// variables (or lua function definitions).  The SourceDB::update operation
+// calls lua's "load" on the code, but all that does is return a loaded closure.
+// Nothing happens to the lua invironment until you *invoke* the loaded closure.
+// That doesn't happen until you do a SourceDB::rebuild operation, described
+// below.
+//
+//
+// SOURCE REBUILDS, IN DEPTH
+//
+// The function SourceDB::rebuild clears and then rebuilds the entire contents
+// of the lua global environment.  The reason to clear the environment is that
+// if we didn't clear it, then removing a function from the lua source would not
+// remove it from the lua environment.  Rebuilding the lua environment is a
+// multi-step process:
+//
+// * Delete everything from the global environment except class tables.
+//
+//     - Class tables are kept, but the contents are cleared.
+//     - If a class has an "actions" subtable, that is kept and cleared.
+//     - Anything else is deleted.
+//
+// * Lua Builtin functions are reinstalled in the global environment.
+//
+//     - To make this possible, the snapshot of builtins is used.
+//
+// * C++ functions registered with "LuaDefine" are reinstalled.
+//
+//     - LuaDefine creates a registry, that registry is iterated over.
+//
+// * Lua code is reinstalled by running the closures in the source DB.
+//
+//     - Simply call the "loadresult" closure to reinstall functions into the
+//       global environment.
+//
+// Note that if you've stored any global data in the lua environment, it's gone.
+// So therefore, we have to provide a separate "safe" space for global data
+// structures.  That is provided elsewhere, in the module "globaldb".
+//
+//
+// UNITTESTS
+//
+// We reserve the lua class name 'unittests' for storing unit tests.  Any
+// function placed into this class is considered a unit test.  We don't separate
+// unit tests from the rest of the code - they're compiled right into the main
+// binary.
+//
+// Unit tests can be either lua functions, or Lua-registered C functions. Unit
+// tests are executed by calling 'source_run_unittests'.
+//
+// Each unit test is run in a protected 'pcall' environment.  Any errors are
+// printed out to console. (At least for now).
+//
+////////////////////////////////////////////////////////////
+
+#ifndef SOURCE_HPP
+#define SOURCE_HPP
+
+#include "wrap-string.hpp"
+
+#include "util.hpp"
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+#include "debugcollector.hpp"
+
+class SourceDB : public eng::nevernew  {
+private:
+    lua_State *lua_state_;
+
+public:
+    void init(lua_State *L);
+    
+    // Update
+    //
+    // Update the database using the specified lua source code.
+    // Compiles these files using lua's "load" function.
+    //
+    void update(const util::LuaSourceVec &source);
+
+    // Rebuild
+    //
+    // Rebuild the lua environment: clear it out, then reinstall all the
+    // functions that should be there.  See above for more information.
+    //
+    // Any error messages will be collected into a single long string
+    // containing one error per line, and returned.  If the return value
+    // is the empty string, there were no errors.
+    //
+    eng::string rebuild();
+    
+    // Difference transmission.
+    //
+    // Note: The patch routine applies the differences to the source
+    // database, and if there are any changes, it does a source rebuild.
+    // The patch routine returns true if anything was modified.
+    //
+    void diff(const SourceDB &auth, StreamBuffer *sb);
+    bool patch(StreamBuffer *sb, DebugCollector *dbc);
+
+    // run_unittests
+    //
+    // Run all the lua unit tests.  Print any errors to console.  If there
+    // are any errors, exits the program.
+    //
+    void run_unittests();
+
+    // Get/Set code (for unit testing).
+    //
+    // These functions are direct getters/setters for values in the source
+    // database.  They are intended only for unit testing.
+    //
+    void set(const eng::string &fn, const eng::string &code, int sequence);
+    eng::string get(const eng::string &fn);
+
+    // Add builtins to the global function registry.
+    static void register_lua_builtins();
+    
+    // Get function documentation.
+    static eng::string function_docs(const LuaStack &LS, LuaSlot slot);
+
+    // Serialize and unserialize a source vector.
+    //
+    static void serialize_source(const util::LuaSourceVec &sv, StreamBuffer *sb);
+    static void deserialize_source(util::LuaSourceVec *sv, StreamBuffer *sb);
+};
+
+#endif // SOURCE_HPP
+
+

luprex/cpp/core/spookyv2.cpp

@@ -0,0 +1,360 @@
+// Spooky Hash
+// A 128-bit noncryptographic hash, for checksums and table lookup
+// By Bob Jenkins.  Public domain.
+//   Oct 31 2010: published framework, disclaimer ShortHash isn't right
+//   Nov 7 2010: disabled ShortHash
+//   Oct 31 2011: replace End, ShortMix, ShortEnd, enable ShortHash again
+//   April 10 2012: buffer overflow on platforms without unaligned reads
+//   July 12 2012: was passing out variables in final to in/out in short
+//   July 30 2012: I reintroduced the buffer overflow
+//   August 5 2012: SpookyV2: d = should be d += in short hash, and remove extra mix from long hash
+
+#include <memory.h>
+#include "spookyv2.hpp"
+
+#define ALLOW_UNALIGNED_READS 1
+
+#define INLINE inline
+
+// number of uint64_t's in internal state
+static const size_t sc_numVars = 12;
+
+// size of the internal state
+static const size_t sc_blockSize = sc_numVars*8;
+
+// size of buffer of unhashed data, in bytes
+static const size_t sc_bufSize = 2*sc_blockSize;
+
+//
+// sc_const: a constant which:
+//  * is not zero
+//  * is odd
+//  * is a not-very-regular mix of 1's and 0's
+//  * does not need any other special mathematical properties
+//
+static const uint64_t sc_const = 0xdeadbeefdeadbeefLL;
+
+//
+// left rotate a 64-bit value by k bytes
+//
+static INLINE uint64_t Rot64(uint64_t x, int k)
+{
+    return (x << k) | (x >> (64 - k));
+}
+
+//
+// This is used if the input is 96 bytes long or longer.
+//
+// The internal state is fully overwritten every 96 bytes.
+// Every input bit appears to cause at least 128 bits of entropy
+// before 96 other bytes are combined, when run forward or backward
+//   For every input bit,
+//   Two inputs differing in just that input bit
+//   Where "differ" means xor or subtraction
+//   And the base value is random
+//   When run forward or backwards one Mix
+// I tried 3 pairs of each; they all differed by at least 212 bits.
+//
+static INLINE void Mix(
+    const uint64_t *data, 
+    uint64_t &s0, uint64_t &s1, uint64_t &s2, uint64_t &s3,
+    uint64_t &s4, uint64_t &s5, uint64_t &s6, uint64_t &s7,
+    uint64_t &s8, uint64_t &s9, uint64_t &s10,uint64_t &s11)
+{
+    s0 += data[0];    s2 ^= s10;    s11 ^= s0;    s0 = Rot64(s0,11);    s11 += s1;
+    s1 += data[1];    s3 ^= s11;    s0 ^= s1;    s1 = Rot64(s1,32);    s0 += s2;
+    s2 += data[2];    s4 ^= s0;    s1 ^= s2;    s2 = Rot64(s2,43);    s1 += s3;
+    s3 += data[3];    s5 ^= s1;    s2 ^= s3;    s3 = Rot64(s3,31);    s2 += s4;
+    s4 += data[4];    s6 ^= s2;    s3 ^= s4;    s4 = Rot64(s4,17);    s3 += s5;
+    s5 += data[5];    s7 ^= s3;    s4 ^= s5;    s5 = Rot64(s5,28);    s4 += s6;
+    s6 += data[6];    s8 ^= s4;    s5 ^= s6;    s6 = Rot64(s6,39);    s5 += s7;
+    s7 += data[7];    s9 ^= s5;    s6 ^= s7;    s7 = Rot64(s7,57);    s6 += s8;
+    s8 += data[8];    s10 ^= s6;    s7 ^= s8;    s8 = Rot64(s8,55);    s7 += s9;
+    s9 += data[9];    s11 ^= s7;    s8 ^= s9;    s9 = Rot64(s9,54);    s8 += s10;
+    s10 += data[10];    s0 ^= s8;    s9 ^= s10;    s10 = Rot64(s10,22);    s9 += s11;
+    s11 += data[11];    s1 ^= s9;    s10 ^= s11;    s11 = Rot64(s11,46);    s10 += s0;
+}
+
+//
+// Mix all 12 inputs together so that h0, h1 are a hash of them all.
+//
+// For two inputs differing in just the input bits
+// Where "differ" means xor or subtraction
+// And the base value is random, or a counting value starting at that bit
+// The final result will have each bit of h0, h1 flip
+// For every input bit,
+// with probability 50 +- .3%
+// For every pair of input bits,
+// with probability 50 +- 3%
+//
+// This does not rely on the last Mix() call having already mixed some.
+// Two iterations was almost good enough for a 64-bit result, but a
+// 128-bit result is reported, so End() does three iterations.
+//
+static INLINE void EndPartial(
+    uint64_t &h0, uint64_t &h1, uint64_t &h2, uint64_t &h3,
+    uint64_t &h4, uint64_t &h5, uint64_t &h6, uint64_t &h7, 
+    uint64_t &h8, uint64_t &h9, uint64_t &h10,uint64_t &h11)
+{
+    h11+= h1;    h2 ^= h11;   h1 = Rot64(h1,44);
+    h0 += h2;    h3 ^= h0;    h2 = Rot64(h2,15);
+    h1 += h3;    h4 ^= h1;    h3 = Rot64(h3,34);
+    h2 += h4;    h5 ^= h2;    h4 = Rot64(h4,21);
+    h3 += h5;    h6 ^= h3;    h5 = Rot64(h5,38);
+    h4 += h6;    h7 ^= h4;    h6 = Rot64(h6,33);
+    h5 += h7;    h8 ^= h5;    h7 = Rot64(h7,10);
+    h6 += h8;    h9 ^= h6;    h8 = Rot64(h8,13);
+    h7 += h9;    h10^= h7;    h9 = Rot64(h9,38);
+    h8 += h10;   h11^= h8;    h10= Rot64(h10,53);
+    h9 += h11;   h0 ^= h9;    h11= Rot64(h11,42);
+    h10+= h0;    h1 ^= h10;   h0 = Rot64(h0,54);
+}
+
+static INLINE void End(
+    const uint64_t *data, 
+    uint64_t &h0, uint64_t &h1, uint64_t &h2, uint64_t &h3,
+    uint64_t &h4, uint64_t &h5, uint64_t &h6, uint64_t &h7, 
+    uint64_t &h8, uint64_t &h9, uint64_t &h10,uint64_t &h11)
+{
+    h0 += data[0];   h1 += data[1];   h2 += data[2];   h3 += data[3];
+    h4 += data[4];   h5 += data[5];   h6 += data[6];   h7 += data[7];
+    h8 += data[8];   h9 += data[9];   h10 += data[10]; h11 += data[11];
+    EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
+    EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
+    EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
+}
+
+//
+// The goal is for each bit of the input to expand into 128 bits of 
+//   apparent entropy before it is fully overwritten.
+// n trials both set and cleared at least m bits of h0 h1 h2 h3
+//   n: 2   m: 29
+//   n: 3   m: 46
+//   n: 4   m: 57
+//   n: 5   m: 107
+//   n: 6   m: 146
+//   n: 7   m: 152
+// when run forwards or backwards
+// for all 1-bit and 2-bit diffs
+// with diffs defined by either xor or subtraction
+// with a base of all zeros plus a counter, or plus another bit, or random
+//
+static INLINE void ShortMix(uint64_t &h0, uint64_t &h1, uint64_t &h2, uint64_t &h3)
+{
+    h2 = Rot64(h2,50);  h2 += h3;  h0 ^= h2;
+    h3 = Rot64(h3,52);  h3 += h0;  h1 ^= h3;
+    h0 = Rot64(h0,30);  h0 += h1;  h2 ^= h0;
+    h1 = Rot64(h1,41);  h1 += h2;  h3 ^= h1;
+    h2 = Rot64(h2,54);  h2 += h3;  h0 ^= h2;
+    h3 = Rot64(h3,48);  h3 += h0;  h1 ^= h3;
+    h0 = Rot64(h0,38);  h0 += h1;  h2 ^= h0;
+    h1 = Rot64(h1,37);  h1 += h2;  h3 ^= h1;
+    h2 = Rot64(h2,62);  h2 += h3;  h0 ^= h2;
+    h3 = Rot64(h3,34);  h3 += h0;  h1 ^= h3;
+    h0 = Rot64(h0,5);   h0 += h1;  h2 ^= h0;
+    h1 = Rot64(h1,36);  h1 += h2;  h3 ^= h1;
+}
+
+//
+// Mix all 4 inputs together so that h0, h1 are a hash of them all.
+//
+// For two inputs differing in just the input bits
+// Where "differ" means xor or subtraction
+// And the base value is random, or a counting value starting at that bit
+// The final result will have each bit of h0, h1 flip
+// For every input bit,
+// with probability 50 +- .3% (it is probably better than that)
+// For every pair of input bits,
+// with probability 50 +- .75% (the worst case is approximately that)
+//
+static INLINE void ShortEnd(uint64_t &h0, uint64_t &h1, uint64_t &h2, uint64_t &h3)
+{
+    h3 ^= h2;  h2 = Rot64(h2,15);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,52);  h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,26);  h1 += h0;
+    h2 ^= h1;  h1 = Rot64(h1,51);  h2 += h1;
+    h3 ^= h2;  h2 = Rot64(h2,28);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,9);   h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,47);  h1 += h0;
+    h2 ^= h1;  h1 = Rot64(h1,54);  h2 += h1;
+    h3 ^= h2;  h2 = Rot64(h2,32);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,25);  h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,63);  h1 += h0;
+}
+
+//
+// short hash ... it could be used on any message, 
+// but it's used by Spooky just for short messages.
+//
+static void Short(
+    const void *message,
+    size_t length,
+    uint64_t *hash1,
+    uint64_t *hash2)
+{
+    uint64_t buf[2*sc_numVars];
+    union 
+    { 
+        const uint8_t *p8; 
+        uint32_t *p32;
+        uint64_t *p64; 
+        size_t i; 
+    } u;
+
+    u.p8 = (const uint8_t *)message;
+    
+    if (!ALLOW_UNALIGNED_READS && (u.i & 0x7))
+    {
+        memcpy(buf, message, length);
+        u.p64 = buf;
+    }
+
+    size_t remainder = length%32;
+    uint64_t a=*hash1;
+    uint64_t b=*hash2;
+    uint64_t c=sc_const;
+    uint64_t d=sc_const;
+
+    if (length > 15)
+    {
+        const uint64_t *end = u.p64 + (length/32)*4;
+        
+        // handle all complete sets of 32 bytes
+        for (; u.p64 < end; u.p64 += 4)
+        {
+            c += u.p64[0];
+            d += u.p64[1];
+            ShortMix(a,b,c,d);
+            a += u.p64[2];
+            b += u.p64[3];
+        }
+        
+        //Handle the case of 16+ remaining bytes.
+        if (remainder >= 16)
+        {
+            c += u.p64[0];
+            d += u.p64[1];
+            ShortMix(a,b,c,d);
+            u.p64 += 2;
+            remainder -= 16;
+        }
+    }
+    
+    // Handle the last 0..15 bytes, and its length
+    d += ((uint64_t)length) << 56;
+    switch (remainder)
+    {
+    case 15:
+    d += ((uint64_t)u.p8[14]) << 48;
+    case 14:
+        d += ((uint64_t)u.p8[13]) << 40;
+    case 13:
+        d += ((uint64_t)u.p8[12]) << 32;
+    case 12:
+        d += u.p32[2];
+        c += u.p64[0];
+        break;
+    case 11:
+        d += ((uint64_t)u.p8[10]) << 16;
+    case 10:
+        d += ((uint64_t)u.p8[9]) << 8;
+    case 9:
+        d += (uint64_t)u.p8[8];
+    case 8:
+        c += u.p64[0];
+        break;
+    case 7:
+        c += ((uint64_t)u.p8[6]) << 48;
+    case 6:
+        c += ((uint64_t)u.p8[5]) << 40;
+    case 5:
+        c += ((uint64_t)u.p8[4]) << 32;
+    case 4:
+        c += u.p32[0];
+        break;
+    case 3:
+        c += ((uint64_t)u.p8[2]) << 16;
+    case 2:
+        c += ((uint64_t)u.p8[1]) << 8;
+    case 1:
+        c += (uint64_t)u.p8[0];
+        break;
+    case 0:
+        c += sc_const;
+        d += sc_const;
+    }
+    ShortEnd(a,b,c,d);
+    *hash1 = a;
+    *hash2 = b;
+}
+
+
+
+
+// do the whole hash in one call
+void SpookyHash::ChainHash128(
+    const void *message, 
+    size_t length, 
+    uint64_t *hash1, 
+    uint64_t *hash2)
+{
+    if ((*hash1 == 0) && (*hash2 == 0)) {
+        *hash1 = 0x9438478934792837;
+        *hash2 = 0x8347848738748378;
+    }
+    
+    if (length < sc_bufSize)
+    {
+        Short(message, length, hash1, hash2);
+        return;
+    }
+
+    uint64_t h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11;
+    uint64_t buf[sc_numVars];
+    uint64_t *end;
+    union 
+    { 
+        const uint8_t *p8; 
+        uint64_t *p64; 
+        size_t i; 
+    } u;
+    size_t remainder;
+    
+    h0=h3=h6=h9  = *hash1;
+    h1=h4=h7=h10 = *hash2;
+    h2=h5=h8=h11 = sc_const;
+    
+    u.p8 = (const uint8_t *)message;
+    end = u.p64 + (length/sc_blockSize)*sc_numVars;
+
+    // handle all whole sc_blockSize blocks of bytes
+    if (ALLOW_UNALIGNED_READS || ((u.i & 0x7) == 0))
+    {
+        while (u.p64 < end)
+        { 
+            Mix(u.p64, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
+	    u.p64 += sc_numVars;
+        }
+    }
+    else
+    {
+        while (u.p64 < end)
+        {
+            memcpy(buf, u.p64, sc_blockSize);
+            Mix(buf, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
+	    u.p64 += sc_numVars;
+        }
+    }
+
+    // handle the last partial block of sc_blockSize bytes
+    remainder = (length - ((const uint8_t *)end-(const uint8_t *)message));
+    memcpy(buf, end, remainder);
+    memset(((uint8_t *)buf)+remainder, 0, sc_blockSize-remainder);
+    ((uint8_t *)buf)[sc_blockSize-1] = remainder;
+    
+    // do some final mixing 
+    End(buf, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
+    *hash1 = h0;
+    *hash2 = h1;
+}
+

luprex/cpp/core/spookyv2.hpp

@@ -0,0 +1,83 @@
+//
+// SpookyHash: a 128-bit noncryptographic hash function
+// By Bob Jenkins, public domain
+//   Oct 31 2010: alpha, framework + SpookyHash::Mix appears right
+//   Oct 31 2011: alpha again, Mix only good to 2^^69 but rest appears right
+//   Dec 31 2011: beta, improved Mix, tested it for 2-bit deltas
+//   Feb  2 2012: production, same bits as beta
+//   Feb  5 2012: adjusted definitions of uint* to be more portable
+//   Mar 30 2012: 3 bytes/cycle, not 4.  Alpha was 4 but wasn't thorough enough.
+//   August 5 2012: SpookyV2 (different results)
+// 
+// Up to 3 bytes/cycle for long messages.  Reasonably fast for short messages.
+// All 1 or 2 bit deltas achieve avalanche within 1% bias per output bit.
+//
+// This was developed for and tested on 64-bit x86-compatible processors.
+// It assumes the processor is little-endian.  There is a macro
+// controlling whether unaligned reads are allowed (by default they are).
+// This should be an equally good hash on big-endian machines, but it will
+// compute different results on them than on little-endian machines.
+//
+// Google's CityHash has similar specs to SpookyHash, and CityHash is faster
+// on new Intel boxes.  MD4 and MD5 also have similar specs, but they are orders
+// of magnitude slower.  CRCs are two or more times slower, but unlike 
+// SpookyHash, they have nice math for combining the CRCs of pieces to form 
+// the CRCs of wholes.  There are also cryptographic hashes, but those are even 
+// slower than MD5.
+//
+
+#ifndef SPOOKYV2_HPP
+#define SPOOKYV2_HPP
+
+#include <cstddef>
+#include <cstdint>
+#include <string_view>
+#include <utility>
+
+class SpookyHash
+{
+public:
+    // A hash is two uint64's.
+    //
+    using HashValue = std::pair<uint64_t, uint64_t>;
+
+    //
+    // SpookyHash: hash a single message in one call, produce 128-bit output
+    //
+    static void ChainHash128(
+        const void *message,  // message to hash
+        size_t length,        // length of message in bytes
+        uint64_t *hash1,      // input seed0, output hash0
+        uint64_t *hash2);     // input seed1, output hash1
+
+
+    static inline HashValue QkHash128(const void *message, size_t len) {
+        uint64_t hash1 = 0;
+        uint64_t hash2 = 0;
+        ChainHash128(message, len, &hash1, &hash2);
+        return std::make_pair(hash1, hash2);
+    }
+
+    static inline HashValue QkHash128(std::string_view v) {
+        uint64_t hash1 = 0;
+        uint64_t hash2 = 0;
+        ChainHash128(v.data(), v.size(), &hash1, &hash2);
+        return std::make_pair(hash1, hash2);
+    }
+
+    static inline uint64_t QkHash64(const void *message, size_t len) {
+        uint64_t hash1 = 0;
+        uint64_t hash2 = 0;
+        ChainHash128(message, len, &hash1, &hash2);
+        return hash1;
+    }
+
+    static inline uint64_t QkHash64(std::string_view v) {
+        uint64_t hash1 = 0;
+        uint64_t hash2 = 0;
+        ChainHash128(v.data(), v.size(), &hash1, &hash2);
+        return hash1;
+    }
+};
+
+#endif // SPOOKYV2_HPP

luprex/cpp/core/streambuffer.cpp

@@ -0,0 +1,685 @@
+#include "wrap-string.hpp"
+
+#include "eng-malloc.hpp"
+#include "streambuffer.hpp"
+#include "spookyv2.hpp"
+
+#include <cassert>
+#include <cstring>
+
+void StreamBuffer::init(bool fixed, bool owned, char *buf, int64_t size) {
+    buf_lo_ = buf;
+    buf_hi_ = buf_lo_ + size;
+    read_cursor_ = buf_lo_;
+    write_cursor_ = buf_lo_;
+    pre_read_count_ = 0;
+    owned_ = owned;
+    fixed_size_ = fixed;
+    lua_reader_data_ = 0;
+    lua_reader_size_ = 0;
+}
+
+StreamBuffer::StreamBuffer() {
+    init(false, true, 0, 0);
+}
+
+StreamBuffer::StreamBuffer(int64_t size, bool fixed) {
+    assert(size >= 0);
+    init(fixed, true, (char*)eng::malloc(size), size);
+}
+
+StreamBuffer::StreamBuffer(const char *s, int64_t size) {
+    assert(size >= 0);
+    init(true, false, const_cast<char *>(s), size);
+    write_cursor_ = buf_hi_;
+}
+
+StreamBuffer::StreamBuffer(const eng::string &src) {
+    init(true, false, const_cast<char *>(src.c_str()), src.size());
+    write_cursor_ = buf_hi_;
+}
+
+StreamBuffer::~StreamBuffer() {
+    if (owned_ && (buf_lo_ != 0)) eng::free(buf_lo_);
+}
+
+int64_t StreamBuffer::total_reads() const {
+    return (read_cursor_ - buf_lo_) + pre_read_count_;
+}
+
+int64_t StreamBuffer::total_writes() const {
+    return (write_cursor_ - buf_lo_) + pre_read_count_;
+}
+
+int64_t StreamBuffer::fill() const {
+    return write_cursor_ - read_cursor_;
+}
+
+const char *StreamBuffer::data() const {
+    return read_cursor_;
+}
+
+std::string_view StreamBuffer::view() const {
+    return std::string_view(read_cursor_, write_cursor_ - read_cursor_);
+}
+
+bool StreamBuffer::layout_is(int64_t a, int64_t b, int64_t c) {
+    if (read_cursor_ - buf_lo_ != a) return false;
+    if (write_cursor_ - read_cursor_ != b) return false;
+    if (buf_hi_ - write_cursor_ != c) return false;
+    return true;
+}
+
+void StreamBuffer::make_space_slow(int64_t bytes) {
+    assert(owned_ && "We don't own this buffer, can't grow it");
+
+    // Decide whether the current buffer is big enough.
+    int64_t data_size = (write_cursor_ - read_cursor_);
+    int64_t existing_size = (buf_hi_ - buf_lo_);
+    int64_t desired_size = 8192 + ((data_size + bytes) * 2);
+
+    // Update some simple things.
+    pre_read_count_ += (read_cursor_ - buf_lo_);
+    lua_reader_data_ = 0;
+    lua_reader_size_ = 0;
+
+    // Move the data to the beginning of the buffer, or to
+    // the beginning of a new buffer.
+    if (fixed_size_) {
+        assert((data_size + bytes <= existing_size) && "Not enough space in fixed-size buffer");
+        if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size);
+    } else if (existing_size >= desired_size) {
+        if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size);
+    } else {
+        char *nbuf = (char *)eng::malloc(desired_size);
+        if (data_size > 0) memcpy(nbuf, read_cursor_, data_size);
+        if (buf_lo_ != nullptr) eng::free(buf_lo_);
+        buf_lo_ = nbuf;
+        buf_hi_ = nbuf + desired_size;
+    }
+
+    // Update the pointers to the data region.
+    read_cursor_ = buf_lo_;
+    write_cursor_ = buf_lo_ + data_size;
+}
+
+void StreamBuffer::wrote_space(int64_t bytes) {
+    int64_t available = buf_hi_ - write_cursor_;
+    assert(bytes >= 0);
+    assert(available >= bytes);
+    write_cursor_ += bytes;
+}
+
+char *StreamBuffer::get_overwrite(int64_t size, int64_t write_count_after) {
+    int64_t write_count_before = write_count_after - size;
+    assert(write_count_before >= total_reads());
+    assert(write_count_after <= total_writes());
+    return buf_lo_ + (write_count_before - pre_read_count_);
+}
+
+void StreamBuffer::clear() {
+    assert(owned_);
+    if (!fixed_size_) {
+        if (buf_lo_ != nullptr) eng::free(buf_lo_);
+        buf_lo_ = 0;
+        buf_hi_ = 0;
+    }
+    owned_ = true;
+    read_cursor_ = buf_lo_;
+    write_cursor_ = buf_lo_;
+    pre_read_count_ = 0;
+    lua_reader_data_ = 0;
+    lua_reader_size_ = 0;
+}
+
+eng::string StreamBuffer::readline() {
+    char *p = read_cursor_;
+    while ((p < write_cursor_) && (*p != '\n')) p++;
+    if (p == write_cursor_) {
+        return "";
+    } else {
+        p++;
+        eng::string result(read_cursor_, p - read_cursor_);
+        read_cursor_ = p;
+        return result;
+    }
+}
+
+// These routines return true if you can losslessly cast the
+// specified value to the specified type.
+
+static inline bool safe_to_cast_to_int8(int64_t vv) {
+    return ((vv + 0x80LL) & 0xFFFFFFFFFFFFFF00LL) == 0;
+}
+
+static inline bool safe_to_cast_to_int16(int64_t vv) {
+    return ((vv + 0x8000LL) & 0xFFFFFFFFFFFF0000LL) == 0;
+}
+
+static inline bool safe_to_cast_to_int32(int64_t vv) {
+    return ((vv + 0x80000000LL) & 0xFFFFFFFF00000000LL) == 0;
+}
+
+static inline bool safe_to_cast_to_int64(int64_t vv) {
+    return true;
+}
+
+static inline bool safe_to_cast_to_uint8(uint64_t vv) {
+    return (vv & 0xFFFFFFFFFFFFFF00LL) == 0;
+}
+
+static inline bool safe_to_cast_to_uint16(uint64_t vv) {
+    return (vv & 0xFFFFFFFFFFFF0000LL) == 0;
+}
+
+static inline bool safe_to_cast_to_uint32(uint64_t vv) {
+    return (vv & 0xFFFFFFFF00000000LL) == 0;
+}
+
+static inline bool safe_to_cast_to_uint64(uint64_t vv) {
+    return true;
+}
+
+void StreamBuffer::write_bytes(const char *s, int64_t len) {
+    make_space(len);
+    memcpy(write_cursor_, s, len);
+    write_cursor_ += len;
+}
+
+void StreamBuffer::write_bytes(std::string_view s) {
+    make_space(s.size());
+    memcpy(write_cursor_, s.data(), s.size());
+    write_cursor_ += s.size();
+}
+
+const char *StreamBuffer::read_bytes(int64_t bytes) {
+    check_available(bytes);
+    char *data = read_cursor_;
+    read_cursor_ += bytes;
+    return data;
+}
+
+void StreamBuffer::write_int8(int64_t vv) {
+    assert(safe_to_cast_to_int8(vv));
+    int8_t v = vv;
+    make_space(1);
+    memcpy(write_cursor_, &v, 1);
+    write_cursor_ += 1;
+}
+
+void StreamBuffer::write_int16(int64_t vv) {
+    assert(safe_to_cast_to_int16(vv));
+    int16_t v = vv;
+    make_space(2);
+    memcpy(write_cursor_, &v, 2);
+    write_cursor_ += 2;
+}
+
+void StreamBuffer::write_int32(int64_t vv) {
+    assert(safe_to_cast_to_int32(vv));
+    int32_t v = vv;
+    make_space(4);
+    memcpy(write_cursor_, &v, 4);
+    write_cursor_ += 4;
+}
+
+void StreamBuffer::write_int64(int64_t vv) {
+    assert(safe_to_cast_to_int64(vv));
+    int64_t v = vv;
+    make_space(8);
+    memcpy(write_cursor_, &v, 8);
+    write_cursor_ += 8;
+}
+
+void StreamBuffer::write_uint8(uint64_t vv) {
+    assert(safe_to_cast_to_uint8(vv));
+    uint8_t v = vv;
+    make_space(1);
+    memcpy(write_cursor_, &v, 1);
+    write_cursor_ += 1;
+}
+
+void StreamBuffer::write_uint16(uint64_t vv) {
+    assert(safe_to_cast_to_uint16(vv));
+    uint16_t v = vv;
+    make_space(2);
+    memcpy(write_cursor_, &v, 2);
+    write_cursor_ += 2;
+}
+
+void StreamBuffer::write_uint32(uint64_t vv) {
+    assert(safe_to_cast_to_uint32(vv));
+    uint32_t v = vv;
+    make_space(4);
+    memcpy(write_cursor_, &v, 4);
+    write_cursor_ += 4;
+}
+
+void StreamBuffer::write_uint64(uint64_t vv) {
+    assert(safe_to_cast_to_uint64(vv));
+    uint64_t v = vv;
+    make_space(8);
+    memcpy(write_cursor_, &v, 8);
+    write_cursor_ += 8;
+}
+
+void StreamBuffer::write_char(char c) {
+    make_space(1);
+    write_cursor_[0] = c;
+    write_cursor_ += 1;
+}
+
+void StreamBuffer::write_float(float f) {
+    make_space(4);
+    memcpy(write_cursor_, &f, 4);
+    write_cursor_ += 4;
+}
+
+void StreamBuffer::write_double(double d) {
+    make_space(8);
+    memcpy(write_cursor_, &d, 8);
+    write_cursor_ += 8;
+}
+
+int8_t StreamBuffer::read_int8() {
+    check_available(1);
+    int8_t v;
+    memcpy(&v, read_cursor_, 1);
+    read_cursor_ += 1;
+    return v;
+}
+
+int16_t StreamBuffer::read_int16() {
+    check_available(2);
+    int16_t v;
+    memcpy(&v, read_cursor_, 2);
+    read_cursor_ += 2;
+    return v;
+}
+
+int32_t StreamBuffer::read_int32() {
+    check_available(4);
+    int32_t v;
+    memcpy(&v, read_cursor_, 4);
+    read_cursor_ += 4;
+    return v;
+}
+
+int64_t StreamBuffer::read_int64() {
+    check_available(8);
+    int64_t v;
+    memcpy(&v, read_cursor_, 8);
+    read_cursor_ += 8;
+    return v;
+}
+
+char StreamBuffer::read_char() {
+    check_available(1);
+    char c = read_cursor_[0];
+    read_cursor_ += 1;
+    return c;
+}
+
+float StreamBuffer::read_float() {
+    check_available(4);
+    float f;
+    memcpy(&f, read_cursor_, 4);
+    read_cursor_ += 4;
+    return f;
+}
+
+double StreamBuffer::read_double() {
+    check_available(8);
+    double d;
+    memcpy(&d, read_cursor_, 8);
+    read_cursor_ += 8;
+    return d;
+}
+
+void StreamBuffer::write_hashvalue(const util::HashValue &hv) {
+    write_uint64(hv.first);
+    write_uint64(hv.second);
+}
+
+void StreamBuffer::write_string(std::string_view s) {
+    if (s.size() >= 255) {
+        write_uint8(0xFF);
+        write_uint64(s.size());
+        write_bytes(s);
+    } else {
+        write_uint8(s.size());
+        write_bytes(s);
+    }
+}
+
+util::HashValue StreamBuffer::read_hashvalue() {
+    uint64_t f = read_uint64();
+    uint64_t s = read_uint64();
+    return util::HashValue(f,s);
+}
+
+eng::string StreamBuffer::read_string() {
+    return read_string_limit(0xFFFFFFF);
+}
+
+eng::string StreamBuffer::read_string_limit(int64_t max_allowed) {
+    int64_t len = read_uint8();
+    if (len == 255) {
+        len = read_int64();
+    }
+    if (len < 0) throw StreamCorruption();
+    if (len > max_allowed) throw StreamCorruption();
+    const char *bytes = read_bytes(len);
+    return eng::string(bytes, len);
+}
+
+eng::string StreamBuffer::read_entire_contents() {
+    eng::string result(read_cursor_, fill());
+    read_cursor_ = write_cursor_;
+    return result;
+}
+
+void StreamBuffer::overwrite_int8(int64_t write_count_after, int64_t vv) {
+    assert(safe_to_cast_to_int8(vv));
+    int8_t v = vv;
+    char *target = get_overwrite(1, write_count_after);
+    memcpy(target, &v, 1);
+}
+
+void StreamBuffer::overwrite_int16(int64_t write_count_after, int64_t vv) {
+    assert(safe_to_cast_to_int16(vv));
+    int16_t v = vv;
+    char *target = get_overwrite(2, write_count_after);
+    memcpy(target, &v, 2);
+}
+
+void StreamBuffer::overwrite_int32(int64_t write_count_after, int64_t vv) {
+    assert(safe_to_cast_to_int32(vv));
+    int32_t v = vv;
+    char *target = get_overwrite(4, write_count_after);
+    memcpy(target, &v, 4);
+}
+
+void StreamBuffer::overwrite_int64(int64_t write_count_after, int64_t vv) {
+    assert(safe_to_cast_to_int64(vv));
+    int64_t v = vv;
+    char *target = get_overwrite(8, write_count_after);
+    memcpy(target, &v, 8);
+}
+
+void StreamBuffer::overwrite_uint8(int64_t write_count_after, uint64_t vv) {
+    assert(safe_to_cast_to_uint8(vv));
+    uint8_t v = vv;
+    char *target = get_overwrite(1, write_count_after);
+    memcpy(target, &v, 1);
+}
+
+void StreamBuffer::overwrite_uint16(int64_t write_count_after, uint64_t vv) {
+    assert(safe_to_cast_to_uint16(vv));
+    uint16_t v = vv;
+    char *target = get_overwrite(2, write_count_after);
+    memcpy(target, &v, 2);
+}
+
+void StreamBuffer::overwrite_uint32(int64_t write_count_after, uint64_t vv) {
+    assert(safe_to_cast_to_uint32(vv));
+    uint32_t v = vv;
+    char *target = get_overwrite(4, write_count_after);
+    memcpy(target, &v, 4);
+}
+
+void StreamBuffer::overwrite_uint64(int64_t write_count_after, uint64_t vv) {
+    assert(safe_to_cast_to_uint64(vv));
+    uint64_t v = vv;
+    char *target = get_overwrite(8, write_count_after);
+    memcpy(target, &v, 8);
+}
+
+bool StreamBuffer::empty() {
+    return (read_cursor_ == write_cursor_);
+}
+
+void StreamBuffer::verify_empty() {
+    if (read_cursor_ != write_cursor_) {
+        throw StreamCorruption();
+    }
+}
+
+void StreamBuffer::unread_to(int64_t rd_count) {
+    assert(rd_count >= pre_read_count_);
+    assert(rd_count <= total_reads());
+    read_cursor_ = buf_lo_ + (rd_count - pre_read_count_);
+}
+
+void StreamBuffer::unwrite_to(int64_t wr_count) {
+    assert(wr_count >= total_reads());
+    assert(wr_count <= total_writes());
+    write_cursor_ = buf_lo_ + (wr_count - pre_read_count_);
+}
+
+void StreamBuffer::copy_into(StreamBuffer *sb) {
+    sb->write_bytes(read_cursor_, write_cursor_ - read_cursor_);
+}
+
+void StreamBuffer::transfer_into(StreamBuffer *sb) {
+    sb->write_bytes(read_cursor_, write_cursor_ - read_cursor_);
+    read_cursor_ = write_cursor_;
+}
+
+bool StreamBuffer::contents_equal(const StreamBuffer *other) const {
+    int64_t len = fill();
+    if (len != other->fill()) {
+        return false;
+    }
+    return memcmp(read_cursor_, other->read_cursor_, len) == 0;
+}
+
+util::HashValue StreamBuffer::hash() const {
+    uint64_t hash1 = 0x82A7912E7893AC87;
+    uint64_t hash2 = 0x81D402740DE458F3;
+    SpookyHash::ChainHash128(read_cursor_, write_cursor_ - read_cursor_, &hash1, &hash2);
+    return std::make_pair(hash1, hash2);
+}
+
+int StreamBuffer::lua_writer(lua_State *L, const void* p, size_t sz, void* ud) {
+    StreamBuffer *sb = (StreamBuffer *)ud;
+    memcpy(sb->make_space(sz), p, sz);
+    sb->write_cursor_ += sz;
+    return 0;
+}
+
+void *StreamBuffer::lua_writer_ud() {
+    return this;
+}
+
+const char *StreamBuffer::lua_reader(lua_State *L, void *ud, size_t *size) {
+    StreamBuffer *sb = (StreamBuffer *)ud;
+    *size = sb->lua_reader_size_;
+    const char *data = sb->lua_reader_data_;
+    // Next time the reader gets called, there's no data left.
+    sb->lua_reader_data_ = 0;
+    sb->lua_reader_size_ = 0;
+    return data;
+}
+
+void *StreamBuffer::lua_reader_ud(int64_t size) {
+    const char *data = read_bytes(size);
+    lua_reader_data_ = data;
+    lua_reader_size_ = size;
+    return this;
+}
+
+class StreamBufferWriter : public std::streambuf, public eng::opnew {
+private:
+    StreamBuffer *target_;
+public:
+    StreamBufferWriter(StreamBuffer *t) : target_(t) {}
+
+    virtual int_type overflow(int_type c) {
+    	if (c != EOF) {
+            target_->write_uint8(c);
+	    }
+        return c;
+    }
+};
+
+class StreamBufferOStream : public std::ostream, public eng::opnew {
+private:
+    StreamBufferWriter writer_;
+public:
+    StreamBufferOStream(StreamBuffer *t) : std::ostream(nullptr), writer_(t) {
+        rdbuf(&writer_);
+    }
+    virtual ~StreamBufferOStream() {
+    }
+};
+
+std::ostream &StreamBuffer::ostream() {
+    if (ostream_ == nullptr) {
+        ostream_.reset(new StreamBufferOStream(this));
+    }
+    return *ostream_;
+}
+
+static bool streq(const char *str, const char *data) {
+    int len = strlen(str);
+    return memcmp(str, data, len) == 0;
+}
+
+static void write_ztbytes(StreamBuffer *sb, const char *bytes) {
+    sb->write_bytes(bytes, strlen(bytes));
+}
+
+LuaDefine(unittests_streambuffer, "", "some unit tests") {
+    // An 11-byte fixed-size stream buffer.
+    StreamBuffer sb11(11, true);
+
+    // Check the initial state.
+    LuaAssert(L, sb11.layout_is(0, 0, 11));
+
+    // Write a few bytes.
+    write_ztbytes(&sb11, "abcdef");
+    LuaAssert(L, sb11.layout_is(0, 6, 5));
+
+    // Try reading some bytes.
+    LuaAssert(L, streq("abcd", sb11.read_bytes(4)));
+    LuaAssert(L, sb11.layout_is(4, 2, 5));
+    
+    // Put back two bytes.
+    sb11.unread_to(2);
+    LuaAssert(L, sb11.layout_is(2, 4, 5));
+
+    // Read some more bytes.
+    LuaAssert(L, streq("cdef", sb11.read_bytes(4)));
+    LuaAssert(L, sb11.layout_is(6, 0, 5));
+
+    // Reading bytes now should raise an EOF and should not alter layout
+    try {
+        sb11.read_bytes(1);
+        LuaAssert(L, false && "This should have thrown an exception");
+    } catch (const StreamEof &) {}
+    LuaAssert(L, sb11.layout_is(6, 0, 5));
+
+    // Write some more bytes into the stream, forcing a shift-left
+    write_ztbytes(&sb11, "ghijkl");
+    LuaAssert(L, sb11.layout_is(0, 6, 5));
+
+    // Test buffer wrapping a little more.
+    LuaAssert(L, streq("ghi", sb11.read_bytes(3)));
+    LuaAssert(L, sb11.layout_is(3, 3, 5));
+    write_ztbytes(&sb11, "mnopqr");
+    LuaAssert(L, sb11.layout_is(0, 9, 2));
+    LuaAssert(L, streq("jklmnopqr", sb11.read_bytes(9)));
+    LuaAssert(L, sb11.layout_is(9, 0, 2));
+
+    // Test 1-byte integer ops.
+    sb11.clear();
+    for (int i = 0; i < 10; i++) {
+        sb11.write_int8(i);
+        sb11.write_int8(i+100);
+        LuaAssert(L, sb11.read_int8() == i);
+        LuaAssert(L, sb11.read_int8() == i+100);
+    }
+
+    // Test 2-byte integer ops.
+    for (int i = 0; i < 10; i++) {
+        sb11.write_int16(i);
+        sb11.write_int16(i+10000);
+        LuaAssert(L, sb11.read_int16() == i);
+        LuaAssert(L, sb11.read_int16() == i+10000);
+    }
+
+    // Test 4-byte integer ops.
+    for (int i = 0; i < 10; i++) {
+        sb11.write_int32(i);
+        sb11.write_int32(i+1000000);
+        LuaAssert(L, sb11.read_int32() == i);
+        LuaAssert(L, sb11.read_int32() == i+1000000);
+    }
+
+    // Test 8-byte integer ops.
+    for (int i = 0; i < 10; i++) {
+        sb11.write_int64(i + 1000000);
+        LuaAssert(L, sb11.read_int64() == i + 1000000);
+    }
+
+    // Check the write count and read count accumulator ability.
+    sb11.clear();
+    LuaAssert(L, sb11.total_writes() == 0);
+    LuaAssert(L, sb11.total_reads() == 0);
+    for (int i = 0; i < 10; i++) {
+        LuaAssert(L, sb11.total_writes() == i * 8);
+        sb11.write_int32(i);
+        sb11.write_int32(i+1000000);
+        LuaAssert(L, sb11.total_reads() == i * 8);
+        LuaAssert(L, sb11.read_int32() == i);
+        LuaAssert(L, sb11.read_int32() == i+1000000);
+    }
+
+    // Try clearing the buffer.
+    sb11.clear();
+    LuaAssert(L, sb11.layout_is(0, 0, 11));
+
+    // Write a number then overwrite.
+    for (int i = 0; i < 2; i++) {
+        sb11.write_int16(12);
+        sb11.write_int16(34);
+        int64_t wc = sb11.total_writes();
+        sb11.write_int16(56);
+        sb11.write_int16(78);
+        sb11.overwrite_int16(wc, 90);
+        LuaAssert(L, sb11.read_int16() == 12);
+        LuaAssert(L, sb11.read_int16() == 90);
+        LuaAssert(L, sb11.read_int16() == 56);
+        LuaAssert(L, sb11.read_int16() == 78);
+    }
+
+    // Try compact string encoding.
+    sb11.clear();
+    sb11.write_string("abc");
+    sb11.write_string("");
+    sb11.write_string("de");
+    LuaAssert(L, sb11.layout_is(0, 8, 3));
+    LuaAssert(L, sb11.read_string() == "abc");
+    LuaAssert(L, sb11.read_string() == "");
+    LuaAssert(L, sb11.read_string() == "de");
+
+    // Make sure that contents_equal is not obviously broken.
+    StreamBuffer eqsb1, eqsb2;
+    eqsb1.write_int32(12);
+    eqsb1.write_int32(34);
+    eqsb2.write_int32(34);
+    LuaAssert(L, !eqsb1.contents_equal(&eqsb2));
+    eqsb1.read_int32();
+    LuaAssert(L, eqsb1.contents_equal(&eqsb2));
+    eqsb1.write_int32(34);
+    LuaAssert(L, !eqsb1.contents_equal(&eqsb2));
+    
+    // Check the OStream functionality.
+    StreamBuffer ossb;
+    ossb.ostream() << "Testing.";
+    ossb.ostream() << "Foo.";
+    LuaAssertStrEq(L, ossb.read_entire_contents(), "Testing.Foo.");
+
+    return 0;
+}

luprex/cpp/core/streambuffer.hpp

@@ -0,0 +1,462 @@
+//////////////////////////////////////////////////////////////
+//
+// STREAMBUFFER
+//
+// Serves as a buffer for buffered I/O operations.  Has rather sophisticated
+// methods to help serialize and deserialize data.
+//
+// The semantics of this class contain a lot of subtlety!  Please read the
+// documentation carefully.
+//
+// TELLING LINUX TO READ A FILE DESCRIPTOR INTO A STREAMBUFFER
+//
+// It is possible to read from a linux file descriptor, directly into a stream
+// buffer.  You should do this, it's very efficient.  Here is how you do it:
+//
+//     // With linux read, you have to pick an arbitrary buffer size.
+//     const int bufsize = 16384;
+//
+//     // Allocate transient space in the streambuffer.
+//     char *space = streambuffer.make_space(bufsize);
+//
+//     // Call the linux 'read' function.
+//     ssize_t bytes_read = read(fd, space, bufsize);
+//
+//     // Append the bytes read to the streambuffer.
+//     streambuffer.wrote_space(bytes_read);
+//
+// The make_space operation allocates an array of bytes where the data can be
+// written, and returns a pointer to that array of bytes.  The read operation
+// fills some or all of the allocated bytes. Finally, the wrote_space operation
+// notifies the StreamBuffer that some of the bytes have been filled with data.
+// These bytes are appended to the StreamBuffer. 
+//
+// The pointer returned by 'make_space' is only valid until you mutate the
+// StreamBuffer. Therefore, you should call 'make_space', then immediately fill
+// the bytes.  It is imperative that 'wrote_space' be the first mutator after
+// 'make_space.'  You should think of 'make_space' followed by 'wrote_space' as
+// a single two-phase operation.
+//
+// THE OVERWRITE_INT METHODS:
+//
+// These overwrite methods are meant to help deal with this situation: you want
+// to write a length followed by some data, but you don't know the length until
+// after you've written the data.  The workaround: write a dummy length, then
+// write the data, and then overwrite the previously-written length with the
+// correct length. This is the construction that accomplishes this:
+//
+//     // Write the dummy length, this will get overwritten.
+//     streambuffer.write_int32(0);
+//
+//     // Write the data, and calculate its length in bytes.
+//     int64_t write_count_1 = streambuffer.total_writes();
+//     write_data(stream);
+//     int64_t write_count_2 = streambuffer.total_writes();
+//     int64_t data_len = write_count_2 - write_count_1;
+//
+//     // Overwrite the previously-written dummy length.
+//     streambuffer.overwrite_int32(write_count_1, data_len); 
+//
+// Almost all of this is self-explanatory, but the last line is interesting. In
+// order to know what part of the buffer to overwrite, overwrite_int uses
+// write_count_1 as a pointer into the buffer - it points immedately to the
+// right of the integer to overwrite.
+//
+// OVERWRITE_INT LIMITS
+//
+// If you use write_int to write an integer into the buffer, you are allowed to
+// overwrite that integer UNTIL you do a read from the buffer.  Once you do a
+// read, it is no longer legal to overwrite ints that you wrote BEFORE the read.
+//
+// WRITE_STRING STORES THE STRING LENGTH, WRITE_BYTES DOES NOT
+//
+// write_string writes a string into the buffer and prepends a length. The
+// encoding of the length field is designed to be efficient for short strings
+// but still capable of encoding long lengths.
+//
+// write_bytes doesn't store the data length in the buffer.  It's just a raw
+// write of bytes.
+//
+// STREAM EXCEPTIONS
+//
+// If you do a read_int64, but the buffer doesn't contain the necessary 8 bytes,
+// it throws a StreamEof exception. In general, during reading, the following
+// common situations generate StreamEof or StreamCorruption exceptions:
+//
+//   * not enough bytes to satisfy a 'read' call: StreamEof
+//   * call read_eof, but the buffer is not empty: StreamCorruption
+//   * call read_string, but the string is unreasonably long: StreamCorruption
+//
+// Exceptions are only generated when reading from a stream that contains bad
+// data.  Any other error generates a full-blown abort.  For example, if you try
+// to write to a stream that's not open for writing, that's an abort, not an
+// exception.  Write operations never generate exceptions.
+//
+// Sometimes, it is convenient to throw StreamCorruption yourself, if you detect
+// that the data you've read from a stream is invalid.  This can make error
+// handling a little cleaner.
+//
+// READ BYTES POINTER VALIDITY
+//
+// When you call read_bytes, it returns a pointer to a block of bytes. This
+// pointer only remains valid until you do a 'write' into the stream.
+//
+// UNREADING BYTES
+//
+// It's possible to 'unread' bytes that you've already read from a stream. This
+// makes it possible to read those same bytes again.
+//
+// A common situation where this might be useful is: you're decoding a message,
+// but you discover halfway through the process of decoding the message that you
+// haven't received the whole message yet.  In that case, it may be desirable to
+// unread the partial message, so that you can wait for the rest of the message
+// to be received.
+//
+// Here is the construction that accomplishes this:
+//
+//     // Get the stream's read count before parsing the message.
+//     size_t read_count_before = streambuffer.total_reads();
+//
+//     // Parse the message, but if there's an EOF, deal with it:
+//     try {
+//         // Parse the message.
+//         int32_t value1 = streambuffer.read_int32();
+//         eng::string value2 = streambuffer.read_string(maxlen);
+//         int64_t value3 = streambuffer.read_int64();
+//
+//         // Great! I got the whole message.
+//         execute_message(value1, value2, value3);
+//     } catch (StreamEof) {
+//         // I ran out of bytes.  Unread the message.
+//         streambuffer.unread(read_count_before);
+//     }
+//
+// UNREAD LIMITS
+//
+// If you read bytes from a stream, that data can be 'unread' until you do a
+// write.  After a write, it is no longer possible to 'unread' data that you
+// read before the write.
+//
+// STREAMBUFFERS THAT DON'T OWN THEIR OWN MEMORY
+//
+// If you create a streambuffer using this constructor:
+//
+//     StreamBuffer(const char *data, uint64_t len);
+//
+// This StreamBuffer reads from an external (unowned) block of bytes, which is
+// not copied! The StreamBuffer saves the pointer that you passed in.  This
+// pointer must remain valid until you're done with the StreamBuffer.
+//
+// A StreamBuffer that reads from an external block of bytes is read-only.
+// Attempts to write to this buffer will be caught and will cause an abort. The
+// total_writes for such a buffer returns the 'len' value that you initialized
+// the buffer with.
+//
+// NESTED DECODING
+//
+// Here is an interesting construct:
+//
+//     // Read a message from the stream.
+//     size_t len = streambuffer.read_int32()
+//     const char *bytes = streambuffer.read_bytes(len);
+//
+//     // Construct another stream object to decode the message.
+//     StreamBuffer substream(bytes, len);
+//     decode(substream);
+//
+// This is perfectly valid and a potentially convenient way to parse the
+// contents of a message.  Note that the substream contains a pointer to
+// the parent stream's buffer, and therefore, data corruption will occur
+// if you mutate the parent stream while reading the substream.
+// 
+// USING A STREAMBUFFER TO READ AN ENTIRE FILE
+//
+// If you wish to read an entire file and store the file contents in a
+// StreamBuffer, you should probe the size of the file, then allocate a
+// StreamBuffer of the correct size using this constructor:
+//
+//     StreamBuffer(int64_t size);
+//
+// Then, you can use 'alloc_space' and 'wrote_space' to read the file into the
+// buffer in a single read call.
+//
+// USING A STREAMBUFFER AS A LUA_WRITER OR LUA_READER
+//
+// You can use a streambuffer as a lua_Writer, as follows:
+//
+//       lua_dump(L, stream.lua_writer(), stream.lua_writer_ud());
+//
+// Anything written to the lua_writer gets appended to the streambuffer, the
+// same as if it had been written using write_bytes.
+//
+// You can use a streambuffer as a lua_Reader, as follows:
+//
+//      lua_load (L, stream.lua_reader(), stream.lua_reader_ud(nbytes), ...)
+//
+// The exact semantics of the lua_reader are tricky, so be careful:
+// lua_reader_ud calls 'read_bytes' immediately, and it stores the bytes in a
+// "cache of bytes for lua."  Then, when the lua_reader gets invoked, the reader
+// returns the entire contents of the cache, and it clears the cache. Here are
+// some consequences of this design:
+//
+//    1. The number of bytes read from the stream is always exactly equal to
+//       nbytes, even if lua never calls the lua_reader.
+//
+//    2. If the stream doesn't contain nbytes, a StreamEof exception gets thrown
+//       from lua_reader_ud, not from the lua_Reader.  This is good, because it
+//       means exceptions don't get thrown from inside the lua runtime.
+//
+//////////////////////////////////////////////////////////////
+
+
+#ifndef STREAMBUFFER_HPP
+#define STREAMBUFFER_HPP
+
+#include "wrap-string.hpp"
+#include "wrap-sstream.hpp"
+
+#include <utility>
+#include <cstdint>
+#include <cassert>
+
+#include "luastack.hpp"
+#include "util.hpp"
+
+class StreamException : public eng::nevernew
+{
+public:
+    virtual char const *what() const { return "General stream exception"; }
+};
+
+class StreamEof : public StreamException
+{
+public:
+    virtual char const *what() const { return "Stream ran out of data"; }
+};
+
+class StreamCorruption : public StreamException
+{
+public:
+    virtual char const *what() const { return "Stream contained invalid data"; }
+};
+
+class StreamBuffer : public eng::nevernew {
+public:    
+    // Construct an empty buffer.
+    StreamBuffer();
+
+    // Construct an empty buffer, preallocate the specified amount of space.
+    StreamBuffer(int64_t size, bool fixed_size);
+
+    // Construct a streambuffer that reads from an external block of bytes.
+    StreamBuffer(const char *s, int64_t len);
+
+    // Construct a streambuffer that reads from an external block of bytes.
+    StreamBuffer(const eng::string &data);
+
+    // Delete a StreamBuffer.
+    ~StreamBuffer();
+    
+    // Get the total number of bytes ever read from this buffer.
+    int64_t total_reads() const;
+    
+    // Get the total number of bytes ever written to this buffer.
+    int64_t total_writes() const;
+
+    // Amount of data inside the buffer.
+    int64_t fill() const;
+    
+    // Get a pointer to the data.
+    const char *data() const;
+    
+    // Get entire contents as a string_view
+    std::string_view view() const;
+
+    // Discard all data.  Reset total read and write counts.
+    // Frees up as much space as possible.
+    void clear();
+
+    // Attempt to do a "readline".  If there is no newline in
+    // the buffer, returns empty string.  If there is a newline,
+    // returns a block of text that ends in newline.
+    eng::string readline();
+
+    // Write block of bytes into the buffer.
+    //
+    // Caution: this function doesn't write the length!  
+    // It just writes the bytes.
+    //
+    void write_bytes(const char *bytes, int64_t len);
+    void write_bytes(std::string_view bytes);
+    
+    // Read a block of bytes from the buffer.
+    //
+    // Caution: the pointer returned is a pointer to the stream's buffer. It is
+    // only valid until you mutate the buffer. Throws StreamEof if the specified
+    // number of bytes aren't present.
+    //
+    const char *read_bytes(int64_t bytes);
+
+    // Write integers and floats into the buffer.
+    //
+    // Note that integral parameters are all 64 bits.  That's so that I can do
+    // runtime error checking to verify that the numbers are all in-range.
+    //
+    void write_int8(int64_t v);
+    void write_int16(int64_t v);
+    void write_int32(int64_t v);
+    void write_int64(int64_t v);
+    void write_uint8(uint64_t v);
+    void write_uint16(uint64_t v);
+    void write_uint32(uint64_t v);
+    void write_uint64(uint64_t v);
+    void write_char(char c);
+    void write_float(float f);
+    void write_double(double d);
+
+    // Read fixed-size integers from the buffer.
+    //
+    // May throw StreamEof if the specified number of bytes aren't present.
+    //
+    int8_t read_int8();
+    int16_t read_int16();
+    int32_t read_int32();
+    int64_t read_int64();
+    uint8_t read_uint8() { return read_int8(); }
+    uint16_t read_uint16() { return read_int16(); }
+    uint32_t read_uint32() { return read_int32(); }
+    uint64_t read_uint64() { return read_int64(); }
+    char read_char();
+    float read_float();
+    double read_double();
+
+    // Write other types into the buffer.
+    //
+    // Note that strings are preceded by a length field.  Reading
+    // a string works by reading the length field, and then reading
+    // the correct number of bytes.
+    //
+    void write_bool(bool b) { write_int8(b ? 1 : 0); }
+    void write_hashvalue(const util::HashValue &hv);
+    void write_string(std::string_view s);
+
+    // Read other types from the buffer.
+    //
+    // Throws StreamEof if the specified number of bytes aren't present.
+    // Read string with a length limit will throw 'StreamCorruption' if the
+    // length is too long.
+    //
+    bool read_bool() { return read_int8(); } 
+    util::HashValue read_hashvalue();
+    eng::string read_string();
+    eng::string read_string_limit(int64_t max_allowed);
+
+    // Read the entire contents of the buffer as a string.
+    //
+    eng::string read_entire_contents();
+
+    // Overwrite values previously written to the buffer.
+    //
+    // See the comment at the top of this file for an explanation.
+    //
+    void overwrite_int8(int64_t write_count_after, int64_t v);
+    void overwrite_int16(int64_t write_count_after, int64_t v);
+    void overwrite_int32(int64_t write_count_after, int64_t v);
+    void overwrite_int64(int64_t write_count_after, int64_t v);
+    void overwrite_uint8(int64_t write_count_after, uint64_t v);
+    void overwrite_uint16(int64_t write_count_after, uint64_t v);
+    void overwrite_uint32(int64_t write_count_after, uint64_t v);
+    void overwrite_uint64(int64_t write_count_after, uint64_t v);
+
+    // This function checks to see if the buffer is empty.
+    bool empty();
+
+    // Verify that the buffer is empty, if not, throw StreamCorruption.
+    void verify_empty();
+
+    // Make sure the specified number of bytes are available to read.
+    void check_available(int64_t bytes) {
+        int64_t avail = write_cursor_ - read_cursor_;
+        if (avail < bytes) {
+            throw StreamEof();
+        }
+    }
+
+    // Rewind the read cursor to a previous position.
+    void unread_to(int64_t total_reads);
+    
+    // Rewind the write cursor to a previous position.
+    void unwrite_to(int64_t total_writes);
+    
+    // Copy the entire contents of this streambuffer into another one.
+    void copy_into(StreamBuffer *sb);
+
+    // Transfer the entire contents of this streambuffer into another one.
+    void transfer_into(StreamBuffer *sb);
+
+    // Compare the contents of this streambuffer to another one.
+    bool contents_equal(const StreamBuffer *sb) const;
+
+    // Calculate a noncryptographic but good hash of what's in the buffer.
+    util::HashValue hash() const;
+
+    // Use the stream buffer as a lua_Writer.
+    static int lua_writer(lua_State *L, const void* p, size_t sz, void* ud);
+    void *lua_writer_ud();
+    
+    // Use the stream buffer as a lua_Reader.
+    static const char *lua_reader(lua_State *L, void *data, size_t *size);
+    void *lua_reader_ud(int64_t bytes);
+
+    // Get an ostream that writes into the StreamBuffer.
+    std::ostream &ostream();
+
+private:
+    // Start and end of the allocated block.
+    char *buf_lo_;
+    char *buf_hi_;
+
+    // The write and read cursors.
+    char *write_cursor_;
+    char *read_cursor_;
+
+    // Number of bytes read before buffer was last aligned.
+    int64_t pre_read_count_;
+
+    // True if we own this buffer.
+    bool owned_;
+    
+    // True if we're not allowed to expand this buffer.
+    bool fixed_size_;
+
+    // Lua reader return value.
+    const char *lua_reader_data_;
+    int64_t lua_reader_size_;
+
+    // The ostream.  Only allocated on demand.
+    std::unique_ptr<std::ostream> ostream_;
+
+    // Initialize with a new buffer.
+    void init(bool fixed, bool owned, char *buf, int64_t size);
+
+    // Make the specified amount of space in the buffer for writing.
+    // Return a pointer to the space.
+    char *make_space(int64_t bytes) {
+        int64_t available = buf_hi_ - write_cursor_;
+        if (available < bytes) make_space_slow(bytes);
+        return write_cursor_;
+    }
+    void make_space_slow(int64_t bytes);
+
+    void wrote_space(int64_t bytes);
+    
+    // Implementation for the overwrite_int functions.
+    char *get_overwrite(int64_t size, int64_t write_count_after);
+
+    // This is for unit testing.
+    bool layout_is(int64_t a, int64_t b, int64_t c);
+    
+    friend int lfn_unittests_streambuffer(lua_State *L);
+};
+
+#endif // STREAMBUFFER_HPP

luprex/cpp/core/table.cpp

@@ -0,0 +1,622 @@
+
+#include "wrap-string.hpp"
+
+#include "table.hpp"
+#include "source.hpp"
+
+
+bool table_equal(LuaStack &LS, LuaSlot t1, LuaSlot t2) {
+    lua_State *L = LS.state();
+    int top = lua_gettop(L);
+    LS.checktable(t1, "table1");
+    LS.checktable(t2, "table2");
+    int nkeys1 = lua_nkeys(L, t1.index());
+    int nkeys2 = lua_nkeys(L, t2.index());
+    if (nkeys1 != nkeys2) return false;
+    int total = 0;
+    lua_pushnil(L);
+    while (lua_next(L, t1.index()) != 0) {
+        lua_pushvalue(L, -2);   // k v1 k
+        lua_rawget(L, t2.index());  // k v1 v2
+        if (!lua_rawequal(L, -1, -2)) {
+            lua_settop(L, top);
+            return false;
+        }
+        lua_pop(L, 2);
+        total += 1;
+    }
+    assert(total == nkeys1);
+    lua_settop(L, top);
+    return true;
+}
+
+LuaDefine(table_equal, "table1,table2", "return true if two tables contain the same keys and values") {
+    LuaArg t1, t2;
+    LuaRet eql;
+    LuaStack LS(L, t1, t2, eql);
+    LS.set(eql, table_equal(LS, t1, t2));
+    return LS.result();
+}
+
+LuaDefine(table_findremove, "vector,value", "remove all occurrences of value from vector") {
+    luaL_checktype(L, -2, LUA_TTABLE);
+    int src = 1;
+    int dst = 1;
+    while (true) {
+        lua_pushinteger(L, src);
+        lua_rawget(L, -3);
+        if (lua_rawequal(L, -1, -2)) {
+            src++;
+            lua_pop(L, 1);
+        } else if (lua_isnil(L, -1)) {
+            lua_pop(L, 1);
+            int removed = src - dst;
+            while (src > dst) {
+                lua_pushinteger(L, dst);
+                lua_pushnil(L);
+                lua_rawset(L, -4);
+                dst++;
+            }
+            lua_pop(L, 2);
+            lua_pushinteger(L, removed);
+            return 1;
+        } else {
+            if (src > dst) {
+                lua_pushinteger(L, dst);
+                lua_insert(L, lua_gettop(L) - 1);
+                lua_rawset(L, -4);
+            } else {
+                lua_pop(L, 1);
+            }
+            src++;
+            dst++;
+        }
+    }
+}
+
+
+LuaDefine(table_push, "vector,value", "push a value onto the end of a vector") {
+    luaL_checktype(L, -2, LUA_TTABLE);
+    int len = lua_rawlen(L, -2);
+    lua_pushinteger(L, len+1);
+    lua_pushvalue(L, -2);
+    lua_rawset(L, -4);
+    lua_pop(L, 2);
+    return 0;
+}
+
+LuaDefine(table_find, "vector,value", "find the first occurrence of value in vector") {
+    luaL_checktype(L, -2, LUA_TTABLE);
+    for (int i = 1; ; i++) {
+        lua_pushinteger(L, i);
+        lua_rawget(L, -3);
+        if (lua_rawequal(L, -1, -2)) {
+            lua_pop(L, 3);
+            lua_pushinteger(L, i);
+            return 1;
+        } else if (lua_isnil(L, -1)) {
+            lua_pop(L, 3);
+            lua_pushnil(L);
+            return 1;
+        } else {
+            lua_pop(L, 1);
+        }
+    }
+}
+
+LuaDefine(table_empty, "table", "return true if the table has zero keys") {
+    luaL_checktype(L, -1, LUA_TTABLE);
+    int total = lua_nkeys(L, -1);
+    lua_pushboolean(L, (total == 0)?1:0);
+    return 1;
+}
+
+LuaDefine(table_count, "table", "return the number of keys in table") {
+    luaL_checktype(L, -1, LUA_TTABLE);
+    int total = lua_nkeys(L, -1);
+    lua_pushinteger(L, total);
+    return 1;
+}
+
+LuaDefine(table_clear, "table,metaflag", "clear all keys, and optionally the metatable") {
+    LuaArg tab, clearmeta;
+    LuaVar metatable, metafield;
+    LuaStack LS(L, tab, clearmeta, metatable, metafield);
+    LS.checktable(tab, "table");
+    if (LS.ckboolean(clearmeta)) {
+        LS.getmetatable(metatable, tab);
+        if (LS.istable(metatable)) {
+            LS.rawget(metafield, metatable, "__metatable");
+            if (!LS.isnil(metafield)) {
+                luaL_error(L, "Cannot clear metatable.");
+                return LS.result();
+            }
+        }
+        LS.cleartable(tab, true);
+    } else {
+        LS.cleartable(tab, false);
+    }
+    return LS.result();
+}
+
+LuaDefine(table_getflagbits, "table", "get the table's flag bits (debugging only)") {
+    LuaArg tab;
+    LuaRet bits;
+    LuaStack LS(L, tab, bits);
+    uint16_t ubits = lua_getflagbits(L, tab.index());
+    LS.set(bits, ubits);
+    return LS.result();
+}
+
+LuaDefine(table_setflagbits, "table,bits", "set the table's flag bits (debugging only)") {
+    LuaArg tab, bits;
+    LuaStack LS(L, tab, bits);
+    uint16_t ubits = LS.ckinteger(bits);
+    lua_setflagbits(L, tab.index(), ubits);
+    return LS.result();
+}
+
+/////////////////////////////////////////////////////////////
+//
+// Deque operators.
+//
+/////////////////////////////////////////////////////////////
+
+#define DEQUE_LEFT 1
+#define DEQUE_FILL 2
+#define DEQUE_MAX 3
+#define DEQUE_BASE 4
+
+void deque_checktype(lua_State *L, int slot, int type) {
+    if (lua_type(L, slot) != type) {
+        luaL_error(L, "object is not a valid deque");
+    }
+}
+
+void deque_get_info(lua_State *L, int deque, int *left, int *fill, int *max) {
+    luaL_checktype(L, deque, LUA_TTABLE);
+    if (left) {
+        lua_rawgeti(L, deque, DEQUE_LEFT);
+        deque_checktype(L, -1, LUA_TNUMBER);
+        *left = lua_tointeger(L, -1);
+        lua_pop(L, 1);
+    }
+    if (fill) {
+        lua_rawgeti(L, deque, DEQUE_FILL);
+        deque_checktype(L, -1, LUA_TNUMBER);
+        *fill = lua_tointeger(L, -1);
+        lua_pop(L, 1);
+    }
+    if (max) {
+        lua_rawgeti(L, deque, DEQUE_MAX);
+        deque_checktype(L, -1, LUA_TNUMBER);
+        *max = lua_tointeger(L, -1);
+        lua_pop(L, 1);
+    }
+}
+
+void deque_put_info(lua_State *L, int deque, int *left, int *fill, int *max) {
+    if (left) {
+        lua_pushinteger(L, *left);
+        lua_rawseti(L, deque, DEQUE_LEFT);
+    }
+    if (fill) {
+        lua_pushinteger(L, *fill);
+        lua_rawseti(L, deque, DEQUE_FILL);
+    }
+    if (max) {
+        lua_pushinteger(L, *max);
+        lua_rawseti(L, deque, DEQUE_MAX);
+    }
+}
+    
+int deque_make_room(lua_State *L, int deque, int left, int fill, int max) {
+    if (fill == max) {
+        for (int i = 0; i < left; i++) {
+            lua_rawgeti(L, deque, DEQUE_BASE + i);
+            lua_rawseti(L, deque, DEQUE_BASE + i + max);
+            lua_pushinteger(L, 0);
+            lua_rawseti(L, deque, DEQUE_BASE + i);
+        }
+        for (int i = left; i < max; i++) {
+            lua_pushinteger(L, 0);
+            lua_rawseti(L, deque, DEQUE_BASE + i + max);
+        }
+        max *= 2;
+        lua_pushinteger(L, max);
+        lua_rawseti(L, deque, DEQUE_MAX);
+    }
+    return max;
+}
+
+LuaDefine(deque_create, "", "create a deque") {
+    LuaRet rdeque;
+    LuaVar classobj;
+    LuaStack LS(L, rdeque, classobj);
+    const int imax = 4;
+    eng::string err = LS.getclass(classobj, "deque");
+    if (err != "") {
+        luaL_error(L, "Class deque has been corrupted");
+    }
+    LS.createtable(rdeque, DEQUE_BASE + imax - 1, 0);
+    LS.rawset(rdeque, DEQUE_LEFT, 0);
+    LS.rawset(rdeque, DEQUE_FILL, 0);
+    LS.rawset(rdeque, DEQUE_MAX, imax);
+    for (int i = 0; i < imax; i++) {
+        LS.rawset(rdeque, DEQUE_BASE + i, 0);
+    }
+    LS.setmetatable(rdeque, classobj);
+    return LS.result();
+}
+
+LuaDefine(deque_pushl, "deque,value", "push onto the left end of a deque") {
+    LuaArg deque, elt;
+    LuaStack LS(L, deque, elt);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    max = deque_make_room(L, deque.index(), left, fill, max);
+    int target = (left - 1) & (max-1);
+    LS.rawset(deque, DEQUE_BASE + target, elt);
+    fill += 1;
+    left = (left - 1) & (max - 1);
+    deque_put_info(L, deque.index(), &left, &fill, NULL);
+    return LS.result();
+}
+
+LuaDefine(deque_pushr, "deque,value", "push onto the right end of a deque") {
+    LuaArg deque, elt;
+    LuaStack LS(L, deque, elt);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    max = deque_make_room(L, deque.index(), left, fill, max);
+    int target = (left + fill) & (max-1);
+    LS.rawset(deque, DEQUE_BASE + target, elt);
+    fill += 1;
+    deque_put_info(L, deque.index(), NULL, &fill, NULL);
+    return LS.result();
+}
+
+LuaDefine(deque_popl, "deque", "pop the left end of a deque") {
+    LuaArg deque;
+    LuaRet result;
+    LuaStack LS(L, deque, result);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    if (fill == 0) {
+        LS.set(result, LuaNil);
+        return LS.result();
+    }
+    LS.rawget(result, deque, DEQUE_BASE + left);
+    LS.rawset(deque, DEQUE_BASE + left, 0);
+    left = (left + 1) & (max - 1);
+    fill -= 1;
+    deque_put_info(L, deque.index(), &left, &fill, NULL);
+    return LS.result();
+}
+
+LuaDefine(deque_popr, "deque", "pop the right end of a deque") {
+    LuaArg deque;
+    LuaRet result;
+    LuaStack LS(L, deque, result);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    if (fill == 0) {
+        LS.set(result, LuaNil);
+        return LS.result();
+    }
+    int target = (left + fill - 1) & (max - 1);
+    LS.rawget(result, deque, DEQUE_BASE + target);
+    LS.rawset(deque, DEQUE_BASE + target, 0);
+    fill -= 1;
+    deque_put_info(L, deque.index(), NULL, &fill, NULL);
+    return LS.result();
+}
+
+LuaDefine(deque_nthl, "deque,n", "return the nth item from the left end of a deque") {
+    LuaArg deque, nn;
+    LuaRet result;
+    LuaStack LS(L, deque, nn, result);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    int n = LS.ckint(nn);
+    if ((n < 1) || (n > fill)) {
+        LS.set(result, LuaNil);
+        return LS.result();
+    }
+    int target = (left + n - 1) & (max - 1);
+    LS.rawget(result, deque, DEQUE_BASE + target);
+    return LS.result();
+}
+
+LuaDefine(deque_nthr, "deque,n", "return the nth item from the right end of a deque") {
+    LuaArg deque, nn;
+    LuaRet result;
+    LuaStack LS(L, deque, nn, result);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    int n = LS.ckint(nn);
+    if ((n < 1) || (n > fill)) {
+        LS.set(result, LuaNil);
+        return LS.result();
+    }
+    int target = (left + fill - n) & (max - 1);
+    LS.rawget(result, deque, DEQUE_BASE + target);
+    return LS.result();
+}
+
+LuaDefine(deque_setl, "deque,n,value", "set the nth item from the left end of a deque") {
+    LuaArg deque, nn, val;
+    LuaStack LS(L, deque, nn, val);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    int n = LS.ckint(nn);
+    if ((n < 1) || (n > fill)) {
+        luaL_error(L, "invalid index");
+        return LS.result();
+    }
+    int target = (left + n - 1) & (max - 1);
+    LS.rawset(deque, DEQUE_BASE + target, val);
+    return LS.result();
+}
+
+LuaDefine(deque_setr, "deque,n,value", "set the nth item from the right end of a deque") {
+    LuaArg deque, nn, val;
+    LuaStack LS(L, deque, nn, val);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    int n = LS.ckint(nn);
+    if ((n < 1) || (n > fill)) {
+        luaL_error(L, "invalid index");
+        return LS.result();
+    }
+    int target = (left + fill - n) & (max - 1);
+    LS.rawset(deque, DEQUE_BASE + target, val);
+    return LS.result();
+}
+
+LuaDefine(deque_findl, "deque,value", "find the first occurence of value in deque, starting from left") {
+    LuaArg deque, val;
+    LuaRet pos;
+    LuaVar check;
+    LuaStack LS(L, deque, val, pos, check);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    for (int i = 0; i < fill; i++) {
+        int index = (left + i) & (max - 1);
+        LS.rawget(check, deque, DEQUE_BASE + index);
+        if (LS.rawequal(check, val)) {
+            LS.set(pos, i + 1);
+            return LS.result();
+        }
+    }
+    LS.set(pos, LuaNil);
+    return LS.result();
+}
+
+LuaDefine(deque_findr, "deque,value", "find the first occurrence of value in deque, starting from right") {
+    LuaArg deque, val;
+    LuaRet pos;
+    LuaVar check;
+    LuaStack LS(L, deque, val, pos, check);
+    int left, fill, max;
+    deque_get_info(L, deque.index(), &left, &fill, &max);
+    int base = left + fill - 1;
+    for (int i = 0; i < fill; i++) {
+        int index = (base - i) & (max - 1);
+        LS.rawget(check, deque, DEQUE_BASE + index);
+        if (LS.rawequal(check, val)) {
+            LS.set(pos, i + 1);
+            return LS.result();
+        }
+    }
+    LS.set(pos, LuaNil);
+    return LS.result();
+}
+
+LuaDefine(deque_size, "deque", "return the number of items in the deque") {
+    LuaArg deque;
+    LuaRet size;
+    LuaStack LS(L, deque, size);
+    LS.checktable(deque, "deque");
+    LS.rawget(size, deque, DEQUE_FILL);
+    LS.checknumber(size, "deque size");
+    return LS.result();
+}
+
+
+/////////////////////////////////////////////////////////////
+//
+// table_getpairs
+//
+// Given a table, return all the pairs in the table as a sequence.
+// The resulting sequence contains a 1 in the first position,
+// followed by the pairs, like so:
+//
+// sortedpairs({a=1,b=2,c=3}) => {1,"a",1,"b",2,"c",3}
+// 
+// Note that this function is not directly exposed to lua.
+// It's exposed only through the 'sortedpairs' iterator.
+//
+/////////////////////////////////////////////////////////////
+
+static void pushkey (lua_State *L, int tab, int i) {
+    lua_rawgeti(L, tab, i*2);
+}
+
+static void push2keys (lua_State *L, int tab, int i, int j) {
+    lua_rawgeti(L, tab, i*2);
+    lua_rawgeti(L, tab, j*2);
+}
+
+static void pop2keys (lua_State *L, int tab, int i, int j) {
+    lua_rawseti(L, tab, i*2);
+    lua_rawseti(L, tab, j*2);
+}
+
+static void swap2values (lua_State *L, int tab, int i, int j) {
+    lua_rawgeti(L, tab, i*2+1);
+    lua_rawgeti(L, tab, j*2+1);
+    lua_rawseti(L, tab, i*2+1);
+    lua_rawseti(L, tab, j*2+1);
+}
+
+static void auxsort (lua_State *L, int tab, int l, int u) {
+    while (l < u) { /* for tail recursion */
+        int i, j;
+        /* sort elements a[l], a[(l+u)/2] and a[u] */
+        push2keys(L, tab, l, u);
+        if (lua_genlt(L, -1, -2)) {
+            pop2keys(L, tab, l, u);
+            swap2values(L, tab, l, u);
+        } else {
+            lua_pop(L, 2);
+        }
+        if (u - l == 1) break; /* only 2 elements */
+        i = (l + u) / 2;
+        push2keys(L, tab, i, l);
+        if (lua_genlt(L, -2, -1)) {
+            pop2keys(L, tab, i, l);
+            swap2values(L, tab, i, l);
+        } else {
+            lua_pop(L, 1); /* remove a[l] */
+            pushkey(L, tab, u);
+            if (lua_genlt(L, -1, -2)) {
+                pop2keys(L, tab, i, u);
+                swap2values(L, tab, i, u);
+            } else {
+                lua_pop(L, 2);
+            }
+        }
+        if (u - l == 2) break; /* only 3 elements */
+        /* put the pivot value on top of the stack and keep it there */
+        pushkey(L, tab, i);
+        /* move the pivot from i to u-1 */
+        lua_pushvalue(L, -1);
+        pushkey(L, tab, u-1);
+        pop2keys(L, tab, i, u-1);
+        swap2values(L, tab, i, u-1);
+        /* a[l] <= P == a[u-1] <= a[u], only need to sort from l+1 to u-2 */
+        i = l;
+        j = u - 1;
+        for (;;) { /* invariant: a[l..i] <= P <= a[j..u] */
+            /* repeat ++i until a[i] >= P */
+            while (pushkey(L, tab, ++i), lua_genlt(L, -1, -2)) {
+                if (i >= u) luaL_error(L, "invalid order function for sorting");
+                lua_pop(L, 1); /* remove a[i] */
+            }
+            /* repeat --j until a[j] <= P */
+            while (pushkey(L, tab, --j), lua_genlt(L, -3, -1)) {
+                if (j <= l) luaL_error(L, "invalid order function for sorting");
+                lua_pop(L, 1); /* remove a[j] */
+            }
+            if (j < i) {
+                lua_pop(L, 3); /* pop pivot, a[i], a[j] */
+                break;
+            }
+            pop2keys(L, tab, i, j);
+            swap2values(L, tab, i, j);
+        }
+        push2keys(L, tab, u-1, i);
+        pop2keys(L, tab, u-1, i);
+        swap2values(L, tab, u-1, i);
+        /* a[l..i-1] <= a[i] == P <= a[i+1..u] */
+        /* adjust so that smaller half is in [j..i] and larger one in [l..u] */
+        if (i - l < u - i) {
+            j = l;
+            i = i - 1;
+            l = i + 2;
+        } else {
+            j = i + 1;
+            i = u;
+            u = j - 2;
+        }
+        auxsort(L, tab, j, i); /* call recursively the smaller one */
+    }                          /* repeat the routine for the larger one */
+}
+
+bool table_getpairs(LuaStack &LS0, LuaSlot tab, LuaSlot pairs, bool sort) {
+    lua_State *L = LS0.state();
+    LuaVar key, value;
+    LuaStack LS(L, key, value);
+    bool sorted = true;
+    // Create the table, store the initial 1.
+    int total = lua_nkeys(L, tab.index());
+    LS.createtable(pairs, total * 2 + 1, 0);
+    LS.rawset(pairs, 1, 1);
+    // Transfer the pairs into the sequence.
+    lua_pushnil(L);
+    int offset = 2;
+    while (lua_next(L, tab.index()) != 0) {
+        int ktype = lua_type(L, -2);
+        if (ktype != LUA_TNUMBER && ktype != LUA_TSTRING && ktype != LUA_TBOOLEAN) {
+            sorted = false;
+        }
+        lua_pushvalue(L, -2); // K V K
+        lua_rawseti(L, pairs.index(), offset++);
+        lua_rawseti(L, pairs.index(), offset++);
+    }
+    if (sort) {
+        auxsort(L, pairs.index(), 1, total);
+    }
+    LS.result();
+    return sorted;
+}
+
+/////////////////////////////////////////////////////////////
+//
+// Given a sortedpairs vector, return the (key, value) pairs
+// one by one.  The first element of the vector is used as a
+// counter to keep track of our position.
+//
+/////////////////////////////////////////////////////////////
+
+LuaDefine(table_nextsortedpair, "sortedpairs,dummy", "next function used by sortedpairs") {
+    if (lua_gettop(L) < 2) {
+        luaL_error(L, "Not enough arguments to nextpair");
+    }
+    luaL_checktype(L, 1, LUA_TTABLE);
+    lua_rawgeti(L, 1, 1);
+    lua_Integer i = luaL_checkinteger(L, -1);
+    lua_pop(L, 1);
+    lua_pushinteger(L, i+1);
+    lua_rawseti(L, 1, 1);
+    lua_rawgeti(L, 1, i*2);
+    if (lua_isnil(L, -1)) {
+        return 1;
+    } else {
+        lua_rawgeti(L, 1, i*2+1);
+        return 2;
+    }
+}
+
+LuaDefine(table_sortedpairs, "table", "iterate over table, sorting all keys") {
+    LuaArg tab;
+    LuaRet closure, rtab, key;
+    LuaStack LS(L, tab, closure, rtab, key);
+    bool sorted = table_getpairs(LS, tab, rtab, true);
+    if (!sorted) {
+        luaL_error(L, "Cannot sort the table keys");
+    }
+    LS.set(closure, lfn_table_nextsortedpair);
+    LS.set(key, LuaNil);
+    return LS.result();
+}
+
+LuaDefine(table_semisortedpairs, "table", "iterate over table, sorting those keys that can be sorted") {
+    LuaArg tab;
+    LuaRet closure, rtab, key;
+    LuaStack LS(L, tab, closure, rtab, key);
+    table_getpairs(LS, tab, rtab, true);
+    LS.set(closure, lfn_table_nextsortedpair);
+    LS.set(key, LuaNil);
+    return LS.result();
+}
+
+LuaDefine(genlt, "obj1,obj2", "return true if obj1 is less than obj2 in general ordering") {
+    LuaArg o1,o2;
+    LuaRet lt;
+    LuaStack LS(L, o1, o2, lt);
+    int ltf = lua_genlt(L, o1.index(), o2.index());
+    LS.set(lt, ltf ? true:false);
+    return LS.result();
+}

luprex/cpp/core/table.hpp

@@ -0,0 +1,28 @@
+////////////////////////////////////////////////////////////
+//
+// This module contains a library of lua functions
+// for manipulating tables.  It also provides a library
+// of useful classes based on tables.
+//
+////////////////////////////////////////////////////////////
+
+
+#ifndef TABLE_HPP
+#define TABLE_HPP
+
+#include "luastack.hpp"
+
+// table_equal
+//
+// True if two tables contain the same key/value pairs.
+//
+bool table_equal(LuaStack &LS0, LuaSlot tab1, LuaSlot tab2);
+
+// table_getpairs
+//
+// Get a table containing the key-value pairs in tab.  Optionally sort
+// the pairs.  Return true if all keys were sortable.
+//
+bool table_getpairs(LuaStack &LS0, LuaSlot tab, LuaSlot pairs, bool sort);
+
+#endif // TABLE_HPP

luprex/cpp/core/textgame.cpp

@@ -0,0 +1,133 @@
+
+#include "wrap-vector.hpp"
+#include "wrap-string.hpp"
+#include "luastack.hpp"
+#include "util.hpp"
+#include "gui.hpp"
+#include "invocation.hpp"
+#include "world.hpp"
+#include "traceback.hpp"
+#include "luaconsole.hpp"
+#include "pprint.hpp"
+#include "printbuffer.hpp"
+#include "drivenengine.hpp"
+
+#include <memory>
+#include <algorithm>
+#include <cstdlib>
+#include <cstring>
+#include <csignal>
+
+class TextGame : public DrivenEngine {
+private:
+    using StringVec = LuaConsole::StringVec;
+    UniqueWorld world_;
+    LuaConsole console_;
+    PrintChanneler print_channeler_;
+    Gui gui_;
+    int64_t actor_id_;
+    
+    void do_luainvoke_command(const StringVec &words) {
+        world_->invoke(Invocation(Invocation::KIND_LUA, actor_id_, actor_id_, words[1]));
+    }
+
+    void do_luaprobe_command(const StringVec &words) {
+        world_->snapshot();
+        stdostream() << world_->probe_lua(actor_id_, words[1]);
+        world_->rollback();
+    }
+
+    void do_syntax_command(const StringVec &words) {
+        stdostream() << "Syntax Error: " << words[1] << std::endl;
+    }
+
+    void do_view_command(const StringVec &cmd) {
+        stdostream() << world_->tangibles_near_debug_string(actor_id_, 100);
+    }
+
+    void do_menu_command(const StringVec &cmd) {
+        int64_t place = sv::to_int64(cmd[1], actor_id_);
+        world_->update_gui(actor_id_, place, &gui_);
+        stdostream() << gui_.menu_debug_string();
+    }
+
+    void do_choose_command(const StringVec &cmd) {
+        eng::string action = gui_.get_action(sv::to_int64(cmd[1]));
+        if (action == "") {
+            stdostream() << "Invalid menu item #" << std::endl;
+            return;
+        }
+        Invocation inv(Invocation::KIND_PLAN, actor_id_, gui_.place(), action);
+        stdostream() << "Invoking: " << inv.debug_string() << std::endl;
+        world_->invoke(inv);
+    }
+
+    void do_tick_command(const StringVec &cmd) {
+        world_->invoke(Invocation(Invocation::KIND_TICK, actor_id_, actor_id_, ""));
+    }
+
+    void do_quit_command(const StringVec &cmd) {
+        actor_id_ = 0;
+    }
+
+    void do_command(const StringVec &words) {
+        if (words.empty()) return;
+        else if (words[0] == "luainvoke") do_luainvoke_command(words);
+        else if (words[0] == "luaprobe") do_luaprobe_command(words);
+        else if (words[0] == "syntax") do_syntax_command(words);
+        else if (words[0] == "view") do_view_command(words);
+        else if (words[0] == "menu") do_menu_command(words);
+        else if (words[0] == "quit") do_quit_command(words);
+        else if (words[0] == "tick") do_tick_command(words);
+        else if (words[0] == "choose") do_choose_command(words);
+        else {
+            stdostream() << "Unsupported command: " << words[0] << std::endl;
+        }
+    }
+
+    void check_redirects() {
+        World::Redirects redir = world_->fetch_redirects();
+        for (const auto &p : redir) {
+            if (p.first == actor_id_) {
+                actor_id_ = p.second;
+                stdostream() << "Login actor ID: " << actor_id_ << std::endl;
+                gui_.clear(0);
+            }
+        }
+    }
+
+    void event_init(int argc, char *argv[])
+    {
+        world_.reset(new World(util::WORLD_TYPE_STANDALONE));
+        world_->update_source(get_lua_source());
+        world_->run_unittests();
+        actor_id_ = world_->create_login_actor();
+        stdostream() << "Login actor ID: " << actor_id_ << std::endl;
+        get_stdio_channel()->set_prompt(console_.get_prompt());
+    }
+
+    void event_update()
+    {
+        world_->update_source(get_lua_source());
+        while (true) {
+            eng::string line = get_stdio_channel()->in()->readline();
+            if (line == "") break;
+            console_.add(line);
+            get_stdio_channel()->set_prompt(console_.get_prompt());
+            do_command(console_.get_command());
+            if (print_channeler_.channel(world_->get_printbuffer(actor_id_), stdostream())) {
+                world_->invoke(print_channeler_.invocation(actor_id_));
+            }
+            check_redirects();
+            if (actor_id_ == 0) {
+                stop_driver();
+                break;
+            }
+        }
+    }
+};
+
+UniqueDrivenEngine make_TextGame() {
+    return UniqueDrivenEngine(new TextGame);
+}
+static DrivenEngineReg reg_TextGame("textgame", make_TextGame);

luprex/cpp/core/textgame.hpp

@@ -0,0 +1,9 @@
+
+#ifndef TEXTGAME_HPP
+#define TEXTGAME_HPP
+
+#include "drivenengine.hpp"
+
+UniqueDrivenEngine make_TextGame();
+
+#endif // TEXTGAME_HPP

luprex/cpp/core/traceback.cpp

@@ -0,0 +1,97 @@
+#include "traceback.hpp"
+
+#include <cstring>
+#include <cassert>
+
+#define TRACEBACK_LEVELS1 12
+#define TRACEBACK_LEVELS2 10
+
+// Call this with the error message on top of the stack.
+// The error message is replaced with a traceback.
+//
+int traceback_coroutine(lua_State *L) {
+    lua_checkstack(L, 20);
+    int top = lua_gettop(L);
+
+    // Convert message to a string
+    if (!lua_tostring(L, top)) {
+        luaL_callmeta(L, top, "__tostring");
+        // If callmeta didn't produce exactly one string, clear the stack
+        // and push "unknown error"
+        if ((lua_gettop(L) == top + 1) && (lua_tostring(L, -1))) {
+            lua_remove(L, top);
+        } else {
+            lua_settop(L, top - 1);
+            lua_pushstring(L, "unknown error");
+        }
+    }
+
+    // Append the traceback.
+    lua_Debug ar;
+    int firstpart = 1;
+    bool any = false;
+    for (int level = 0; lua_getstack(L, level, &ar); level++) {
+        if (level > TRACEBACK_LEVELS1 && firstpart) {
+            /* no more than `LEVELS2' more levels? */
+            if (!lua_getstack(L, level + TRACEBACK_LEVELS2, &ar))
+                level--; /* keep going */
+            else {
+                lua_pushliteral(L, "\n\t...");                 /* too many levels */
+                while (lua_getstack(L, level + TRACEBACK_LEVELS2, &ar)) /* find last levels */
+                    level++;
+            }
+            firstpart = 0;
+            continue;
+        }
+        lua_getinfo(L, "Snl", &ar);
+        if ((!any) && (*ar.what == 'C') && (ar.name != 0)) {
+            if (strcmp(ar.name, "__newindex") == 0) continue;
+        }
+        if ((ar.currentline > 0) || (*ar.namewhat != 0) || (*ar.what != 'C')) {
+            any = true;
+            lua_pushliteral(L, "\n\t");
+            lua_pushfstring(L, "%s:", ar.short_src);
+            if (ar.currentline > 0)
+                lua_pushfstring(L, "%d:", ar.currentline);
+            if (*ar.namewhat != '\0') /* is there a name? */
+                lua_pushfstring(L, " in function " LUA_QS, ar.name);
+            else {
+                if (*ar.what == 'm') /* main? */
+                    lua_pushfstring(L, " in main chunk");
+                else if (*ar.what == 'C' || *ar.what == 't')
+                    lua_pushliteral(L, " ?"); /* C function or tail call */
+                else
+                    lua_pushfstring(L, " in function <%s:%d>",
+                                    ar.short_src, ar.linedefined);
+            }
+            if (1 + lua_gettop(L) - top > 5) {
+                lua_concat(L, 1 + lua_gettop(L) - top);
+            }
+        }
+    }
+    lua_pushstring(L, "\n");
+    if (1 + lua_gettop(L) - top > 1) {
+        lua_concat(L, 1 + lua_gettop(L) - top);
+    }
+    return 1;
+}
+
+
+eng::string traceback_pcall(lua_State *L, int narg, int nret) {
+    int status;
+    int base = lua_gettop(L) - narg; /* function index */
+    lua_pushcfunction(L, traceback_coroutine); /* push traceback function */
+    lua_insert(L, base);             /* put it under chunk and args */
+    status = lua_pcall(L, narg, nret, base);
+    lua_remove(L, base); /* remove traceback function */
+    if (status != LUA_OK) {
+        const char *msg = lua_tostring(L, -1);
+        if ((msg == NULL) || (msg[0] == 0)) {
+            msg = "unknown error";
+        }
+        eng::string result = msg;
+        assert(result != "attempt to yield from outside a coroutine");
+        return result;
+    }
+    return "";
+}

luprex/cpp/core/traceback.hpp

@@ -0,0 +1,33 @@
+/////////////////////////////////////////////////////////////////
+//
+// TRACEBACK ROUTINES
+//
+// The following routines are meant to help produce good-quality
+// tracebacks from errors in lua code.
+//
+/////////////////////////////////////////////////////////////////
+
+
+#ifndef TRACEBACK_HPP
+#define TRACEBACK_HPP
+
+#include "luastack.hpp"
+
+// traceback_coroutine
+//
+// Given a coroutine which contains an error message, replace
+// the error message with a full traceback.  Always returns 1.
+//
+int traceback_coroutine(lua_State *L);
+
+// traceback_pcall
+//
+// Similar to lua_pcall, except that it automatically supplies 
+// traceback_coroutine as a message handler.  It also automatically
+// returns any error message.  Returns empty string if there's
+// no error.  Also checks for a yield inside a pcall, which is 
+// not allowed, and does an assert-fail in that case.
+//
+eng::string traceback_pcall(lua_State *L, int narg, int nret);
+
+#endif // TRACEBACK_HPP

luprex/cpp/core/util.cpp

@@ -0,0 +1,864 @@
+#include "wrap-string.hpp"
+#include "wrap-vector.hpp"
+#include "util.hpp"
+#include "fast-float.hpp"
+
+
+#include <algorithm>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <iomanip>
+#include <cassert>
+#include <cstdlib>
+#include <cmath>
+#include <charconv>
+
+
+#ifdef WIN32
+#endif
+
+#ifndef WIN32
+#include <time.h>
+#include <unistd.h>
+#endif
+
+namespace sv {
+
+bool case_insensitive_eq(string_view s1, string_view s2) {
+    if (s1.size() != s2.size()) return false;
+    for (int i = 0; i < int(s1.size()); i++) {
+        char c1 = s1[i];
+        char c2 = s2[i];
+        if (ascii_isupper(c1)) c1 += 'a'-'A';
+        if (ascii_isupper(c2)) c2 += 'a'-'A';
+        if (c1 != c2) return false;
+    }
+    return true;
+}
+
+bool valid_int64(string_view value) {
+    int64_t result;
+    const char *last = value.data() + value.size();
+    auto r = std::from_chars(value.data(), last, result, 10);
+    if (r.ec != std::errc()) return false;
+    if (r.ptr != last) return false;
+    return true;
+}
+
+bool valid_hex64(string_view value) {
+    int64_t result;
+    const char *last = value.data() + value.size();
+    auto r = std::from_chars(value.data(), last, result, 16);
+    if (r.ec != std::errc()) return false;
+    if (r.ptr != last) return false;
+    return true;
+}
+
+bool valid_double(string_view value) {
+    double result;
+    const char *last = value.data() + value.size();
+    auto r = fast_float::from_chars(value.data(), last, result);
+    if (r.ec != std::errc()) return false;
+    if (r.ptr != last) return false;
+    return true;
+}
+
+int64_t to_int64(string_view value, int64_t errval) {
+    int64_t result;
+    const char *p = value.data();
+    const char *last = p + value.size();
+    if ((p < last) && (*p == '+')) p++;
+    auto r = std::from_chars(p, last, result, 10);
+    if (r.ec != std::errc()) return errval;
+    if (r.ptr != last) return errval;
+    return result;
+}
+
+uint64_t to_hex64(string_view value, uint64_t errval) {
+    uint64_t result;
+    if (sv::zfront(value) == '-') return errval;
+    const char *last = value.data() + value.size();
+    auto r = std::from_chars(value.data(), last, result, 16);
+    if (r.ec != std::errc()) return errval;
+    if (r.ptr != last) return errval;
+    return result;
+}
+
+double to_double(string_view value, double errval) {
+    double result;
+    const char *last = value.data() + value.size();
+    auto r = fast_float::from_chars(value.data(), last, result);
+    if (r.ec != std::errc()) return errval;
+    if (r.ptr != last) return errval;
+    return result;
+}
+
+string_view ltrim(string_view v) {
+    while ((!v.empty()) && (ascii_isspace(v.front()))) {
+        v.remove_prefix(1);
+    }
+    return v;
+}
+
+string_view rtrim(string_view v) {
+    while ((!v.empty()) && (ascii_isspace(v.back()))) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+string_view trim(string_view v) {
+    while ((!v.empty()) && (ascii_isspace(v.front()))) {
+        v.remove_prefix(1);
+    }
+    while ((!v.empty()) && (ascii_isspace(v.back()))) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+string_view ltrim(string_view v, char c) {
+    while ((!v.empty()) && (v.front() == c)) {
+        v.remove_prefix(1);
+    }
+    return v;
+}
+
+string_view rtrim(string_view v, char c) {
+    while ((!v.empty()) && (v.back() == c)) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+string_view trim(string_view v, char c) {
+    while ((!v.empty()) && (v.front() == c)) {
+        v.remove_prefix(1);
+    }
+    while ((!v.empty()) && (v.back() == c)) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+bool has_prefix(string_view s, string_view prefix) {
+    return 0 == s.compare(0, prefix.size(), prefix);
+}
+
+bool has_suffix(string_view s, string_view suffix) {
+    if (s.length() >= suffix.length()) {
+        return (0 == s.compare (s.length() - suffix.length(), suffix.length(), suffix));
+    } else {
+        return false;
+    }
+}
+
+int common_prefix_length(string_view a, string_view b) {
+    int minlen = std::min(a.size(), b.size());
+    for (int i = 0; i < minlen; i++) {
+        if (a[i] != b[i]) return i;
+    }
+    return minlen;
+}
+
+bool is_lua_id(string_view str) {
+    if (str.size() == 0) return false;
+    char c=str[0];
+    if ((!ascii_isalpha(c)) && (c!='_')) return false;
+    for (int i = 1; i < int(str.size()); i++) {
+        char c = str[i];
+        if ((!ascii_isalpha(c)) && (!ascii_isdigit(c)) && (c!='_')) return false;
+    }
+    return true;
+}
+
+bool is_lua_comment(string_view s) {
+    int start = 0;
+    while ((start < int(s.size())) && ((s[start]==' ') || (s[start]=='\t'))) start++;
+    return s.substr(start, 2) == "--";
+}
+
+string_view read_to_sep(string_view &source, char sep) {
+    size_t pos = source.find(sep);
+    string_view result;
+    if (pos == string_view::npos) {
+        result = source;
+        source = string_view();
+    } else {
+        result = source.substr(0, pos);
+        source = source.substr(pos + 1);
+    }
+    return result;
+}
+
+string_view read_to_line(string_view &source) {
+    size_t pos = source.find('\n');
+    string_view result;
+    if (pos == string_view::npos) {
+        result = source;
+        source = string_view();
+    } else {
+        result = source.substr(0, pos);
+        source = source.substr(pos + 1);
+    }
+    if ((!result.empty()) && (result.back() == '\r')) {
+        result.remove_suffix(1);
+    }
+    return result;
+}
+
+bool read_prefix(string_view &source, string_view prefix) {
+    if (0 == source.compare(0, prefix.size(), prefix)) {
+        source.remove_prefix(prefix.size());
+        return true;
+    } else {
+        return false;
+    }
+}
+
+string_view read_to_space(string_view &source) {
+    size_t pos1 = 0;
+    while ((pos1 < source.size()) && (!ascii_isspace(source[pos1]))) {
+        pos1 += 1;
+    }
+    string_view result = source.substr(0, pos1);
+    if (pos1 == source.size()) {
+        source = string_view();
+        return result;
+    }
+    size_t pos2 = pos1 + 1;
+    while ((pos2 < source.size()) && (ascii_isspace(source[pos2]))) {
+        pos2 += 1;
+    }
+    source = source.substr(pos2);
+    return result;
+}
+
+string_view read_nbytes(string_view &source, int nbytes) {
+    if (nbytes < 0) nbytes = 0;
+    if (nbytes > int(source.size())) nbytes = source.size();
+    string_view result = source.substr(0, nbytes);
+    source = source.substr(nbytes);
+    return result;
+}
+
+string_view read_ascii_identifier(string_view &source) {
+    size_t len = 0;
+    if ((len < source.size()) && (sv::ascii_isalpha(source[len]))) {
+        len += 1;
+        while ((len < source.size()) && (sv::ascii_isalnum(source[len]))) {
+            len += 1;
+        }
+    }
+    string_view result = source.substr(0, len);
+    source.remove_prefix(len);
+    return result;
+}
+
+std::string_view read_number(string_view &source, bool plus, bool minus, bool dec, bool exp) {
+    const char *p = source.data();
+    const char *l = p + source.size();
+    if (p == l) return source.substr(0, 0);
+    char sign = *p;
+    if (sign == '+') {
+        if (!plus) return source.substr(0, 0);
+        p++;
+    }
+    if (sign == '-') {
+        if (!minus) return source.substr(0, 0);
+        p++;
+    }
+    if (p == l) return source.substr(0, 0);
+    bool have_digits = false;
+    while ((p < l) && (ascii_isdigit(*p))) {
+        have_digits = true;
+        p++;
+    }
+    if ((p < l) && dec && (*p == '.')) {
+        p++;
+        while ((p < l) && (ascii_isdigit(*p))) {
+            have_digits = true;
+            p++;
+        }
+    }
+    if (!have_digits) return source.substr(0, 0);
+    if ((p < l) && exp && ((*p == 'e')||(*p == 'E'))) {
+        p++;
+        if ((p < l) && ((*p == '+') || (*p == '-'))) {
+            p++;
+        }
+        bool have_exp = false;
+        while ((p < l) && (ascii_isdigit(*p))) {
+            have_exp = true;
+            p++;
+        }
+        if (!have_exp) return source.substr(0, 0);
+    }
+    string_view result = source.substr(0, p - source.data());
+    source.remove_prefix(result.size());
+    return result;
+}
+
+int32_t read_ascii_char(string_view &source) {
+    if (source.empty()) return -1;
+    int32_t result = source.front();
+    source.remove_prefix(1);
+    return result;
+}
+
+int32_t read_codepoint_utf8(string_view &source) {
+    size_t size = source.size();
+    if (size == 0) return -1;
+    const unsigned char *bytes = (const unsigned char *)source.data();
+    int codepoint;
+    size_t seqlen;
+    if ((bytes[0] & 0x80) == 0x00) {
+        // U+0000 to U+007F
+        codepoint = (bytes[0] & 0x7F);
+        seqlen = 1;
+    } else if ((bytes[0] & 0xE0) == 0xC0) {
+        // U+0080 to U+07FF
+        codepoint = (bytes[0] & 0x1F);
+        seqlen = 2;
+    } else if ((bytes[0] & 0xF0) == 0xE0) {
+        // U+0800 to U+FFFF
+        codepoint = (bytes[0] & 0x0F);
+        seqlen = 3;
+    } else if ((bytes[0] & 0xF8) == 0xF0) {
+        // U+10000 to U+10FFFF
+        codepoint = (bytes[0] & 0x07);
+        seqlen = 4;
+    } else {
+        return -1;
+    }
+
+    if (seqlen > size) {
+        return -1;
+    }
+
+    for (size_t i = 1; i < seqlen; ++i) {
+        if ((bytes[i] & 0xC0) != 0x80) return -1;
+        codepoint = (codepoint << 6) | (bytes[i] & 0x3F);
+    }
+
+    if ((codepoint > 0x10FFFF) ||
+        ((codepoint >= 0xD800) && (codepoint <= 0xDFFF)) ||
+        ((codepoint <= 0x007F) && (seqlen != 1)) ||
+        ((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) ||
+        ((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) ||
+        ((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) {
+        return -1;
+    }
+
+    source.remove_prefix(seqlen);
+    return codepoint;
+}
+
+bool valid_utf8(string_view s)
+{
+    while (!s.empty()) {
+        int32_t codepoint = read_codepoint_utf8(s);
+        if (codepoint < 0) return false;
+    }
+    return true;
+}
+
+bool valid_number(string_view s, bool plus, bool minus, bool dec, bool exp) {
+    read_number(s, plus, minus, dec, exp);
+    return s.empty();
+}
+
+} // namespace sv
+
+
+namespace util {
+
+eng::string ascii_tolower(std::string_view s) {
+    eng::string mod(s);
+    for (int i = 0; i < int(mod.size()); i++) {
+        if (sv::ascii_isupper(mod[i])) {
+            mod[i] += 'a' - 'A';
+        }
+    }
+    return mod;
+}
+
+eng::string ascii_toupper(std::string_view s) {
+    eng::string mod(s);
+    for (int i = 0; i < int(mod.size()); i++) {
+        if (sv::ascii_islower(mod[i])) {
+            mod[i] += 'A' - 'a';
+        }
+    }
+    return mod;
+}
+
+void quote_string(const eng::string &s, std::ostream *os) {
+    bool anysq = false;
+    bool anydq = false;
+    for (char c : s) {
+        if (c == '\'') anysq = true;
+        if (c == '"') anydq = true;
+    }
+    bool usesinglequote = (!anysq)||(anydq);
+    (*os) << (usesinglequote ? '\'' : '"');
+    for (char c : s) {
+        if (c >= 32) {
+            if (c == '"') {
+                (*os) << (usesinglequote ? "\"" : "\\\"");
+            } else if (c == '\'') {
+                (*os) << (usesinglequote ? "\\'" : "'");
+            } else if (c == '\\') {
+                (*os) << "\\\\"; 
+            } else {
+                (*os) << c;
+            }
+        } else {
+            unsigned int value = ((unsigned char)c);
+            switch (c) {
+            case '\n': (*os) << "\\n"; break;
+            case '\t': (*os) << "\\t"; break;
+            case '\r': (*os) << "\\r"; break;
+            default:
+                (*os) << "\\" << dec.width(3).fill('0').val(value);
+                break;
+            }
+        }
+    }
+    (*os) << (usesinglequote ? '\'' : '"');
+}
+
+void base64_encode(std::string_view str, std::ostream *oss) {
+    const char *encode_tab = 
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+    const char *s = str.data();
+    size_t size = str.size();
+    for (size_t i = 0; i < size; i += 3) {
+        uint32_t block = ((unsigned char)(s[i])) << 16;
+        if (i + 1 < size) block |= ((unsigned char)(s[i + 1])) << 8;
+        if (i + 2 < size) block |= ((unsigned char)(s[i + 2]));
+        (*oss) << encode_tab[(block>>18)&0x3F];
+        (*oss) << encode_tab[(block>>12)&0x3F];
+        (*oss) << ((i + 1 < size) ? encode_tab[(block>>6)&0x3F] : '=');
+        (*oss) << ((i + 2 < size) ? encode_tab[(block>>0)&0x3F] : '=');
+    }
+}
+
+bool base64_decode(std::string_view str, std::ostream *oss) {
+    uint32_t chunk = 0;
+    int fill = 0;
+    int skip = 0;
+    bool clean = true;
+    for (int i = 0; i < int(str.size()); i++) {
+        char c = str[i];
+        uint32_t value;
+
+        if      ((c >= 'A') && (c <= 'Z')) value = c - 'A';
+        else if ((c >= 'a') && (c <= 'z')) value = c - 'a' + 26;
+        else if ((c >= '0') && (c <= '9')) value = c - '0' + 52;
+        else if (c == '+') value = 62;
+        else if (c == '/') value = 63;
+        else if (c == '=') { value = 0; skip ++; }
+        else { clean=false; continue; }
+
+        chunk = (chunk << 6) | value;
+        fill ++;
+        if (fill == 4) {
+            oss->put((chunk>>16) & 0xFF);
+            if (skip < 2) oss->put((chunk>>8) & 0xFF);
+            if (skip < 1) oss->put(chunk & 0xFF);
+            chunk = 0; fill = 0; skip = 0;
+        }
+    }
+    if (fill != 0) clean = false;
+    return clean;
+}
+
+IdVector id_vector_create(int64_t id1, int64_t id2, int64_t id3, int64_t id4) {
+    IdVector result;
+    if (id1 >= 0) result.push_back(id1);
+    if (id2 >= 0) result.push_back(id2);
+    if (id3 >= 0) result.push_back(id3);
+    if (id4 >= 0) result.push_back(id4);
+    return result;
+}
+
+void print_id_vector(const IdVector &idv, std::ostream *os) {
+    bool first = true;
+    for (int64_t id : idv) {
+        if (!first) (*os) << ",";
+        (*os) << id;
+        first = false;
+    }
+}
+    
+eng::string id_vector_debug_string(const IdVector &idv) {
+    eng::ostringstream oss;
+    print_id_vector(idv, &oss);
+    return oss.str();
+}
+
+IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2) {
+    IdVector result(v1.size() + v2.size());
+    int next = 0;
+    for (int64_t id : v1) result[next++] = id;
+    for (int64_t id : v2) result[next++] = id;
+    std::sort(result.begin(), result.end());
+    int64_t prev = -1;
+    int64_t count = 0;
+    for (int64_t id : result) {
+        if (id != prev) {
+            prev = id;
+            result[count++] = id;
+        }
+    }
+    result.resize(count);
+    return result;
+}
+
+HashValue hash_string(const eng::string &s) {
+    uint64_t hash1 = 0;
+    uint64_t hash2 = 0;
+    SpookyHash::ChainHash128(s.c_str(), s.size(), &hash1, &hash2);
+    return util::HashValue(hash1, hash2);
+}
+
+HashValue hash_id_vector(const IdVector &idv) {
+    uint64_t hash1 = 0;
+    uint64_t hash2 = 0;
+    SpookyHash::ChainHash128(&idv[0], idv.size() * sizeof(int64_t), &hash1, &hash2);
+    return util::HashValue(hash1, hash2);
+}
+
+eng::string hash_to_hex(const HashValue &hv) {
+    eng::ostringstream oss;
+    oss << hex64.val(hv.first) << hex64.val(hv.second);
+    return oss.str();    
+}
+static inline uint64_t Rot64(uint64_t x, int k)
+{
+    return (x << k) | (x >> (64 - k));
+}
+
+uint64_t hash_ints(uint64_t a, uint64_t b, uint64_t c, uint64_t d) {
+    uint64_t h0 = c ^ 0xc548cebf3714dbb9;
+    uint64_t h1 = d ^ 0xd23a7edd44383f8d;
+    uint64_t h2 = a ^ 0x7356f92e4b154df7;
+    uint64_t h3 = b ^ 0x55ce09295766838d;
+    
+    h3 ^= h2;  h2 = Rot64(h2,15);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,52);  h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,26);  h1 += h0;
+    h2 ^= h1;  h1 = Rot64(h1,51);  h2 += h1;
+    h3 ^= h2;  h2 = Rot64(h2,28);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,9);   h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,47);  h1 += h0;
+    h2 ^= h1;  h1 = Rot64(h1,54);  h2 += h1;
+    h3 ^= h2;  h2 = Rot64(h2,32);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,25);  h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,63);  h1 += h0;
+
+    return h1;
+}
+
+double hash_to_double(uint64_t hash) {
+    return (hash >> (64-53)) * 0x1p-53;
+}
+
+StringVec split(const eng::string &s, char sep) {
+    StringVec result;
+    int start = 0;
+    for (int i = 0; i < int(s.size()); i++) {
+        if (s[i] == sep) {
+            result.push_back(s.substr(start, i-start));
+            start = i+1;
+        }
+    }
+    if (start < int(s.size())) {
+        result.push_back(s.substr(start));
+    }
+    return result;
+}
+
+static eng::string substr_nocr(const eng::string &s, int start, int len) {
+    if ((len > 0) && (s[start + len - 1] == '\r')) {
+        len -= 1;
+    }
+    return s.substr(start, len);
+}
+
+StringVec split_lines(const eng::string &s) {
+    StringVec result;
+    int start = 0;
+    for (int i = 0; i < int(s.size()); i++) {
+        if (s[i]=='\n') {
+            result.push_back(substr_nocr(s, start, i-start));
+            start = i + 1;
+        }
+    }
+    if (start < int(s.size())) {
+        result.push_back(substr_nocr(s, start, s.size()-start));
+    }
+    return result;
+}
+
+StringVec split_docstring(const eng::string &s) {
+    StringVec result;
+    int start = 0;
+    for (int i = 0; i < int(s.size()); i++) {
+        if (s[i]=='|') {
+            int len = i-start;
+            if ((len > 0)||(start > 0)) {
+                result.push_back(s.substr(start, i-start));
+            }
+            start = i + 1;
+        }
+    }
+    if (start < int(s.size())) {
+        result.push_back(s.substr(start, s.size()-start));
+    }
+    return result;
+}
+
+eng::string join(const StringVec &strs, const eng::string &sep) {
+    if (strs.empty()) return "";
+    eng::ostringstream oss;
+    oss << strs[0];
+    for (int i = 1; i < int(strs.size()); i++) {
+        oss << sep << strs[i];
+    }
+    return oss.str();
+}
+
+eng::string repeat_string(const eng::string &a, int n) {
+    int len = a.size();
+    eng::string result(len * n, ' ');
+    for (int i = 0; i < n; i++) {
+        for (int j = 0; j < len; j++) {
+            result[i*len + j] = a[j];
+        }
+    }
+    return result;
+}
+    
+eng::string tolower(eng::string input) {
+    for (int i = 0; i < int(input.size()); i++) {
+        input[i] = std::tolower(input[i]);
+    }
+    return input;
+}
+
+eng::string toupper(eng::string input) {
+    for (int i = 0; i < int(input.size()); i++) {
+        input[i] = std::toupper(input[i]);
+    }
+    return input;
+}
+
+static void buffer_codepoint_utf8(int32_t scp, char *buffer) {
+    uint32_t cp = (uint32_t)scp;
+    unsigned char *c = (unsigned char *)buffer;
+    if (cp <= 0x7F) {
+        c[0] = cp;
+        c[1] = 0;
+    }
+    else if (cp <= 0x7FF) {
+        c[0] = (cp>>6)+192;
+        c[1] = (cp&63)+128;
+        c[2] = 0;
+    }
+    else if (cp <= 0xFFFF) {
+        if (0xd800 <= cp && cp <= 0xdfff) {
+            c[0] = 0;
+        } else {
+            c[0] = (cp>>12)+224;
+            c[1] = ((cp>>6)&63)+128;
+            c[2] = (cp&63)+128;
+            c[3] = 0;
+        }
+    }
+    else if (cp <= 0x10FFFF) {
+        c[0] = (cp>>18)+240;
+        c[1] = ((cp>>12)&63)+128;
+        c[2] = ((cp>>6)&63)+128;
+        c[3] = (cp&63)+128;
+        c[4] = 0;
+    } else {
+        c[0] = 0;
+    }
+}
+
+eng::string get_codepoint_utf8(uint32_t cp) {
+    char buffer[5];
+    buffer_codepoint_utf8(cp, buffer);
+    return eng::string(buffer);
+}
+
+bool write_codepoint_utf8(int32_t cp, std::ostream *s) {
+    char buffer[5];
+    buffer_codepoint_utf8(cp, buffer);
+    (*s) << buffer;
+    return buffer[0] != 0;
+}
+
+double distance_squared(double x1, double y1, double x2, double y2) {
+    double dx = x1 - x2;
+    double dy = y1 - y2;
+    return dx*dx + dy*dy;
+}
+
+LuaSourcePtr make_lua_source(const eng::string &code) {
+    LuaSourcePtr result(new LuaSourceVec);
+    eng::string fn = "file.lua";
+    result->push_back(std::make_pair(fn, code));
+    return result;
+}
+
+eng::string XYZ::debug_string() const {
+    eng::ostringstream oss;
+    oss << "(" << x << "," << y << "," << z << ")";
+    return oss.str();
+}
+
+} // namespace util
+
+
+static std::string_view read_number_x(const char *p, bool plus, bool minus, bool dec, bool exp) {
+    std::string_view source = p;
+    return sv::read_number(source, plus, minus, dec, exp);
+}
+
+LuaDefine(unittests_util, "", "some unit tests") {
+        // test str_to_int64, str_to_double
+    LuaAssert(L, sv::to_int64("123") == 123);
+    LuaAssert(L, sv::to_int64("123.4") == INT64_MAX);
+    LuaAssert(L, sv::to_int64("12ab") == INT64_MAX);
+    LuaAssert(L, sv::to_int64("") == INT64_MAX);
+    LuaAssert(L, sv::to_double("123.5") == 123.5);
+    LuaAssert(L, std::isnan(sv::to_double("12ab")));
+    LuaAssert(L, std::isnan(sv::to_double("")));
+    
+    // Test trim, ltrim, rtrim
+    LuaAssert(L, sv::ltrim("  foo  ") == "foo  ");
+    LuaAssert(L, sv::rtrim("  foo  ") == "  foo");
+    LuaAssert(L, sv::trim("  foo  ") == "foo");
+    LuaAssert(L, sv::trim("foo") == "foo");
+    LuaAssert(L, sv::trim("") == "");
+    LuaAssert(L, sv::ltrim("**foo**", '*') == "foo**");
+    LuaAssert(L, sv::rtrim("**foo**", '*') == "**foo");
+    LuaAssert(L, sv::trim("**foo**", '*') == "foo");
+    LuaAssert(L, sv::trim("foo", '*') == "foo");
+    LuaAssert(L, sv::trim("", '*') == "");
+    
+    // Test read_to_line
+    std::string_view v = "foo\nbar\r\nbaz";
+    
+    std::string_view v1 = sv::read_to_line(v);
+    LuaAssertStrEq(L, v1, "foo");
+    LuaAssertStrEq(L, v, "bar\r\nbaz");
+
+    std::string_view v2 = sv::read_to_line(v);
+    LuaAssertStrEq(L, v2, "bar");
+    LuaAssertStrEq(L, v, "baz");
+
+    std::string_view v3 = sv::read_to_line(v);
+    LuaAssertStrEq(L, v3, "baz");
+    LuaAssert(L, sv::isnull(v));
+
+    // Test read_to_space
+    v = "foo  bar  baz";
+
+    std::string_view s1 = sv::read_to_space(v);
+    LuaAssertStrEq(L, s1, "foo");
+    LuaAssertStrEq(L, v, "bar  baz");
+
+    std::string_view s2 = sv::read_to_space(v);
+    LuaAssertStrEq(L, s2, "bar");
+    LuaAssertStrEq(L, v, "baz");
+
+    std::string_view s3 = sv::read_to_space(v);
+    LuaAssertStrEq(L, s3, "baz");
+    LuaAssert(L, sv::isnull(v));
+
+    // Test the unioning of ID vectors.
+    util::IdVector idv1,idv2;
+    idv1.push_back(1);
+    idv1.push_back(6);
+    idv1.push_back(4);
+    idv2.push_back(5);
+    idv2.push_back(1);
+    idv2.push_back(6);
+    util::IdVector joined = util::sort_union_id_vectors(idv1, idv2);
+    LuaAssert(L, joined.size() == 4);
+    LuaAssert(L, joined[0] == 1);
+    LuaAssert(L, joined[1] == 4);
+    LuaAssert(L, joined[2] == 5);
+    LuaAssert(L, joined[3] == 6);
+
+    // Test the string split routine.
+    util::StringVec sv1 = util::split("foo,bar,baz", ',');
+    LuaAssert(L, sv1.size() == 3);
+    LuaAssert(L, sv1[0] == "foo");
+    LuaAssert(L, sv1[1] == "bar");
+    LuaAssert(L, sv1[2] == "baz");
+    util::StringVec sv2 = util::split(",foo,,bar", ',');
+    LuaAssert(L, sv2.size() == 4);
+    LuaAssert(L, sv2[0]=="");
+    LuaAssert(L, sv2[1]=="foo");
+    LuaAssert(L, sv2[2]=="");
+    LuaAssert(L, sv2[3]=="bar");
+
+    // Test the split_lines routine.
+    util::StringVec sv3 = util::split_lines("foo\n\nbar\r\nbaz\r\n\r\n");
+    LuaAssert(L, sv3.size() == 5);
+    LuaAssert(L, sv3[0] == "foo");
+    LuaAssert(L, sv3[1] == "");
+    LuaAssert(L, sv3[2] == "bar");
+    LuaAssert(L, sv3[3] == "baz");
+    LuaAssert(L, sv3[4] == "");
+
+    // Test the repeat string routine.
+    LuaAssertStrEq(L, util::repeat_string("abc", 3), "abcabcabc");
+
+    // test toupper and tolower
+    LuaAssert(L, util::toupper("fooBar") == "FOOBAR");
+    LuaAssert(L, util::tolower("fooBar") == "foobar");
+
+    // Test distance_squared
+    LuaAssert(L, util::distance_squared(1, 1, 5, 4) == 25.0);
+    LuaAssert(L, util::distance_squared(5, 4, 1, 1) == 25.0);
+
+    // Test XYZ.
+    util::XYZ xyza(3,4,5), xyzb(3,4,5), xyzc(3,4,6);
+    LuaAssert(L, xyza.x == 3);
+    LuaAssert(L, xyza.y == 4);
+    LuaAssert(L, xyza.z == 5);
+    LuaAssert(L, xyza == xyzb);
+    LuaAssert(L, xyza != xyzc);
+    LuaAssert(L, xyza.debug_string() == "(3,4,5)");
+    
+    // Test hash_to_string
+    LuaAssertStrEq(L, util::hash_to_hex(util::HashValue(0x1234,0x789a)),
+        "0000000000001234000000000000789a");
+    
+    // Test hash_to_double
+    LuaAssert(L, util::hash_to_double(0x1000000000000000) == 1.0/16.0);
+    LuaAssert(L, util::hash_to_double(0x7000000000000000) == 7.0/16.0);
+    LuaAssert(L, util::hash_to_double(0xF000000000000000) == 15.0/16.0);
+
+    // Test read_number allowing everything.
+    LuaAssert(L, read_number_x("123x", true, true, true, true) == "123");
+    LuaAssert(L, read_number_x("123.3x", true, true, true, true) == "123.3");
+    LuaAssert(L, read_number_x("123.x", true, true, true, true) == "123.");
+    LuaAssert(L, read_number_x("123..x", true, true, true, true) == "123.");
+    LuaAssert(L, read_number_x("-123x", true, true, true, true) == "-123");
+    LuaAssert(L, read_number_x("+123x", true, true, true, true) == "+123");
+    LuaAssert(L, read_number_x("+-123x", true, true, true, true) == "");
+    LuaAssert(L, read_number_x("-123.02e05x", true, true, true, true) == "-123.02e05");
+    LuaAssert(L, read_number_x("-123e-5x", true, true, true, true) == "-123e-5");
+    LuaAssert(L, read_number_x("-123e+5x", true, true, true, true) == "-123e+5");
+    LuaAssert(L, read_number_x("-123e+x", true, true, true, true) == "");
+    
+    return 0;
+}
+

luprex/cpp/core/util.hpp

@@ -0,0 +1,412 @@
+///////////////////////////////////////////////////////////////////////
+//
+// NAMESPACE SV
+//
+//  * Operate on string_view or just characters.
+//  * Do not allocate memory.
+//  * Do not copy strings.
+//
+// NAMESPACE UTIL
+//
+//  * General purpose utility functions.
+//  * Sort of a catch-all.
+//
+///////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_HPP
+#define UTIL_HPP
+
+#include "wrap-string.hpp"
+#include "wrap-set.hpp"
+#include "wrap-map.hpp"
+#include "wrap-vector.hpp"
+#include "wrap-sstream.hpp"
+#include <ostream>
+#include <memory>
+#include <utility>
+#include <algorithm>
+#include <string_view>
+#include <limits>
+#include <iomanip>
+// #include <cstdint>
+
+#include "luastack.hpp"
+#include "spookyv2.hpp"
+
+namespace sv {
+
+// Bring this into our namespace.
+using string_view = std::string_view;
+
+// Test character class, ignoring current locale and unicode issues.
+inline bool ascii_isupper(char c) { return (c >= 'A') && (c <= 'Z'); }
+inline bool ascii_islower(char c) { return (c >= 'a') && (c <= 'z'); }
+inline bool ascii_isdigit(char c) { return (c >= '0') && (c <= '9'); }
+inline bool ascii_isalpha(char c) { return ascii_isupper(c) || ascii_islower(c); }
+inline bool ascii_isalnum(char c) { return ascii_isalpha(c) || ascii_isdigit(c); }
+inline bool ascii_isspace(char c) { return (c==' ')||(c=='\t')||(c=='\r')||(c=='\n')||(c=='\f')||(c=='\v'); }
+
+// Check for the null string_view
+//
+// Note that the null string view is an empty string, 
+// but not every empty string is the null string view.
+//
+inline bool isnull(string_view v) { return v.data() == nullptr; }
+
+// Return true if the two strings are equal, ignoring case.
+//
+bool case_insensitive_eq(std::string_view s1, std::string_view s2);
+
+// Check if numbers can be parsed as int64/double
+bool valid_double(string_view v);
+bool valid_int64(string_view v);
+bool valid_hex64(string_view v);
+
+// Convert strings to numbers.  Returns errval on failure.
+//
+// The integer parser accepts a sequence of digits,
+// with or without a + or - sign.   The hex parser
+// does not allow a + or - sign.  For both the int64
+// and hex64 parser, it is a failure if the number
+// does not fit in 64 bits.  The double parser does
+// not accept the strings 'nan' or 'inf'.
+//
+double to_double(string_view v, double errval = std::numeric_limits<double>::quiet_NaN());
+int64_t to_int64(string_view v, int64_t errval = std::numeric_limits<int64_t>::max());
+uint64_t to_hex64(string_view v, uint64_t errval = std::numeric_limits<uint64_t>::max());
+
+// Trim whitspace from a string_view.
+string_view ltrim(string_view v);
+string_view rtrim(string_view v);
+string_view trim(string_view v);
+
+// Trim specific character (all occurrences) from a string_view.
+string_view ltrim(string_view v, char c);
+string_view rtrim(string_view v, char c);
+string_view trim(string_view v, char c);
+
+// Return true if the string has the specified prefix or suffix.
+bool has_prefix(string_view s, string_view prefix);
+bool has_suffix(string_view s, string_view suffix);
+
+// Return the length of the common prefix of A and B.
+int common_prefix_length(string_view a, string_view b);
+
+// Return true if the string is a lua identifier.
+bool is_lua_id(string_view s);
+
+// Return true if the line of code is a lua comment.
+bool is_lua_comment(string_view s);
+
+// Return the first character, but if the view is empty,
+// return zero.
+inline char zfront(string_view &s) {
+    return s.empty() ? char(0) : s.front();
+}
+
+// Read from a string_view until separator is reached.
+//
+// If the separator appears in the source, returns everything
+// before the separator, and updates the source to everything
+// after the separator.
+//
+// If the separator doesn't appear in the source, returns
+// the entire source, and replaces source with the null string_view.
+//
+string_view read_to_sep(string_view &source, char sep);
+
+// Read from a string_view until newline is reached.
+//
+// If there's a line-break in the source (newline or CRLF),
+// returns the text before the line-break, and updates the
+// source to the text after the line-break.
+//
+// If there's no line-break in the source, returns the entire source,
+// and updates source to the null string_view.
+//
+string_view read_to_line(string_view &source);
+
+// Read a prefix string from a string_view.
+//
+// Returns false if the string view doesn't start with
+// the specified prefix.
+//
+bool read_prefix(string_view &source, string_view prefix); 
+
+// Read from a string_view until whitespace is reached.
+//
+// If there's any whitespace in the source, returns the text
+// before the whitespace, and update the source to the text
+// after the whitespace.
+//
+// If there's no whitespace in the source, returns the entire
+// source, and updates the source to the null string_view.
+//
+string_view read_to_space(string_view &source);
+
+// Read up to nbytes from a string_view.
+//
+string_view read_nbytes(string_view &source, int nbytes);
+
+// Read an ascii identifier from a string_view
+//
+// If there's no valid identifier, returns empty string.
+//
+string_view read_ascii_identifier(string_view &source);
+
+// Read a number from a string view
+//
+// This is basically a regex pattern matching routine
+// hardwired with the regex for numbers.  You must
+// specify which of the following parts of the regex
+// are allowed or not:
+//
+//  * plus sign
+//  * minus sign
+//  * decimal point
+//  * scientific notation exponents
+//
+// Returns the number as a string_view.  There is
+// no guarantee that the number is small enough to
+// fit into any particular number of bits.  This
+// always uses base 10.
+//
+std::string_view read_number(string_view &source, bool plus, bool minus, bool dec, bool exp);
+
+// Read an ascii character from a string.
+//
+// Returns -1 if the string is empty.
+//
+int32_t read_ascii_char(string_view &source);
+
+// Read a UTF8 codepoint from a string_view.
+//
+// If the next thing in the string_view isn't a valid
+// codepoint, returns -1 and doesn't update the view.
+//
+int32_t read_codepoint_utf8(string_view &source);
+
+// Return true if the string is valid utf-8.
+bool valid_utf8(string_view s);
+
+// Return true if the number conforms to the spec.
+// See read_number for more information.
+//
+bool valid_number(string_view v, bool plus, bool minus, bool dec, bool exp);
+
+} // namespace sv
+
+namespace util {
+
+enum WorldType {
+    WORLD_TYPE_STANDALONE,
+    WORLD_TYPE_C_SYNC,
+    WORLD_TYPE_S_SYNC,
+    WORLD_TYPE_MASTER,
+};
+
+enum MessageType {
+    MSG_NULL,
+    MSG_DIFF,
+    MSG_ACK,
+    MSG_INVOKE,
+};
+
+using StringVec = eng::vector<eng::string>;
+using StringPair = std::pair<eng::string, eng::string>;
+using StringSet = eng::set<eng::string>;
+using LuaSourceVec = eng::vector<StringPair>;
+using LuaSourcePtr = std::unique_ptr<LuaSourceVec>;
+using HashValue = std::pair<uint64_t, uint64_t>;
+using IdVector = eng::vector<int64_t>;
+
+// Ascii uppercase and lowercase.
+eng::string ascii_tolower(std::string_view c);
+eng::string ascii_toupper(std::string_view c);
+
+// Output a string to a stream using Lua string escaping and quoting.
+void quote_string(const eng::string &str, std::ostream *os);
+
+// base64 encode.
+void base64_encode(std::string_view v, std::ostream *oss);
+
+// base64 decode.
+//
+// Returns true if the base64 was 'clean' base64, as
+// opposed to base64 with extraneous characters.
+//
+bool base64_decode(std::string_view v, std::ostream *oss);
+
+// ID vector quick create.
+IdVector id_vector_create(int64_t id1=-1, int64_t id2=-1, int64_t id3=-1, int64_t id4=-1);
+
+// Print an ID vector to a stream.
+void print_id_vector(const IdVector &idv, std::ostream *os);
+
+// ID vector debug string.
+eng::string id_vector_debug_string(const IdVector &idv);
+
+// Unions and sorts two ID vectors.
+IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2);
+
+// Get a 128-bit hashvalue for a string.
+HashValue hash_string(const eng::string &str);
+
+// Get a 128-bit hashvalue for an ID vector.
+HashValue hash_id_vector(const IdVector &idv);
+
+// Convert a 128-bit hash to a hexadecimal string.
+eng::string hash_to_hex(const HashValue &hash);
+
+// Hash four integers together to 64 bits.
+// This is a good hash, but not cryptographically good.
+uint64_t hash_ints(uint64_t n1, uint64_t n2, uint64_t n3, uint64_t n4);
+
+// Hash a single 64-bit integer.
+// This is a good hash, but not cryptographically good.
+// Published by David Stafford in his article 'Better Bit Mixing'.
+inline uint64_t hash_int(uint64_t x) {
+    x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9);
+    x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb);
+    x = x ^ (x >> 31);
+    return x;
+}
+
+// Convert a 64-bit hash value into a floating point number between 0 and 1.
+double hash_to_double(uint64_t hash);
+
+// Split a string into multiple strings
+StringVec split(const eng::string &s, char sep);
+
+// Split a string into multiple strings using \r or \n
+StringVec split_lines(const eng::string &s);
+
+// Split a string into multiple lines using |, remove any leading blank line.
+StringVec split_docstring(const eng::string &s);
+
+// Join multiple strings into one string
+eng::string join(const StringVec &strs, eng::string sep);
+
+// Return N repetitions of string A
+eng::string repeat_string(const eng::string &a, int n);
+
+// String to lowercase/uppercase.  Ascii only, no unicode.
+eng::string tolower(eng::string input);
+eng::string toupper(eng::string input);
+
+// Convert a codepoint number into a utf8 string.
+// If the codepoint is invalid, returns empty string.
+eng::string get_codepoint_utf8(int32_t cp);
+
+// Write a codepoint in utf8 to a stream.
+// If the codepoint is invalid, writes nothing and returns false.
+bool write_codepoint_utf8(int32_t cp, std::ostream *out);
+
+// Calculate distance between two points
+double distance_squared(double x1, double y1, double x2, double y2);
+
+// Make a LuaSourceVec with one element, for unit testing.
+LuaSourcePtr make_lua_source(const eng::string &code);
+
+// Remove items from a vector that are nullptr.
+template<class T>
+void remove_nullptrs(T &vec) {
+    auto iter = std::partition(vec.begin(), vec.end(), [] (const auto &x) { return x != nullptr; });
+    vec.erase(iter, vec.end());
+}
+
+// Remove items from a vector that are marked for deletion.
+template<class T>
+void remove_marked_items(T &vec) {
+    auto iter = std::partition(vec.begin(), vec.end(), [] (const auto &x) { return !x.marked_for_deletion(); });
+    vec.erase(iter, vec.end());
+}
+
+// An XYZ coordinate, general purpose.
+struct XYZ {
+    float x, y, z;
+    XYZ() { x=0; y=0; z=0; }
+    XYZ(float ix, float iy, float iz) { x=ix; y=iy; z=iz; }
+    bool operator ==(const XYZ &o) const { return x==o.x && y == o.y && z==o.z; }
+    bool operator !=(const XYZ &o) const { return x!=o.x || y != o.y || z!=o.z; }
+    XYZ operator -(const XYZ &o) const { return XYZ(x-o.x, y-o.y, z-o.z); }
+    XYZ operator +(const XYZ &o) const { return XYZ(x+o.x, y+o.y, z+o.z); }
+    XYZ operator *(float scale) const { return XYZ(x*scale, y*scale, z*scale); }
+    eng::string debug_string() const;
+};
+
+class NullStreamBuffer : public std::streambuf
+{
+public:
+  int overflow(int c) { return c; }
+};
+
+// send_to_stream: send all arguments to the specified stream.
+inline void send_to_stream(std::ostream &os) {}
+template <class ARG, class... REST>
+inline void send_to_stream(std::ostream &os, const ARG &arg, const REST & ... rest) {
+    os << arg;
+    send_to_stream(os, rest...);
+}
+
+// ss: convert all arguments to a string by sending them to a stringstream.
+template <class... ARGS>
+inline eng::string ss(const ARGS & ... args) {
+    eng::ostringstream oss;
+    send_to_stream(oss, args...);
+    return oss.str();
+}
+
+// A better API than std::setfill, std::hex, std::setw, std::setprecision
+//
+// Usage examples:
+//   std::cout << util::hex.width(5).fill('0').val(123)
+//   std::cout << util::dec.fill('$').precision(val(123)
+//
+// The reason that other API is bad is that it can leave std::cout
+// in an unpredictable state.  This API always leaves the stream clean.
+//
+template <class VALUE>
+class FormattedNumber {
+public:
+    VALUE value_;
+    bool hex_;
+    int width_;
+    char fill_;
+    int precision_;
+
+    constexpr FormattedNumber(VALUE v, bool h, int w, char f, int p)
+        : value_(v), hex_(h), width_(w), fill_(f), precision_(p) {}
+
+    constexpr FormattedNumber width(int w) const { return FormattedNumber(value_, hex_, w, fill_, precision_); }
+    constexpr FormattedNumber fill(char f) const { return FormattedNumber(value_, hex_, width_, f, precision_); }
+    constexpr FormattedNumber precision(int p) const { return FormattedNumber(value_, hex_, width_, fill_, p); }
+
+    template <class NVALUE>
+    constexpr FormattedNumber val(NVALUE v) const { return FormattedNumber(v, hex_, width_, fill_, precision_); }
+};
+
+constexpr auto hex = FormattedNumber<int>(0, true, 0, '0', 6);
+constexpr auto hex8 = FormattedNumber<int>(0, true, 2, '0', 6);
+constexpr auto hex16 = FormattedNumber<int>(0, true, 4, '0', 6);
+constexpr auto hex32 = FormattedNumber<int>(0, true, 8, '0', 6);
+constexpr auto hex64 = FormattedNumber<int>(0, true, 16, '0', 6);
+constexpr auto dec = FormattedNumber<int>(0, false, 0, ' ', 6);
+
+} // namespace util
+
+template<class VALUE>
+inline std::ostream &operator<<(std::ostream &oss, util::FormattedNumber<VALUE> n) {
+    if (n.hex_) oss << std::hex;
+    else oss << std::dec;
+    oss << std::setprecision(n.precision_) << std::setfill(n.fill_) << std::setw(n.width_) << n.value_;
+    oss << std::dec << std::setfill(' ') << std::setprecision(6);
+    return oss;
+}
+
+inline std::ostream &operator<<(std::ostream &oss, const util::XYZ &xyz) {
+    oss << xyz.x << "," << xyz.y << "," << xyz.z;
+    return oss;
+}
+
+#endif // UTIL_HPP

luprex/cpp/core/world-accessor.cpp

@@ -0,0 +1,807 @@
+
+#include "world.hpp"
+#include "pprint.hpp"
+#include <cmath>
+#include <iostream>
+
+static void tangible_getall(LuaStack &LS0, LuaSlot list, const util::IdVector &idv) {
+    LuaVar tangibles, tan;
+    LuaStack LS(LS0.state(), tangibles, tan);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    assert(LS.istable(tangibles));
+    LS.set(list, LuaNewTable);
+    int index = 1;
+    for (int64_t id : idv) {
+        LS.rawget(tan, tangibles, id);
+        assert(LS.istable(tan));
+        LS.rawset(list, index++, tan);
+    }
+    LS.result();
+}
+
+LuaDefine(tangible_animstate, "tan", 
+    "|Get the entire animation state of the tangible."
+    "|Returns six values: graphic,plane,x,y,z,facing.") {
+    LuaArg tanobj;
+    LuaRet graphic, plane, x, y, z, facing;
+    LuaStack LS(L, tanobj, graphic, plane, x, y, z, facing);
+    World *w = World::fetch_global_pointer(L);
+    Tangible *tan = w->tangible_get(LS, tanobj);
+    const AnimStep &aqback = tan->anim_queue_.back();
+    LS.set(graphic, aqback.graphic());
+    LS.set(plane, aqback.plane());
+    LS.set(x, aqback.xyz().x);
+    LS.set(y, aqback.xyz().y);
+    LS.set(z, aqback.xyz().z);
+    LS.set(facing, aqback.facing());
+    return LS.result();
+}
+
+LuaDefine(tangible_xyz, "tan", 
+    "|Get the current coordinates of the tangible."
+    "|Returns three values: x, y, z") {
+    LuaArg tanobj;
+    LuaRet x, y, z;
+    LuaStack LS(L, tanobj, x, y, z);
+    World *w = World::fetch_global_pointer(L);
+    Tangible *tan = w->tangible_get(LS, tanobj);
+    const AnimStep &aqback = tan->anim_queue_.back();
+    LS.set(x, aqback.xyz().x);
+    LS.set(y, aqback.xyz().y);
+    LS.set(z, aqback.xyz().z);
+    return LS.result();
+}
+
+LuaDefine(tangible_animate, "tan,configtable",
+    "|Add an animation step to the tangible."
+    "|The configtable is a table containing any of the following:"
+    "|action,graphic,plane,x,y,z,facing") {
+    LuaArg tanobj, config;
+    LuaStack LS(L, tanobj, config);
+    LuaKeywordParser kp(LS, config);
+    World *w = World::fetch_global_pointer(L);
+    Tangible *tan = w->tangible_get(LS, tanobj);
+    int64_t id = w->alloc_id_predictable();
+    AnimStep step;
+    step.configure(kp, tan->anim_queue_.back());
+    kp.final_check_throw();
+    if (step.action() == "") {
+        luaL_error(L, "animation action must be specified");
+    }
+    tan->anim_queue_.add(id, step);
+    tan->update_plane_item();
+    return LS.result();
+}
+
+LuaDefine(tangible_setclass, "tan,class",
+    "|Set the class of the tangible."
+    "|The class can be a 'class table' (ie, a table of methods), "
+    "|or it can be a string that names a class.  The tangible is "
+    "|given an __index metamethod that points at the class table.") {
+    LuaArg tanobj, classname;
+    LuaVar classtab, mt;
+    LuaStack LS(L, tanobj, classname, classtab, mt);
+    World *w = World::fetch_global_pointer(L);
+    w->tangible_get(LS, tanobj);
+    eng::string err = LS.getclass(classtab, classname);
+    if (err != "") {
+        luaL_error(L, "%s", err.c_str());
+    }
+    LS.getmetatable(mt, tanobj);
+    LS.rawset(mt, "__index", classtab);
+    return LS.result();
+}
+
+LuaDefine(tangible_getclass, "tan", 
+    "|Get the class of the tangible, if any."
+    "|The return value is a string, the class name, not"
+    "|the class table.") {
+    LuaArg tanobj;
+    LuaVar mt, classtab;
+    LuaRet classname;
+    LuaStack LS(L, tanobj, mt, classtab, classname);
+    World *w = World::fetch_global_pointer(L);
+    w->tangible_get(LS, tanobj);
+    LS.getmetatable(mt, tanobj);
+    LS.rawget(classtab, mt, "__index");
+    eng::string name = LS.classname(classtab);
+    if (name == "") {
+        LS.set(classname, LuaNil);
+    } else {
+        LS.set(classname, name);
+    }
+    return LS.result();
+}
+
+LuaDefine(tangible_delete, "tan", 
+    "|Delete the specified tangible."
+    "|This cannot be used to delete player tangibles,"
+    "|To delete a player, use tangible.redirect") {
+    LuaArg tanobj;
+    LuaStack LS(L, tanobj);
+    World *w = World::fetch_global_pointer(L);
+    Tangible *tan = w->tangible_get(LS, tanobj);
+    assert(tan != nullptr); // this should be checked above.
+    if (tan->is_an_actor()) {
+        luaL_error(L, "Cannot delete a player using tangible.delete, use tangible.redirect instead.");
+        return 0;
+    }
+    w->tangible_delete(tan->id());
+    return LS.result();
+}
+
+LuaDefine(tangible_build, "config", 
+    "|Build a new tangible object."
+    "|The config table must contain: class,x,y,z,plane,graphic."
+    "|The config table can optionally contain: facing.") {
+    LuaArg config;
+    LuaVar classname, classtab, mt;
+    LuaRet database;
+    LuaStack LS(L, config, classname, classtab, database, mt);
+    LuaKeywordParser kp(LS, config);
+
+    // Get the class of the new tangible.
+    if (!kp.parse(classname, "class")) {
+        luaL_error(L, "You must specify a class for the tangible");
+    }
+    eng::string err = LS.getclass(classtab, classname);
+    if (err != "") {
+        luaL_error(L, "%s", err.c_str());
+    }
+
+    // Parse the initial animation step.
+    AnimStep initstep, blank;
+    initstep.configure(kp, blank);
+    kp.final_check_throw();
+    if (!initstep.has_xyz()) {
+        luaL_error(L, "You must specify (X,Y,Z) for new tangible");
+    }
+    if (!initstep.has_plane()) {
+        luaL_error(L, "You must specify plane for new tangible");
+    }
+    if (!initstep.has_graphic()) {
+        luaL_error(L, "You must specify graphic for new tangible");
+    }
+
+    // TODO: generate error if there's extra crap in the config table.
+
+    World *w = World::fetch_global_pointer(L);
+    int64_t new_id = w->alloc_id_predictable();
+    Tangible *tan = w->tangible_make(L, new_id, "nowhere", true);
+    lua_replace(L, database.index());
+
+    // Update the class of the new tangible.
+    LS.getmetatable(mt, database);
+    LS.rawset(mt, "__index", classtab);
+
+    // Update the animation queue and planemap of the new tangible.
+    int64_t stepid = w->alloc_id_predictable();
+    tan->anim_queue_.add(stepid, initstep);
+    tan->update_plane_item();
+
+    return LS.result();
+}
+
+LuaDefine(tangible_get, "id", 
+    "|Get the tangible with the specified id."
+    "|This is for debugging only and will be removed in"
+    "|the released version.") {
+    LuaArg id;
+    LuaVar tangibles;
+    LuaRet database;
+    LuaStack LS(L, id, tangibles, database);
+    int64_t nid = LS.ckinteger(id);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(database, tangibles, id);
+    if (!LS.istable(database)) {
+        luaL_error(L, "Not a tangible ID: %d", nid);
+    }
+    return LS.result();
+}
+
+LuaDefine(tangible_redirect, "tan1,tan2,bulldozetan1", 
+    "|Redirect is not working yet") {
+    LuaArg actor1, actor2, bldz;
+    LuaStack LS(L, actor1, actor2, bldz);
+    World *w = World::fetch_global_pointer(L);
+    bool bulldoze = LS.ckboolean(bldz);
+    Tangible *tan1 = w->tangible_get(LS, actor1);
+    if (!tan1->is_an_actor()) {
+        luaL_error(L, "redirect source is not an actor");
+    }
+    if (LS.isnil(actor2)) {
+        w->redirects_[tan1->id()] = 0;
+    } else {
+        Tangible *tan2 = w->tangible_get(LS, actor2);
+        tan2->configure_id_pool_for_actor();
+        w->redirects_[tan1->id()] = tan2->id();
+    }
+    if (bulldoze) {
+        w->tangible_delete(tan1->id());
+    }
+    return LS.result();
+}
+
+LuaDefine(tangible_id, "tan",
+    "|Return the tangible's id number."
+    "|This is for debugging only and will be removed"
+    "|in the released version.") {
+    LuaArg tanobj;
+    LuaRet id;
+    LuaStack LS(L, tanobj, id);
+    int64_t tid = LS.tanid(tanobj);
+    if (tid == 0) {
+        luaL_error(L, "Not a tangible");
+    }
+    LS.set(id, tid);
+    return LS.result();
+}
+
+LuaDefine(tangible_actor, "",
+    "|Return the current actor.") {
+    LuaRet actor;
+    LuaVar tangibles;
+    LuaStack LS(L, tangibles, actor);
+    World *w = World::fetch_global_pointer(L);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(actor, tangibles, w->lthread_actor_id_);
+    return LS.result();
+}
+
+LuaDefine(tangible_place, "",
+    "|Return the current place.") {
+    LuaRet place;
+    LuaVar tangibles;
+    LuaStack LS(L, tangibles, place);
+    World *w = World::fetch_global_pointer(L);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(place, tangibles, w->lthread_place_id_);
+    return LS.result();
+}
+
+LuaDefine(tangible_near, "tan,radius,omit_nowhere,omit_self",
+    "|Scan near the specified tangible."
+    "|If omit_nowhere is true, and the tangible is on the nowhere plane,"
+    "|then the scan returns empty.  If omit_self is true, then the "
+    "|tangible passed in is omitted from the results.") {
+    LuaArg ltan, lradius, lomit_nowhere, lomit_self;
+    LuaRet list;
+    LuaStack LS(L, ltan, lradius, lomit_nowhere, lomit_self, list);
+    World *w = World::fetch_global_pointer(L);
+    Tangible *tan = w->tangible_get(LS, ltan);
+    const AnimStep &aqback = tan->anim_queue_.back();
+
+    PlaneScan scan;
+    scan.set_plane(aqback.plane());
+    scan.set_bbox_given_center_radius(aqback.xyz(), LS.cknumber(lradius));
+    scan.set_shape(PlaneScan::SPHERE);
+    scan.set_sorted(true);
+    scan.set_near(tan->id(), !LS.ckboolean(lomit_self));
+    scan.set_omit_nowhere(LS.ckboolean(lomit_nowhere));
+    
+    util::IdVector idv = w->plane_map_.scan(scan);
+    tangible_getall(LS, list, idv);
+    return LS.result();
+}
+
+LuaDefine(tangible_scan, "plane,x,y,radius,omit_nowhere",
+    "|Scan the specified plane."
+    "|If omit_nowhere is true, and the plane is 'nowhere', then"
+    "|the scan returns empty.") {
+    LuaArg lplane, lx, ly, lradius, lomit_nowhere;
+    LuaRet list;
+    LuaStack LS(L, lplane, lx, ly, lradius, lomit_nowhere, list);
+    World *w = World::fetch_global_pointer(L);
+
+    PlaneScan scan;
+    scan.set_plane(LS.ckstring(lplane));
+    scan.set_bbox_given_center_radius(util::XYZ(LS.cknumber(lx), LS.cknumber(ly), 0), LS.cknumber(lradius));
+    scan.set_shape(PlaneScan::SPHERE);
+    scan.set_sorted(true);
+    scan.set_omit_nowhere(LS.ckboolean(lomit_nowhere));
+
+    util::IdVector idv = w->plane_map_.scan(scan);
+    tangible_getall(LS, list, idv);
+    return LS.result();
+}
+
+LuaDefine(tangible_find, "config",
+    "|Find tangibles by their location."
+    "|"
+    "|There are multiple ways to specify the plane, the bounding"
+    "|box, and the shape of the search.  The most basic is to"
+    "|include these parameters in the table:"
+    "|"
+    "|   plane    : the plane to search (a string)"
+    "|   centerx  : x-coordinate of the center of the search"
+    "|   centery  : y-coordinate of the center of the search"
+    "|   centerz  : z-coordinate of the center of the search"
+    "|   radius   : the radius of the search"
+    "|   shape    : 'box', 'sphere', or 'cylinder'"
+    "|"
+    "|Shape has a default: 'sphere'.  The other parameters do"
+    "|not have default values."
+    "|"
+    "|Instead of specifying the radius as a single float,"
+    "|you may optionally specify separate radii for each dimension."
+    "|"
+    "|   radiusx  : the radius in the x-dimension."
+    "|   radiusy  : the radius in the y-dimension."
+    "|   radiusz  : the radius in the z-dimension."
+    "|"
+    "|If you specify different radii in each dimension, then the"
+    "|'sphere' shape will actually be a spheroid, and the 'cylinder'"
+    "|shape will actually be a cylindroid."
+    "|"
+    "|Instead of specifying the center and plane, you can specify"
+    "|a tangible to search near:"
+    "|"
+    "|   near     : a tangible in whose vicinity the search occurs"
+    "|"
+    "|If you specify 'near', then by default, the near tangible is"
+    "|filtered out of the search.  If you want to include it in the"
+    "|results, set the 'include' flag to true.  In this case, the"
+    "|near tangible will always be the first search result:"
+    "|"
+    "|   include  : include the 'near' object in the results"
+    "|"
+    "|If you want to search an entire plane, you can use the"
+    "|wholeplane option, which replaces all the other options:"
+    "|"
+    "|   wholeplane: the name of the plane to scan in its entirety"
+    "|"
+    "|It is valid to use 0.0 as a radius.  For example, you could"
+    "|use shape='cylinder' and radiusz=0.0 to scan a flat"
+    "|circular area."
+    "|"
+    "|It is also valid to use math.huge (infinity) as a radius.  For"
+    "|example, you could use shape='cylinder' and radiusz=math.huge"
+    "|to scan an infinitely tall cylinder."
+    "|"
+    "|If you are making a 2D game, it works fine to set all object Z"
+    "|coordinates to zero, and then do searches with centerz=0.0"
+    "|and radiusz=0.0."
+    "|") {
+    LuaArg config;
+    LuaRet result;
+    LuaStack LS(L, config, result);
+    LuaKeywordParser kw(LS, config);
+    PlaneScan scan;
+    scan.configure(kw);
+    kw.final_check_throw();
+
+    // When the configure routine sees the 'near' flag, it stores the tangible
+    // ID, but not the center and plane, because doing so would require it to
+    // know about world models.  We have to handle center and plane for 'near'
+    // separately.
+    World *w = World::fetch_global_pointer(L);
+    int64_t near = scan.near();
+    if (near != 0) {
+        Tangible *t = w->tangible_get(near);
+        assert(t != nullptr); // Should never happen.
+        const AnimStep &aqback = t->anim_queue_.back();
+        scan.set_plane(aqback.plane());
+        scan.set_center(aqback.xyz());
+    }
+    // Do the scan.
+    util::IdVector idv = w->plane_map_.scan(scan);
+    tangible_getall(LS, result, idv);
+    return LS.result();
+}
+
+LuaDefine(tangible_start, "tangible,function,arg1,arg2...",
+    "|Start a thread."
+    "|"
+    "|Every thread is owned by a tangible.  The first argument"
+    "|to 'tangible.start' indicates the tangible that owns"
+    "|the new thread.  Instead of passing a single tangible,"
+    "|you can pass a list of tangibles, in which case a thread"
+    "|is started on each tangible."
+    "|"
+    "|The function can be a lua closure, or it can be a string."
+    "|If it's a string, then the tangible's class will be"
+    "|used to look up the relevant closure."
+    "|"
+    "|The arguments arg1,arg2... will be passed to the"
+    "|function."
+    "|"
+    "|Actor and place aren't passed to the function unless"
+    "|you manually include them in the list arg1, arg2, etc."
+    "|The new thread can, however, use the builtin "
+    "|functions 'tangible.actor' and 'tangible.place' to obtain "
+    "|actor and place.  Actor will be the same actor who "
+    "|called 'tangible.start'.  Place will be the tangible that"
+    "|owns the thread, ie, the tangible passed to 'tangible.start'."
+    "|"
+    "|The new thread doesn't start running instantly:"
+    "|it waits until the current thread is finished.  The"
+    "|current thread must either block (eg, 'wait') or terminate"
+    "|before the new thread can actually begin execution."
+    "|"
+    "|If you start a thread, then start another, then both of"
+    "|the newly-started threads wait until the current thread"
+    "|is finished.  At that point, it is undefined which of the"
+    "|two new threads runs first."
+    "|"
+    "|Threads are owned by tangibles.  If a tangible is"
+    "|deleted, then none of its threads will ever be resumed,"
+    "|for any reason."
+    "|"
+    "|However, if a thread deletes its own place (ie, the tangible"
+    "|that owns the thread), then the thread will be allowed"
+    "|to continue running until it blocks. But from that point"
+    "|forward, the thread will never be resumed for any reason.") {
+
+    int top = lua_gettop(L);
+    if (top < 2) {
+        luaL_error(L, "Not enough arguments to tangible.start");
+        return 0;
+    }
+    int varlen = top - 2;
+
+    World *w = World::fetch_global_pointer(L);
+    w->guard_blockable(L, "tangible.start");
+
+    LuaVar mt, classtab, plthreads, thread, thinfo, func, tanlist;
+    LuaStack LS(L, mt, classtab, plthreads, thread, thinfo, func, tanlist);
+    LuaSpecial place(1);
+    LuaSpecial fname(2);
+
+    // If they passed in a single tangible, convert it to a tangible list.
+    int64_t place_id = LS.tanid(place);
+    if (place_id != 0) {
+        LS.newtable(tanlist);
+        LS.rawset(tanlist, 1, place);
+    } else {
+        LS.set(tanlist, place);
+        if (!LS.istable(tanlist)) {
+            luaL_error(L, "tangible.start expects a tangible or list of tangibles");
+            return 0;
+        }
+    }
+
+    for (int i = 1; ; i++) {
+        LS.rawget(place, tanlist, i);
+        if (LS.isnil(place)) break;
+
+        // Confirm that the place is a valid tangible,
+        // and get the tangible ID.
+        w->tangible_get(LS, place);
+        place_id = LS.tanid(place);
+
+        // Get place's metatable and threads table.
+        LS.getmetatable(mt, place);
+        assert(LS.istable(mt));
+        LS.rawget(plthreads, mt, "threads");
+        assert(LS.istable(plthreads));
+
+        // Get the function closure.
+        if (LS.isfunction(fname)) {
+            LS.set(func, fname);
+        } else if (LS.isstring(fname)) {
+            LS.rawget(classtab, mt, "__index");
+            assert(LS.istable(classtab));
+            LS.rawget(func, classtab, fname);
+            if (!LS.isfunction(func)) {
+                eng::string cfname = LS.ckstring(fname);
+                luaL_error(L, "tangible doesn't have method: %s", cfname.c_str());
+                return 0;
+            }
+        } else {
+            luaL_error(L, "invalid function, expected closure or string");
+            return 0;
+        }
+
+        // Create a new thread, set up function and arguments.
+        lua_State *CO = LS.newthread(thread);
+        lua_pushvalue(L, func.index());
+        for (int i = 0; i < varlen; i++) {
+            lua_pushvalue(L, i + 3);
+        }
+        lua_xmove(L, CO, varlen + 1);
+    
+        // Create the thread info table.
+        LS.newtable(thinfo);
+        LS.rawset(thinfo, "thread", thread);
+        LS.rawset(thinfo, "actorid", w->lthread_actor_id_);
+        LS.rawset(thinfo, "isnew", true);
+        LS.rawset(thinfo, "useppool", false);
+        LS.rawset(thinfo, "print", false);
+    
+        // Get a thread ID for the new thread, store it in
+        // the thread table.  
+        int64_t tid = w->alloc_id_predictable();
+        LS.rawset(plthreads, tid, thinfo);
+    
+        // Push the thread's ID into the runnable thread queue.
+        w->schedule(0, tid, place_id);
+    }
+    return LS.result();
+}
+
+
+LuaDefine(wait, "nticks",
+    "|Wait the specified number of ticks.") {
+    World *w = World::fetch_global_pointer(L);
+    w->guard_blockable(L, "wait");
+
+    // Parse the argument.
+    LuaArg seconds;
+    LuaStack LS(L, seconds);
+    int64_t n = LS.ckinteger(seconds);
+    if ((n < 0) || (n > 1000000)) {
+        luaL_error(L, "Argument to wait must be between 0 and 1000000");
+        return LS.result();
+    }
+
+    // Schedule a continuation.
+    w->schedule(w->clock_ + n, w->lthread_thread_id_, w->lthread_place_id_);
+    return lua_yield(L, 0);
+}
+
+LuaDefine(nopredict, "",
+    "|Stop predictive execution of this thread.") {
+    World *w = World::fetch_global_pointer(L);
+    w->guard_nopredict(L, "nopredict");
+
+    if (lua_gettop(L) != 0) {
+        luaL_error(L, "nopredict takes no arguments");
+    }
+    return 0;
+}
+
+LuaDefine(math_random, "(args...)",
+    "|Generate random numbers."
+    "|"
+    "|What it generates depends on the arguments:"
+    "|"
+    "|   ()          - a float in range [0.0, 1.0)"
+    "|   (high)      - an int between 1 and high inclusive"
+    "|   (low, high) - an int between low and high inclusive"
+    "|"
+    "|You may also pass in a randomstate table"
+    "|as an optional first argument."
+    "|"
+    "|math.random tries to cooperate with predictive"
+    "|reexecution to be as predictable as possible."
+    "|To achieve predictability, we used an ad-hoc"
+    "|random number generator.  It passes a variety of"
+    "|statistical tests, but it's not well-studied."
+    "|"
+    "|If you want actually want nonpredictability, or"
+    "|if you need the assurance of a well-studied random"
+    "|number generator, use math.mtrandom or"
+    "|math.cryptrandom instead.") {
+    // Parse the arguments.
+    // This is hairy because there's a lot of possibilities.
+    bool passed_in_randomstate = false;
+    int arg = 1;
+    if ((lua_gettop(L) >= arg) && (lua_istable(L, arg))) {
+        passed_in_randomstate = true;
+        arg += 1;
+    }
+    bool have_range = false;
+    int64_t low, high;
+    if ((lua_gettop(L) >= arg) && (lua_type(L, arg) == LUA_TNUMBER)) {
+        double lowf, highf;
+        if ((lua_gettop(L) >= arg+1) && (lua_type(L, arg+1) == LUA_TNUMBER)) {
+            lowf = std::floor(lua_tonumber(L, arg));
+            highf = std::floor(lua_tonumber(L, arg + 1));
+            arg += 2;
+        } else {
+            lowf = 1;
+            highf = std::floor(lua_tonumber(L, arg));
+            arg += 1;
+        }
+        if ((lowf < -LuaStack::MAXINT) || (highf > LuaStack::MAXINT)) {
+            luaL_error(L, "math.random range exceeds MAXINT");
+            return 0;
+        }
+        if (lowf > highf) {
+            luaL_error(L, "math.random range low > high");
+            return 0;
+        }
+        low = int64_t(lowf);
+        high = int64_t(highf);
+        have_range = true;
+    }
+    if (lua_gettop(L) >= arg) {
+        luaL_error(L, "math.random accepts an optional randomstate and an optional range");
+        return 0;
+    }
+
+    // Generate the seed, count, and salt.
+    // The salt prevents accidental duplication between user-specified
+    // seeds and system-generated seeds.
+    uint64_t seed, count, salt;
+    if (passed_in_randomstate) {
+        lua_pushstring(L, "seed");
+        lua_rawget(L, 1);
+        lua_pushstring(L, "count");
+        lua_rawget(L, 1);
+        if ((lua_type(L, -1) != LUA_TNUMBER) ||
+            (lua_type(L, -2) != LUA_TNUMBER)) {
+            luaL_error(L, "Not a valid randomstate table");
+            return 0;
+        }
+        double dseed = lua_tonumber(L, -2);
+        double dcount = lua_tonumber(L, -1);
+        seed = uint64_t(dseed) & LuaStack::MAXINT;
+        count = uint64_t(dcount) & LuaStack::MAXINT;
+        if (dseed < 0) {
+            salt = 0x35c9a6082a097ade;
+        } else {
+            salt = 0x4785d086ead90c20;
+        }
+        lua_pop(L, 2);
+        lua_pushstring(L, "count");
+        lua_pushnumber(L, double((count + 1) & LuaStack::MAXINT));
+        lua_rawset(L, 1);
+    } else {
+        World *w = World::fetch_global_pointer(L);
+        if (w->lthread_use_ppool_) {
+            Tangible *actor = w->tangible_get(w->lthread_actor_id_);
+            seed = w->lthread_actor_id_;
+            count = actor->id_player_pool_.get_seqno();
+            salt = 0x3ab0fb84aedc3764;
+        } else {
+            // TODO: maybe throw in a 'donotpredict' here.
+            seed = 123456;
+            count = w->id_global_pool_.get_seqno();
+            salt = 0x6f493c90faf0139d;
+        }
+    }
+
+    if (!have_range) {
+        // Generate the hash and convert to a double.
+        uint64_t hash = util::hash_ints(seed, count, salt, 456);
+        lua_pushnumber(L, util::hash_to_double(hash));
+    } else {
+        // Generate the hash and scale it into the desired range.
+        // This code is not quite right: the results are not quite
+        // uniform, this is especially true for very long ranges.
+        uint64_t hash = util::hash_ints(seed, count, salt, 456);
+        uint64_t range = (high - low) + 1;
+        uint64_t offset = (hash & 0x7FFFFFFFFFFFFFFF) % range;
+        int64_t result = low + int64_t(offset);
+        lua_pushnumber(L, result);
+    }
+    return 1;
+}
+
+LuaDefine(math_randomstate, "seed",
+    "|Create and return a randomstate table."
+    "|This is a lua table that stores the state for a random"
+    "|number generator.  A randomstate table can be passed"
+    "|to math.random."
+    "|"
+    "|You can optionally omit the seed, in which case a"
+    "|seed will be chosen randomly.  Automatically-generated"
+    "|seeds are guaranteed never to be the same as"
+    "|user-specified seeds.") {
+    double seed;
+    if (lua_gettop(L) == 0) {
+        World *w = World::fetch_global_pointer(L);
+        int64_t iseed = (w->id_global_pool_.get_seqno() & LuaStack::MAXINT) + 1;
+        seed = -iseed;
+    } else if (lua_gettop(L) == 1) {
+        if (lua_type(L, 1) != LUA_TNUMBER) {
+            luaL_error(L, "math.randomstate takes an optional integer seed");
+            return 0;
+        }
+        seed = lua_tonumber(L, 1);
+        if ((seed < 0.0) || (seed > LuaStack::MAXINT) || (std::floor(seed) != seed)) {
+            luaL_error(L, "math.randomstate seed must be an integer 0-MAXINT");
+            return 0;
+        }
+    } else {
+        luaL_error(L, "math.randomstate takes an optional integer seed");
+        return 0;
+    }
+
+    lua_newtable(L);
+    lua_pushstring(L, "seed");
+    lua_pushnumber(L, seed);
+    lua_rawset(L, -3);
+    lua_pushstring(L, "count");
+    lua_pushnumber(L, 0);
+    lua_rawset(L, -3);
+    return 1;
+}
+
+LuaSandboxBuiltin(math_randomseed, "", "");
+
+
+LuaDefine(pprint, "obj1,obj2,...",
+    "|Pretty-print object or objects.") {
+    World *w = World::fetch_global_pointer(L);
+    std::ostream *ostream = w->lthread_print_stream();
+    LuaStack LS(L);
+    for (int i = 1; i <= lua_gettop(L); i++) {
+        LuaSpecial root(i);
+        pprint(LS, root, true, ostream);
+        (*ostream) << std::endl;
+    }
+    return LS.result();
+}
+
+LuaDefine(print, "obj1,obj2,...", 
+    "|Print object or objects.") {
+    World *w = World::fetch_global_pointer(L);
+    std::ostream *ostream = w->lthread_print_stream();
+    LuaStack LS(L);
+    int n = lua_gettop(L);
+    for (int i = 1; i <= n; i++) {
+        LuaSpecial root(i);
+        atomic_print(LS, root, false, ostream);
+        if (i < n) (*ostream) << " ";
+    }
+    (*ostream) << std::endl;
+    return LS.result();
+}
+
+LuaDefine(doc, "function",
+    "|Print documentation for specified function.") {
+    World *w = World::fetch_global_pointer(L);
+    std::ostream *ostream = w->lthread_print_stream();
+    LuaArg func;
+    LuaStack LS(L, func);
+    eng::string doc = SourceDB::function_docs(LS, func);
+    if (doc == "") {
+        (*ostream) << "no doc found" << std::endl;
+    }
+    (*ostream) << doc;
+    return LS.result();
+}
+
+int lfn_http_request(lua_State *L, const char *method) {
+    World *w = World::fetch_global_pointer(L);
+    w->guard_blockable(L, "http.get");
+
+    LuaArg request;
+    LuaRet response;
+    LuaStack LS(L, request, response);
+    LuaKeywordParser kp(LS, request);
+    HttpClientRequest req;
+    
+    // Parse the request and make sure it's valid.
+    // If not, immediately pass a '400 bad request' back to lua.
+    req.set_method(method);
+    req.configure(kp);
+    kp.final_check_throw();
+    req.set_defaults();
+    eng::string error = req.check();
+    if (!error.empty()) {
+        HttpParser::store_fail(LS, response, 400, util::ss("Bad Request: ", error));
+        return LS.result();
+    }
+
+    // Give the request an ID.
+    req.set_request_id(w->id_global_pool_.get_one());
+    req.set_place_id(w->lthread_place_id_);
+    req.set_thread_id(w->lthread_thread_id_);
+
+    // Store it in the global request table.
+    w->http_requests_[req.request_id()] = req;
+
+    // Block.
+    return lua_yield(L, 0);
+}
+
+LuaDefine(http_get, "request",
+    "|Make an HTTP GET request.  Returns an HTTP response."
+    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
+    return lfn_http_request(L, "GET");
+}
+
+LuaDefine(http_head, "request",
+    "|Make an HTTP HEAD request.  Returns an HTTP response."
+    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
+    return lfn_http_request(L, "HEAD");
+}
+
+LuaDefine(http_post, "request",
+    "|Make an HTTP POST request.  Returns an HTTP response."
+    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
+    return lfn_http_request(L, "POST");
+}

luprex/cpp/core/world-difftab.cpp

@@ -0,0 +1,538 @@
+////////////////////////////////////////////////////////////////////
+//
+// This file contains the code to compare the contents of tables.
+// The top level functions in this file are:
+//
+//  World::diff_numbered_tables
+//  World::diff_tangible_databases
+//  World::patch_numbered_tables
+//  World::patch_tangible_databases
+//
+// It also contains these unit testing support routines:
+//
+//  table.diffcompare
+//  table.diffapply
+//
+// This file also contains all the support code needed to implement
+// this stuff.
+//
+////////////////////////////////////////////////////////////////////
+
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+#include "table.hpp"
+#include "world.hpp"
+#include <iostream>
+
+// Given a table and an tnmap, return the table number of the table.
+// Returns zero if the table doesn't have a table number.
+//
+static int get_table_number(LuaStack &MLS, LuaSlot mval, LuaSlot mtnmap) {
+    lua_State *L = MLS.state();
+    lua_pushvalue(L, mval.index());
+    lua_rawget(L, mtnmap.index());
+    int result = 0;
+    if (lua_type(L, -1) == LUA_TNUMBER) {
+        result = lua_tointeger(L, -1);
+    }
+    lua_pop(L, 1);
+    return result;
+}
+
+
+static bool equivalent_values(LuaStack &MLS, LuaSlot mval, LuaSlot mtnmap,
+                              LuaStack &SLS, LuaSlot sval, LuaSlot stnmap) {
+    switch (MLS.xtype(mval)) {
+    case LUA_TBOOLEAN: {
+        if (SLS.type(sval) != LUA_TBOOLEAN) return false;
+        return MLS.ckboolean(mval) == SLS.ckboolean(sval);
+    }
+    case LUA_TNUMBER: {
+        if (SLS.type(sval) != LUA_TNUMBER) return false;
+        return MLS.cknumber(mval) == SLS.cknumber(sval);
+    }
+    case LUA_TSTRING: {
+        // This could be faster if I used lua_tolstring directly.
+        if (SLS.type(sval) != LUA_TSTRING) return false;
+        return MLS.ckstring(mval) == SLS.ckstring(sval);
+    }
+    case LUA_TLIGHTUSERDATA: {
+        if (SLS.type(sval) != LUA_TLIGHTUSERDATA) return false;
+        return MLS.cktoken(mval) == SLS.cktoken(sval);
+    }
+    case LUA_TFUNCTION: {
+        // Cannot really compare.  Just return true if the types match.
+        return SLS.type(sval) == MLS.type(mval);
+    }
+    case LUA_TT_GENERAL: {
+        int midx = get_table_number(MLS, mval, mtnmap);
+        if (midx == 0) {
+            return SLS.isnil(sval);
+        }
+        int sidx = get_table_number(SLS, sval, stnmap);
+        return midx == sidx;
+    }
+    case LUA_TT_CLASS: {
+        if (SLS.xtype(sval) != LUA_TT_CLASS) return false;
+        // What if it's an ill-formed class?
+        return MLS.classname(mval) == SLS.classname(sval);
+    }
+    case LUA_TT_TANGIBLE: {
+        if (SLS.xtype(sval) != LUA_TT_TANGIBLE) return false;
+        // What if it's an ill-formed tangible?
+        return MLS.tanid(mval) == SLS.tanid(sval);
+    }
+    case LUA_TT_GLOBALENV: {
+        return (SLS.xtype(sval) == LUA_TT_GLOBALENV);
+    }
+    default:
+        // We're forcing anything else to go to NIL,
+        // and once it's NIL, we consider it 'equal'.
+        return SLS.type(sval) == LUA_TNIL;
+    }
+}
+
+static void transmit_value(LuaStack &MLS, LuaSlot mval, LuaSlot mtnmap, StreamBuffer *sb) {
+    switch (MLS.xtype(mval)) {
+    case LUA_TBOOLEAN: {
+        sb->write_uint8(LUA_TBOOLEAN);
+        sb->write_bool(MLS.ckboolean(mval));
+        return;
+    }
+    case LUA_TNUMBER: {
+        sb->write_uint8(LUA_TNUMBER);
+        sb->write_double(MLS.cknumber(mval));
+        return;
+    }
+    case LUA_TSTRING: {
+        sb->write_uint8(LUA_TSTRING);
+        sb->write_string(MLS.ckstring(mval));
+        return;
+    }
+    case LUA_TLIGHTUSERDATA: {
+        sb->write_uint8(LUA_TLIGHTUSERDATA);
+        sb->write_uint64(MLS.cktoken(mval).value);
+        return;
+    }
+    case LUA_TT_GENERAL: {
+        int midx = get_table_number(MLS, mval, mtnmap);
+        if (midx == 0) {
+            sb->write_uint8(LUA_TNIL);
+        } else {
+            sb->write_uint8(LUA_TT_GENERAL);
+            sb->write_uint32(midx);
+        }
+        return;
+    }
+    case LUA_TT_CLASS: {
+        sb->write_uint8(LUA_TT_CLASS);
+        sb->write_string(MLS.classname(mval));
+        return;
+    }
+    case LUA_TT_TANGIBLE: {
+        sb->write_uint8(LUA_TT_TANGIBLE);
+        sb->write_int64(MLS.tanid(mval));
+        return;
+    }
+    case LUA_TT_GLOBALENV: {
+        sb->write_uint8(LUA_TT_GLOBALENV);
+        return;
+    }
+    default:
+        sb->write_uint8(LUA_TNIL);
+        return;
+    }
+}
+
+static void transmit_value_debug_string(StreamBuffer *sb, eng::ostringstream &oss) {
+    int kind = sb->read_uint8();
+    switch (kind) {
+    case LUA_TBOOLEAN: {
+        bool b = sb->read_bool();
+        oss << (b ? "true":"false");
+        return;
+    }
+    case LUA_TNUMBER: {
+        oss << sb->read_double();
+        return;
+    }
+    case LUA_TSTRING: {
+        oss << sb->read_string();
+        return;
+    }
+    case LUA_TLIGHTUSERDATA: {
+        LuaToken token(sb->read_uint64());
+        oss << "[" << token.str() << "]";
+    }
+    case LUA_TT_GENERAL: {
+        oss << "table " << sb->read_int32();
+        return;
+    }
+    case LUA_TT_CLASS: {
+        oss << "class " << sb->read_string();
+        return;
+    }
+    case LUA_TT_TANGIBLE: {
+        oss << "tan " << sb->read_int64();
+        return;
+    }
+    case LUA_TT_GLOBALENV: {
+        oss << "globals";
+        return;
+    }
+    case LUA_TNIL: {
+        oss << "nil";
+        return;
+    }
+    default:
+        assert(false); // Should not get here.
+    }
+}
+
+static bool diff_tables(LuaStack &SLS0, LuaSlot stnmap, LuaSlot stab,
+                        LuaStack &MLS0, LuaSlot mtnmap, LuaSlot mtab,
+                        bool cmeta, StreamBuffer *sb) {
+    LuaVar skey, mkey, sval, mval, mnil;
+    LuaStack SLS(SLS0.state(), skey, sval);
+    LuaStack MLS(MLS0.state(), mkey, mval, mnil);
+    assert(MLS.istable(mtab));
+    assert(SLS.istable(stab));
+    MLS.set(mnil, LuaNil);
+    int nupdates = 0;
+
+    sb->write_int32(0);
+    int wc = sb->total_writes();
+
+    MLS.set(mkey, LuaNil);
+    while (MLS.next(mtab, mkey, mval)) {
+        if (!MLS.issortablekey(mkey)) continue;
+        MLS.movesortablekey(mkey, SLS, skey);
+        SLS.rawget(sval, stab, skey);
+        if (!equivalent_values(MLS, mval, mtnmap, SLS, sval, stnmap)) {
+            transmit_value(MLS, mkey, mtnmap, sb);
+            transmit_value(MLS, mval, mtnmap, sb);
+            nupdates += 1;
+        }
+    }
+
+    SLS.set(skey, LuaNil);
+    while (SLS.next(stab, skey, sval)) {
+        if (!SLS.issortablekey(skey)) continue;
+        SLS.movesortablekey(skey, MLS, mkey);
+        MLS.rawget(mval, mtab, mkey);
+        if (MLS.isnil(mval)) {
+            transmit_value(MLS, mkey, mtnmap, sb);
+            transmit_value(MLS, mval, mtnmap, sb);
+            nupdates += 1;
+        }
+    }
+
+    if (cmeta) {
+        SLS.getmetatable(sval, stab);
+        MLS.getmetatable(mval, mtab);
+        if (!equivalent_values(MLS, mval, mtnmap, SLS, sval, stnmap)) {
+            transmit_value(MLS, mnil, mtnmap, sb);
+            transmit_value(MLS, mval, mtnmap, sb);
+            nupdates += 1;
+        }
+    }
+
+    sb->overwrite_int32(wc, nupdates);
+
+    SLS.result();
+    MLS.result();
+    return (nupdates > 0);
+}
+
+static eng::string diff_tables_debug_string(StreamBuffer *sb) {
+    eng::vector<eng::string> sorted;
+    eng::ostringstream oss;
+    int ndiffs = sb->read_int32();
+    for (int i = 0; i < ndiffs; i++) {
+        transmit_value_debug_string(sb, oss);
+        oss << "=";
+        transmit_value_debug_string(sb, oss);
+        sorted.push_back(oss.str());
+        oss.str("");
+    }
+    std::sort(sorted.begin(), sorted.end());
+    for (const eng::string &s : sorted) {
+        oss << s << ";";
+    }
+    return oss.str();
+}
+
+static void set_transmitted_value(LuaStack &LS, LuaSlot tangibles, LuaSlot ntmap, LuaSlot target, StreamBuffer *sb, const char *dbinfo, DebugCollector *dbc) {
+    int kind = sb->read_uint8();
+    switch (kind) {
+    case LUA_TBOOLEAN: {
+        bool value = sb->read_bool();
+        DebugLine(dbc) << dbinfo << (value ? "true" : "false"); 
+        LS.set(target, value);
+        return;
+    }
+    case LUA_TNUMBER: {
+        double value = sb->read_double();
+        DebugLine(dbc) << dbinfo << value;
+        LS.set(target, value);
+        return;
+    }
+    case LUA_TSTRING: {
+        eng::string value = sb->read_string();
+        DebugLine(dbc) << dbinfo << "'" << value << "'";
+        LS.set(target, value);
+        return;
+    }
+    case LUA_TLIGHTUSERDATA: {
+        LuaToken value(sb->read_uint64());
+        DebugLine(dbc) << dbinfo << "[" << value.str() << "]";
+        LS.set(target, value);
+        return;
+    }
+    case LUA_TT_GENERAL: {
+        int index = sb->read_int32();
+        DebugLine(dbc) << dbinfo << "table " << index;
+        LS.rawget(target, ntmap, index);
+        return;
+    }
+    case LUA_TT_CLASS: {
+        eng::string value = sb->read_string();
+        DebugLine(dbc) << dbinfo << "class " << value;
+        LS.makeclass(target, value);
+        return;
+    }
+    case LUA_TT_TANGIBLE: {
+        int64_t id = sb->read_int64();
+        DebugLine(dbc) << dbinfo << "tan " << id;
+        LS.rawget(target, tangibles, id);
+        if (LS.isnil(target)) {
+            World *w = World::fetch_global_pointer(LS.state());
+            w->tangible_make(LS.state(), id, "nowhere", true);
+            lua_replace(LS.state(), target.index());
+        }
+        return;
+    }
+    case LUA_TT_GLOBALENV: {
+        DebugLine(dbc) << dbinfo << "global env";
+        LS.getglobaltable(target);
+        return;
+    }
+    case LUA_TNIL: {
+        DebugLine(dbc) << dbinfo << "nil";
+        LS.set(target, LuaNil);
+        return;
+    }
+    default:
+        assert(false); // Should not get here.
+    }
+}
+
+static void patch_table(LuaStack &LS0, LuaSlot tangibles, LuaSlot ntmap, LuaSlot tab, StreamBuffer *sb, DebugCollector *dbc) {
+    LuaVar key, val;
+    LuaStack LS(LS0.state(), key, val);
+    int ndiffs = sb->read_int32();
+    for (int i = 0; i < ndiffs; i++) {
+        set_transmitted_value(LS, tangibles, ntmap, key, sb, "key=", dbc);
+        set_transmitted_value(LS, tangibles, ntmap, val, sb, "val=", dbc);
+        if (LS.isnil(key)) {
+            LS.setmetatable(tab, val);
+        } else {
+            LS.rawset(tab, key, val);
+        }
+    }
+    LS.result();
+}
+
+void World::patch_numbered_tables(StreamBuffer *sb, DebugCollector *dbc) {
+    lua_State *L = state();
+    LuaVar tangibles, ntmap, tab;
+    LuaStack LS(L, tangibles, ntmap, tab);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(ntmap, LuaRegistry, "ntmap");
+    assert(LS.istable(tangibles));
+    assert(LS.istable(ntmap));
+
+    int nmodified = sb->read_int32();
+    for (int i = 0; i < nmodified; i++) {
+        int index = sb->read_int32();
+        LS.rawget(tab, ntmap, index);
+        assert(LS.istable(tab));
+        DebugHeader(dbc) << "Lua Table " << index << ":";
+        patch_table(LS, tangibles, ntmap, tab, sb, dbc);
+    }
+    LS.result();
+}
+
+void World::diff_numbered_tables(lua_State *master, StreamBuffer *sb) {
+    lua_State *synch = state();
+    LuaVar sntmap, mntmap, stnmap, mtnmap, stab, mtab;
+    LuaStack SLS(synch, sntmap, stnmap, stab);
+    LuaStack MLS(master, mntmap, mtnmap, mtab);
+    SLS.rawget(sntmap, LuaRegistry, "ntmap");
+    MLS.rawget(mntmap, LuaRegistry, "ntmap");
+    SLS.rawget(stnmap, LuaRegistry, "tnmap");
+    MLS.rawget(mtnmap, LuaRegistry, "tnmap");
+    int m_ntables = MLS.rawlen(mntmap);
+    int s_ntables = SLS.rawlen(sntmap);
+    assert(m_ntables == s_ntables);
+    
+    sb->write_int32(0);
+    int write_count_after = sb->total_writes();
+    int nmodified = 0;
+    int s_top = lua_gettop(synch);
+    int m_top = lua_gettop(master);
+    for (int id = 1; id <= m_ntables; id++) {
+        MLS.rawget(mtab, mntmap, id);
+        if (MLS.istable(mtab)) {
+            SLS.rawget(stab, sntmap, id);
+            assert(SLS.istable(stab));
+            int tw = sb->total_writes();
+            sb->write_int32(id);
+            nmodified += 1;
+            if (!diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, true, sb)) {
+                sb->unwrite_to(tw);
+                nmodified -= 1;
+            }
+        }
+        assert(lua_gettop(synch) == s_top);
+        assert(lua_gettop(master) == m_top);
+    }
+    sb->overwrite_int32(write_count_after, nmodified);
+    SLS.result();
+    MLS.result();
+}
+
+void World::patch_tangible_databases(StreamBuffer *sb, DebugCollector *dbc) {
+    lua_State *L = state();
+    LuaVar tangibles, ntmap, tab;
+    LuaStack LS(L, tangibles, ntmap, tab);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(ntmap, LuaRegistry, "ntmap");
+    assert(LS.istable(tangibles));
+    assert(LS.istable(ntmap));
+
+    int nmodified = sb->read_int32();
+    for (int i = 0; i < nmodified; i++) {
+        int64_t id = sb->read_int64();
+        LS.rawget(tab, tangibles, id);
+        assert(LS.istable(tab));
+        DebugHeader(dbc) << "Tangible DB " << id << ":";
+        patch_table(LS, tangibles, ntmap, tab, sb, dbc);
+    }
+    LS.result();
+}
+
+void World::diff_tangible_databases(const IdVector &basis, lua_State *master, StreamBuffer *sb) {
+    lua_State *synch = state();
+    LuaVar stnmap, mtnmap, stangibles, mtangibles, stab, mtab;
+    LuaStack SLS(synch, stnmap, stangibles, stab);
+    LuaStack MLS(master, mtnmap, mtangibles, mtab);
+    SLS.rawget(stnmap, LuaRegistry, "tnmap");
+    MLS.rawget(mtnmap, LuaRegistry, "tnmap");
+    SLS.rawget(stangibles, LuaRegistry, "tangibles");
+    MLS.rawget(mtangibles, LuaRegistry, "tangibles");
+    
+    sb->write_int32(0);
+    int write_count_after = sb->total_writes();
+    int nmodified = 0;
+    int s_top = lua_gettop(synch);
+    int m_top = lua_gettop(master);
+    for (int64_t id : basis) {
+        MLS.rawget(mtab, mtangibles, id);
+        SLS.rawget(stab, stangibles, id);
+        assert(MLS.istable(mtab));
+        assert(SLS.istable(stab));
+        int tw = sb->total_writes();
+        sb->write_int64(id);
+        nmodified += 1;
+        if (!diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, false, sb)) {
+            sb->unwrite_to(tw);
+            nmodified -= 1;
+        }
+        assert(lua_gettop(synch) == s_top);
+        assert(lua_gettop(master) == m_top);
+    }
+    sb->overwrite_int32(write_count_after, nmodified);
+    SLS.result();
+    MLS.result();
+}
+
+LuaDefine(table_diffcompare, "mtnmap,mtab,stnmap,stab", "for unit testing only") {
+    LuaArg mtnmap, mtab, mstnmap, mstab;
+    LuaRet dbgstring;
+    LuaVar tthread;
+    LuaStack MLS(L, mtnmap, mtab, mstnmap, mstab, dbgstring, tthread);
+    // Check the arguments.
+    MLS.checktable(mtnmap, "mtnmap");
+    MLS.checktable(mstnmap, "mstnmap");
+    MLS.checktable(mtab, "mtab");
+    MLS.checktable(mstab, "mstab");
+
+    // Create a temporary thread to be the 'synch model'.  We'll use the
+    // existing thread as the 'master model'.  Move two tables to the synch thread.
+    lua_State *synch = lua_newthread(L);
+    lua_replace(L, tthread.index());
+    lua_pushvalue(L, mstnmap.index());
+    lua_pushvalue(L, mstab.index());
+    lua_xmove(L, synch, 2);
+    LuaArg stnmap,stab;
+    LuaStack SLS(synch, stnmap, stab);
+
+    // Call tablecmp_diff.
+    StreamBuffer sb;
+    diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, true, &sb);
+
+    // Convert the output to a debug string.
+    MLS.set(dbgstring, diff_tables_debug_string(&sb));
+    return MLS.result();
+}
+
+LuaDefine(table_diffapply, "mtnmap,mtab,mstab", "for unit testing only") {
+    LuaArg mtnmap, mtab, mstab;
+    LuaRet eql, eqlstr, rtab;
+    LuaVar tthread, tangibles, mntmap, key, val;
+    LuaStack MLS(L, mtnmap, mtab, mstab, eql, eqlstr, rtab, tthread, tangibles, mntmap, key, val);
+    // Check the arguments.
+    MLS.checktable(mtnmap, "mtnmap");
+    MLS.checktable(mtab, "mtab");
+    MLS.checktable(mstab, "mstab");
+
+    // Get the tangibles map.
+    MLS.rawget(tangibles, LuaRegistry, "tangibles");
+
+    // Invert the tnmap to make the ntmap.
+    MLS.set(mntmap, LuaNewTable);
+    MLS.set(key, LuaNil);
+    while (MLS.next(mtnmap, key, val)) {
+        MLS.rawset(mntmap, val, key);
+    }
+
+    // Create a temporary thread to be the 'synch model'.  We'll use the
+    // existing thread as the 'master model'.
+    lua_State *synch = lua_newthread(L);
+    lua_replace(L, tthread.index());
+    lua_pushvalue(L, mtnmap.index());
+    lua_pushvalue(L, mstab.index());
+    lua_xmove(L, synch, 2);
+    LuaArg stnmap, stab;
+    LuaStack SLS(synch, stnmap, stab);
+
+    // Call diff_tables and patch_tables
+    StreamBuffer sb;
+    diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, true, &sb);
+
+    patch_table(MLS, tangibles, mntmap, mstab, &sb, nullptr);
+
+    bool eq = table_equal(MLS, mstab, mtab);
+    MLS.set(eql, eq);
+    if (eq) {
+        MLS.set(eqlstr, "tables equal");
+    } else {
+        MLS.set(eqlstr, "tables were supposed to be equal");
+    }
+    MLS.set(rtab, stab);
+
+    return MLS.result();
+}
+
+

luprex/cpp/core/world-diffxmit.cpp

@@ -0,0 +1,320 @@
+#include "world.hpp"
+#include <iostream>
+
+util::IdVector World::get_visible_union(int64_t actor_id, World *master) {
+    return util::sort_union_id_vectors(
+            master->get_near(actor_id, RadiusVisibility, true, false, false),
+            get_near(actor_id, RadiusVisibility, true, false, false));
+}
+
+int64_t World::patch_actor(StreamBuffer *sb, DebugCollector *dbc) {
+    DebugBlock dbb(dbc, "patch_actor");
+    int64_t actor_id = sb->read_int64();
+    Tangible *s_actor = tangible_get(actor_id);
+    if (s_actor == nullptr) {
+        DebugLine(dbc) << "create new actor " << actor_id;
+        s_actor = tangible_make(nullptr, actor_id, "", false);
+        s_actor->id_player_pool_.deserialize(sb);
+        s_actor->anim_queue_.deserialize(sb);
+        s_actor->print_buffer_.deserialize(sb);
+    } else {
+        DebugHeader(dbc) << "patching actor " << actor_id << ":";
+        s_actor->id_player_pool_.patch(sb, dbc);
+        s_actor->anim_queue_.patch(sb, dbc);
+        s_actor->print_buffer_.patch(sb, dbc);
+    }
+    s_actor->update_plane_item();
+    return actor_id;
+}
+
+void World::diff_actor(int64_t actor_id, World *master, StreamBuffer *xsb) { 
+    StreamBuffer tsb;
+
+    // Get the actor in both models.  s_actor may be nil.
+    const Tangible *s_actor = tangible_get(actor_id);
+    const Tangible *m_actor = master->tangible_get(actor_id);
+    assert(m_actor != nullptr);
+    
+    // Calculate diffs.
+    tsb.write_int64(actor_id);
+    if (s_actor == nullptr) {
+        m_actor->id_player_pool_.serialize(&tsb);
+        m_actor->anim_queue_.serialize(&tsb);
+        m_actor->print_buffer_.serialize(&tsb);
+    } else {
+        s_actor->id_player_pool_.diff(m_actor->id_player_pool_, &tsb);
+        s_actor->anim_queue_.diff(m_actor->anim_queue_, &tsb);
+        s_actor->print_buffer_.diff(m_actor->print_buffer_, &tsb);
+    }
+
+    // Forward to client, and apply to server-synchronous.
+    tsb.copy_into(xsb);
+    patch_actor(&tsb, nullptr);
+    assert(tsb.empty());
+}
+
+void World::patch_visible(StreamBuffer *sb, DebugCollector *dbc) {
+    DebugBlock dbb(dbc, "patch_visible");
+    // Receive create messages.
+    int count = sb->read_int32();
+    for (int i = 0; i < count; i++) {
+        int64_t id = sb->read_int64();
+        DebugLine(dbc) << "patch_visible create tan " << id;
+        Tangible *t = tangible_make(state(), id, "", false);
+        t->anim_queue_.deserialize(sb);
+        t->update_plane_item();
+    }
+
+    // Receive delete messages
+    count = sb->read_int32();
+    for (int i = 0; i < count; i++) {
+        int64_t id = sb->read_int64();
+        DebugLine(dbc) << "patch_visible delete tan " << id;
+        tangible_delete(id);
+    }
+
+    // Receive update messages
+    count = sb->read_int32();
+    for (int i = 0; i < count; i++) {
+        int64_t id = sb->read_int64();
+        DebugLine(dbc) << "patch_visible animqueue tan " << id;
+        Tangible *t = tangible_get(id);
+        assert(t != nullptr);
+        t->anim_queue_.patch(sb, dbc);
+        t->update_plane_item();
+    }
+}
+
+void World::diff_visible(const util::IdVector &visible, World *master, StreamBuffer *xsb) {
+    StreamBuffer tsb;
+
+    // Get the specified tangibles in both models.
+    // Some tangibles may be missing in the master, some may be missing in the sync.
+    TanVector mvis = master->tangible_get_all(visible);
+    TanVector svis = tangible_get_all(visible);
+    assert(mvis.size() == svis.size());
+    
+    // For each tangible that exists in the master, but not
+    // in the synchronous model, send a create message.
+    tsb.write_int32(0);
+    int count_pos = tsb.total_writes();
+    int count = 0;
+    for (int i = 0; i < int(svis.size()); i++) {
+        const Tangible *mt = mvis[i];
+        const Tangible *st = svis[i];
+        if ((st == nullptr) && (mt != nullptr)) {
+            count += 1;
+            tsb.write_int64(mt->id());
+            mt->anim_queue_.serialize(&tsb);
+        }
+    }
+    tsb.overwrite_int32(count_pos, count);
+
+    // For each tangible present in the synchronous model that doesn't
+    // exist in the master model, send command to delete it.
+    tsb.write_int32(0);
+    count_pos = tsb.total_writes();
+    count = 0;
+    for (int i = 0; i < int(svis.size()); i++) {
+        const Tangible *mt = mvis[i];
+        const Tangible *st = svis[i];
+        if ((mt == nullptr) && (st != nullptr)) {
+            count += 1;
+            tsb.write_int64(st->id());
+        }
+    }
+    tsb.overwrite_int32(count_pos, count);
+
+    // For each tangible present in both models, compare
+    // the animation queues.
+    tsb.write_int32(0);
+    count_pos = tsb.total_writes();
+    count = 0;
+    for (int i = 0; i < int(svis.size()); i++) {
+        const Tangible *mt = mvis[i];
+        const Tangible *st = svis[i];
+        if ((mt != nullptr) && (st != nullptr)) {
+            int64_t unwind = tsb.total_writes();
+            tsb.write_int64(st->id());
+            if (st->anim_queue_.diff(mt->anim_queue_, &tsb)) {
+                count++;
+            } else {
+                tsb.unwrite_to(unwind);
+            }
+        }
+    }
+    tsb.overwrite_int32(count_pos, count);
+
+    // Forward to client, and apply to server-synchronous.
+    tsb.copy_into(xsb);
+    patch_visible(&tsb, nullptr);
+    assert(tsb.empty());
+
+    // Copy the version number from master animqueue to synch animqueue.
+    for (int i = 0; i < int(mvis.size()); i++) {
+        const Tangible *m_tan = mvis[i];
+        if (m_tan != nullptr) {
+            Tangible *s_tan = const_cast<Tangible *>(svis[i]);
+            if (s_tan == nullptr) {
+                s_tan = tangible_get(m_tan->id());
+                assert(s_tan != nullptr);
+            }
+            s_tan->anim_queue_.update_version(m_tan->anim_queue_);
+        }
+    }
+}
+
+void World::patch_luatabs(StreamBuffer *sb, DebugCollector *dbc) {
+    DebugBlock dbb(dbc, "patch_luatabs");
+    int64_t actor_id = sb->read_int64();
+    util::HashValue closehash = sb->read_hashvalue();
+    int ncreate = sb->read_int32();
+    util::IdVector closetans = get_near(actor_id, RadiusClose, true, false, true);
+    assert(closehash == util::hash_id_vector(closetans));
+    number_lua_tables(closetans);
+    create_new_tables(ncreate);
+    // DebugLine(dbc) << "lua tables: " << nt << " existing " << ncreate << " new";
+    patch_tangible_databases(sb, dbc);
+    patch_numbered_tables(sb, dbc);
+    unnumber_lua_tables();
+}
+
+void World::diff_luatabs(int64_t actor_id, World *master, StreamBuffer *xsb) {
+    StreamBuffer tsb;
+
+    // Calculate the set of close tangibles.
+    util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false, true);
+    assert(get_near(actor_id, RadiusClose, true, false, true) == closetans);
+    util::HashValue closehash = util::hash_id_vector(closetans);
+
+    // Number and pair tables in the synchronous and master model.
+    number_lua_tables(closetans);
+    pair_lua_tables(closetans, master->state());
+    int ncreate = number_remaining_tables(closetans, master->state());
+    create_new_tables(ncreate);
+
+    // Difference transmit.
+    tsb.write_int64(actor_id);
+    tsb.write_hashvalue(closehash);
+    tsb.write_int32(ncreate);
+    diff_tangible_databases(closetans, master->state(), &tsb);
+    diff_numbered_tables(master->state(), &tsb);
+    
+    // Forward to client, and apply to server-synchronous.
+    tsb.copy_into(xsb);
+    assert(tsb.read_int64() == actor_id);
+    assert(tsb.read_hashvalue() == closehash);
+    assert(tsb.read_int32() == ncreate);
+    patch_tangible_databases(&tsb, nullptr);
+    patch_numbered_tables(&tsb, nullptr);
+    assert(tsb.empty());  
+
+    // Unnumber tables in both models.
+    unnumber_lua_tables();
+    master->unnumber_lua_tables();
+}
+
+void World::patch_tanclass(StreamBuffer *sb, DebugCollector *dbc) {
+    DebugBlock dbb(dbc, "patch_tanclass");
+    lua_State *L = state();
+    LuaVar tangibles, tab, meta, sclass;
+    LuaStack LS(L, tangibles, tab, meta, sclass);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+
+    int nmodified = sb->read_int32();
+    for (int i = 0; i < nmodified; i++) {
+        int64_t id = sb->read_int64();
+        LS.rawget(tab, tangibles, id);
+        assert(LS.istable(tab));
+        LS.getmetatable(meta, tab);
+        eng::string name = sb->read_string();
+        DebugLine(dbc) << "tanclass " << id << "=" << name;
+        if (name == "") {
+            LS.rawset(meta, "__index", LuaNil);
+        } else {
+            LS.makeclass(sclass, name);
+            LS.rawset(meta, "__index", sclass);
+        }
+    }
+    LS.result();
+}
+
+void World::diff_tanclass(int64_t actor_id, World *master, StreamBuffer *xsb) {
+    StreamBuffer tsb;
+
+    LuaVar stangibles, mtangibles, stab, mtab, smeta, mmeta, sclass, mclass;
+    LuaStack SLS(state(), stangibles, stab, smeta, sclass);
+    LuaStack MLS(master->state(), mtangibles, mtab, mmeta, mclass);
+    SLS.rawget(stangibles, LuaRegistry, "tangibles");
+    MLS.rawget(mtangibles, LuaRegistry, "tangibles");
+
+    // Calculate the set of close tangibles.
+    // TODO: we've already calculated this in an earlier function. This is wasteful.
+    util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false, true);
+
+    tsb.write_int32(0);
+    int write_count_after = tsb.total_writes();
+    int nmodified = 0;
+    for (int64_t id : closetans) {
+        MLS.rawget(mtab, mtangibles, id);
+        SLS.rawget(stab, stangibles, id);
+        MLS.getmetatable(mmeta, mtab);
+        SLS.getmetatable(smeta, stab);
+        MLS.rawget(mclass, mmeta, "__index");
+        SLS.rawget(sclass, smeta, "__index");
+        eng::string mname = MLS.classname(mclass);
+        eng::string sname = SLS.classname(sclass);
+        if (mname != sname) {
+            tsb.write_int64(id);
+            tsb.write_string(mname);
+            nmodified += 1;
+        }
+    }
+    tsb.overwrite_int32(write_count_after, nmodified);
+    SLS.result();
+    MLS.result();
+
+    // Forward to client, and apply to server-synchronous.
+    tsb.copy_into(xsb);
+    patch_tanclass(&tsb, nullptr);
+    assert(tsb.empty());
+}
+
+void World::patch_source(StreamBuffer *sb, DebugCollector *dbc) {
+    DebugBlock dbb(dbc, "patch_source");
+    bool modified = source_db_.patch(sb, dbc);
+    if (modified) {
+        eng::string errs = source_db_.rebuild();
+        DebugLine(dbc) << "Source DB rebuilt";
+        // TODO: I don't currently have any good place to send the
+        // error messages.  This is a stopgap.
+        std::cerr << errs;
+    }
+}
+
+void World::diff_source(World *master, StreamBuffer *sb) {
+    StreamBuffer tsb;
+    source_db_.diff(master->source_db_, &tsb);
+    tsb.copy_into(sb);
+    patch_source(&tsb, nullptr);
+    assert(tsb.empty());
+}
+
+int64_t World::patch_everything(StreamBuffer *sb, DebugCollector *dbc) {
+    DebugBlock dbb(dbc, "patch_everything");
+    int64_t actor_id = patch_actor(sb, dbc);
+    patch_visible(sb, dbc);
+    patch_luatabs(sb, dbc);
+    patch_tanclass(sb, dbc);
+    patch_source(sb, dbc);
+    return actor_id;
+}
+
+void World::diff_everything(int64_t actor_id, World *master, StreamBuffer *sb) {   
+    diff_actor(actor_id, master, sb);
+    util::IdVector visible = get_visible_union(actor_id, master);
+    diff_visible(visible, master, sb);
+    diff_luatabs(actor_id, master, sb);
+    diff_tanclass(actor_id, master, sb);
+    diff_source(master, sb);
+}

luprex/cpp/core/world-pairtab.cpp

@@ -0,0 +1,270 @@
+////////////////////////////////////////////////////////////////////
+//
+// This file contains the code to pair up tables in the synchronous
+// model with tables in the master model.  Here are the top-level
+// routines in this file:
+//
+//  World::number_lua_tables
+//  World::pair_lua_tables
+//  World::number_remaining_tables
+//  World::create_new_tables
+//  World::unnumber_lua_tables
+//
+////////////////////////////////////////////////////////////////////
+
+#include "luastack.hpp"
+#include "streambuffer.hpp"
+#include "world.hpp"
+
+int World::number_lua_tables(const IdVector &basis) {
+    // This is conceptually recursive, but we're going to use an
+    // explicit stack (the lua stack).
+    lua_State *L = state();
+    LuaVar tnmap, ntmap, tangibles, tab, key, val, xid;
+    LuaStack LS(L, tnmap, ntmap, tangibles, tab, key, val, xid);
+    LS.set(tnmap, LuaNewTable);
+    LS.set(ntmap, LuaNewTable);
+    LS.rawset(LuaRegistry, "tnmap", tnmap);
+    LS.rawset(LuaRegistry, "ntmap", ntmap);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    int nextid = 1;
+    int top = lua_gettop(L);
+
+    // Push all subtables onto the stack.  Note that we may push
+    // the same table twice, that's OK.
+    for (int64_t id : basis) {
+        LS.rawget(tab, tangibles, id);
+        assert(LS.istable(tab));
+        // Traverse subtables.
+        LS.set(key, LuaNil);
+        while (LS.next(tab, key, val)) {
+            if (LS.istable(val) && LS.gettabletype(val)==LUA_TT_GENERAL) {
+                lua_checkstack(L, 10);
+                lua_pushvalue(L, val.index());
+            }
+        }
+    }
+
+    // Pop tables from the stack one by one.  If the table is not
+    // already numbered, number it and push subtables onto the stack.
+    while (lua_gettop(L) > top) {
+        lua_replace(L, tab.index());
+        LS.rawget(xid, tnmap, tab);
+        if (LS.isnil(xid)) {
+            int id = nextid++;
+            LS.rawset(tnmap, tab, id);
+            LS.rawset(ntmap, id, tab);
+            // Traverse the metatable.
+            LS.getmetatable(val, tab);
+            if (LS.istable(val) && LS.gettabletype(val)==LUA_TT_GENERAL) {
+                lua_checkstack(L, 10);
+                lua_pushvalue(L, val.index());
+            }
+            // Traverse the subtables.
+            LS.set(key, LuaNil);
+            while (LS.next(tab, key, val)) {
+                if (LS.istable(val) && LS.gettabletype(val)==LUA_TT_GENERAL) {
+                    lua_checkstack(L, 10);
+                    lua_pushvalue(L, val.index());
+                }
+            }
+        }
+    }
+
+    LS.result();
+    assert(stack_is_clear());
+    return nextid - 1;
+}
+
+void World::pair_lua_tables(const IdVector &basis, lua_State *master) {
+    lua_State *synch = state();
+    LuaVar stangibles, mtangibles, sntmap, mntmap, stnmap, mtnmap, stab, mtab, skey, mkey, sval, mval, sidx, midx;
+    LuaStack SLS(synch, stangibles, stab, skey, sval, sntmap, stnmap, sidx);
+    LuaStack MLS(master, mtangibles, mtab, mkey, mval, mntmap, mtnmap, midx);
+
+    // Fetch the tangible databases
+    SLS.rawget(stangibles, LuaRegistry, "tangibles");
+    MLS.rawget(mtangibles, LuaRegistry, "tangibles");
+
+    // Fetch the synchronous model tnmap and ntmap
+    SLS.rawget(stnmap, LuaRegistry, "tnmap");
+    SLS.rawget(sntmap, LuaRegistry, "ntmap");
+    assert(SLS.istable(stnmap));
+    assert(SLS.istable(sntmap));
+
+    // Initialize the master model tnmap and ntmap
+    MLS.set(mtnmap, LuaNewTable);
+    MLS.set(mntmap, LuaNewTable);
+    MLS.rawset(LuaRegistry, "tnmap", mtnmap);
+    MLS.rawset(LuaRegistry, "ntmap", mntmap);
+    int s_ntables = SLS.rawlen(sntmap);
+    for (int i = 1; i <= s_ntables; i++) {
+        MLS.rawset(mntmap, i, 0);
+    }
+
+    // Keep track of which tables are already paired
+    eng::vector<bool> paired;
+    paired.assign(s_ntables + 1, false);
+
+    // This records the top of the stack.
+    int mtop = lua_gettop(master);
+
+    for (int64_t id : basis) {
+        MLS.rawget(mtab, mtangibles, id);
+        SLS.rawget(stab, stangibles, id);
+        assert(MLS.istable(mtab));
+        assert(SLS.istable(stab));
+        MLS.set(mkey, LuaNil);
+        while (MLS.next(mtab, mkey, mval)) {
+            if (!MLS.issortablekey(mkey)) continue;
+            if (!MLS.istable(mval)) continue;
+            MLS.movesortablekey(mkey, SLS, skey);
+            SLS.rawget(sval, stab, skey);
+            if (!SLS.istable(sval)) continue;
+            lua_checkstack(master, 20);
+            lua_checkstack(synch, 20);
+            lua_pushvalue(master, mval.index());
+            lua_pushvalue(synch, sval.index());
+        }
+    }
+
+    while (lua_gettop(master) > mtop) {
+        lua_replace(master, mtab.index());
+        lua_replace(synch, stab.index());
+        // If the master table is not a general table, skip.
+        if (MLS.gettabletype(mtab) != LUA_TT_GENERAL) continue;
+        // If the master table is already paired, skip.
+        MLS.rawget(midx, mtnmap, mtab);
+        if (MLS.isnumber(midx)) continue;
+        // If the synch table is not a table, skip.
+        if (!SLS.istable(stab)) continue;
+        // If the synch table doesn't have a number, skip.
+        SLS.rawget(sidx, stnmap, stab);
+        if (!SLS.isnumber(sidx)) continue;
+        int idx = SLS.ckinteger(sidx);
+        assert((idx >= 1) && (idx <= s_ntables));
+        // Pair the tables.
+        MLS.rawset(mtnmap, mtab, idx);
+        MLS.rawset(mntmap, idx, mtab);
+        paired[idx] = true;
+        // Potentially pair the metatables.
+        MLS.getmetatable(mval, mtab);
+        if (MLS.istable(mval)) {
+            SLS.getmetatable(sval, stab);
+            if (SLS.istable(sval)) {
+                lua_pushvalue(master, mval.index());
+                lua_pushvalue(synch, sval.index());
+            }
+        }
+        // Pair the subtables.
+        MLS.set(mkey, LuaNil);
+        while (MLS.next(mtab, mkey, mval)) {
+            if (!MLS.issortablekey(mkey)) continue;
+            if (!MLS.istable(mval)) continue;
+            MLS.movesortablekey(mkey, SLS, skey);
+            SLS.rawget(sval, stab, skey);
+            if (!SLS.istable(sval)) continue;
+            lua_checkstack(master, 20);
+            lua_checkstack(synch, 20);
+            lua_pushvalue(master, mval.index());
+            lua_pushvalue(synch, sval.index());
+        }
+    }
+
+    MLS.result();
+    SLS.result();
+}
+
+int World::number_remaining_tables(const IdVector &basis, lua_State *master) {
+    // This is conceptually recursive, but we're going to use an
+    // explicit stack (the lua stack).
+    lua_State *L = master;
+    LuaVar tnmap, ntmap, tangibles, tab, key, val, xid;
+    LuaStack LS(L, tnmap, ntmap, tangibles, tab, key, val, xid);
+    LS.rawget(tnmap, LuaRegistry, "tnmap");
+    LS.rawget(ntmap, LuaRegistry, "ntmap");
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    int ntables = LS.rawlen(ntmap);
+    eng::vector<bool> visited;
+    visited.assign(ntables + 1, false);
+    int top = lua_gettop(L);
+
+    // Push all subtables onto the stack.  Note that we may push
+    // the same table twice, that's OK.
+    for (int64_t id : basis) {
+        LS.rawget(tab, tangibles, id);
+        assert(LS.istable(tab));
+        LS.set(key, LuaNil);
+        while (LS.next(tab, key, val)) {
+            if (LS.istable(val) && LS.gettabletype(val)==LUA_TT_GENERAL) {
+                lua_checkstack(L, 10);
+                lua_pushvalue(L, val.index());
+            }
+        }
+    }
+
+    // Pop tables from the stack one by one.  If the table is not
+    // numbered, number it.  If it is not visited, visit it.
+    while (lua_gettop(L) > top) {
+        lua_replace(L, tab.index());
+        int id = 0;
+        LS.rawget(xid, tnmap, tab);
+        if (!LS.isnumber(xid)) {
+            id = visited.size();
+            LS.rawset(tnmap, tab, id);
+            LS.rawset(ntmap, id, tab);
+            visited.push_back(false);
+        } else {
+            id = LS.cknumber(xid);
+            assert((id >= 0) && (id < int(visited.size())));
+        }
+        if (!visited[id]) {
+            visited[id] = true;
+            // Traverse the metatable.
+            LS.getmetatable(val, tab);
+            if (LS.istable(val) && LS.gettabletype(val)==LUA_TT_GENERAL) {
+                lua_checkstack(L, 10);
+                lua_pushvalue(L, val.index());
+            }
+            // Traverse the subtables.
+            LS.set(key, LuaNil);
+            while (LS.next(tab, key, val)) {
+                if (LS.istable(val) && LS.gettabletype(val)==LUA_TT_GENERAL) {
+                    lua_checkstack(L, 10);
+                    lua_pushvalue(L, val.index());
+                }
+            }
+        }
+    }
+
+    LS.result();
+    assert(stack_is_clear());
+    return visited.size() - 1 - ntables;
+}
+
+void World::create_new_tables(int n) {
+    LuaVar tnmap, ntmap, tab;
+    LuaStack LS(state(), tnmap, ntmap, tab);
+    LS.rawget(tnmap, LuaRegistry, "tnmap");
+    LS.rawget(ntmap, LuaRegistry, "ntmap");
+    assert(LS.istable(tnmap));
+    assert(LS.istable(ntmap));
+    int ntables = LS.rawlen(ntmap);
+    int nextid = ntables + 1;
+    for (int i = 0; i < n; i++) {
+        int id = nextid++;
+        LS.newtable(tab);
+        LS.rawset(ntmap, id, tab);
+        LS.rawset(tnmap, tab, id);
+    }
+    LS.result();
+    assert(stack_is_clear());
+}
+
+void World::unnumber_lua_tables() {
+    // All we have to do is remove these tables from the registry.
+    LuaStack LS(state());
+    LS.rawset(LuaRegistry, "tnmap", LuaNil);
+    LS.rawset(LuaRegistry, "ntmap", LuaNil);
+}
+

luprex/cpp/core/world-testing.cpp

@@ -0,0 +1,453 @@
+#include "world.hpp"
+#include "pprint.hpp"
+#include <cassert>
+
+void World::tangible_walkto(int64_t id, int64_t animid, float x, float y) {
+    Tangible *t = tangible_get(id);
+    assert(animid != 0);
+    assert(t != nullptr);
+    AnimStep step;
+    step.set_action("walkto");
+    step.set_x(x);
+    step.set_y(y);
+    t->anim_queue_.add(animid, step);
+}
+
+
+eng::string World::tangible_anim_debug_string(int64_t id) const {
+    const Tangible *t = tangible_get(id);
+    if (t == 0) return "no such tangible";
+    return t->anim_queue_.steps_debug_string();
+}
+
+
+eng::string World::tangible_id_pool_debug_string(int64_t id) const {
+    const Tangible *t = tangible_get(id);
+    if (t == 0) return "no such tangible";
+    return t->id_player_pool_.debug_string();
+}
+
+
+eng::string World::tangible_ids_debug_string() const {
+    util::IdVector idv;
+    for (const auto &pair : tangibles_) {
+        idv.push_back(pair.first);
+    }
+    std::sort(idv.begin(), idv.end());
+    return util::id_vector_debug_string(idv);
+}
+
+eng::string World::tangibles_near_debug_string(int64_t actor, int64_t distance) {
+    eng::ostringstream result;
+    for (int64_t id : get_near(actor, distance, true, false, true)) {
+        const Tangible *tan = tangible_get(id);
+        const AnimStep &aqback = tan->anim_queue_.back();
+        result << id << ": " << aqback.graphic() << " " << aqback.plane() << " " << aqback.xyz().debug_string() << std::endl;
+    }
+    return result.str();
+}
+
+eng::string World::tangible_pprint(int64_t id) const {
+    lua_State *L = state();
+    LuaVar tangibles, tan, meta;
+    LuaStack LS(L, tangibles, tan, meta);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(tan, tangibles, id);
+    eng::ostringstream oss;
+    if (LS.istable(tan)) {
+        LS.getmetatable(meta, tan);
+        LS.clearmetatable(tan);
+        pprint(LS, tan, false, &oss);
+        LS.setmetatable(tan, meta);
+    } else {
+        oss << "<no such tangible: " << id << ">";
+    }
+    LS.result();
+    return oss.str();
+}
+
+eng::string World::numbered_tables_debug_string() const {
+    lua_State *L = state();
+    LuaVar ntmap, tab, tid;
+    LuaStack LS(L, ntmap, tab, tid);
+    eng::vector<eng::string> result;
+    eng::ostringstream oss;
+
+    // Fetch the numbered tables map.
+    LS.rawget(ntmap, LuaRegistry, "ntmap");
+
+    // Iterate over the map.  For each table, if it has
+    // a TID, output that.  Otherwise, output "unknown".
+    for (int i = 1; i < 10000; i++) {
+        LS.rawget(tab, ntmap, i);
+        if (!LS.istable(tab)) break;
+        LS.rawget(tid, tab, "TID");
+        if (LS.isstring(tid)) {
+            result.push_back(LS.ckstring(tid));
+        } else {
+            result.push_back("unknown");
+        }
+    }
+    LS.result();
+    std::sort(result.begin(), result.end());
+    for (const eng::string &s : result) {
+        oss << s << ";";
+    }
+    return oss.str();
+}
+
+eng::string World::paired_tables_debug_string(lua_State *master) const {
+    lua_State *synch = state();
+    LuaVar mntmap, sntmap, mtab, stab, mtid, stid;
+    LuaStack MLS(master, mntmap, mtab, mtid);
+    LuaStack SLS(synch, sntmap, stab, stid);
+    eng::vector<std::pair<eng::string, eng::string>> result;
+    eng::ostringstream oss;
+
+    // Fetch the numbered tables map.
+    MLS.rawget(mntmap, LuaRegistry, "ntmap");
+    SLS.rawget(sntmap, LuaRegistry, "ntmap");
+    int m_ntables = MLS.rawlen(mntmap);
+    int s_ntables = MLS.rawlen(sntmap);
+    assert(m_ntables == s_ntables);
+
+    for (int i = 1; i <= m_ntables; i++) {
+        MLS.rawget(mtab, mntmap, i);
+        SLS.rawget(stab, sntmap, i);
+        if (MLS.istable(mtab) && SLS.istable(stab)) {
+            eng::string mname = "unknown";
+            eng::string sname = "unknown";
+            MLS.rawget(mtid, mtab, "TID");
+            if (MLS.isstring(mtid)) {
+                mname = MLS.ckstring(mtid);
+            }
+            SLS.rawget(stid, stab, "TID");
+            if (SLS.isstring(stid)) {
+                sname = SLS.ckstring(stid);
+            }
+            result.push_back(std::make_pair(mname, sname));
+        }
+    }
+    MLS.result();
+    SLS.result();
+    std::sort(result.begin(), result.end());
+    for (const auto &pair : result) {
+        oss << pair.first << "=" << pair.second << ";";
+    }
+    return oss.str();
+}
+
+void World::tangible_set_string(int64_t id, const eng::string &path, const eng::string &value) {
+    lua_State *L = state();
+    LuaVar tangibles, tab, subtab;
+    LuaStack LS(L, tangibles, tab, subtab);
+
+    // Fetch the lua side of the tangible.
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(tab, tangibles, id);
+    assert(LS.istable(tab));
+    
+    // Split the path parts into the table names and final part.
+    util::StringVec pathparts = util::split(path, '.');
+    assert(pathparts.size() >= 1);
+    eng::string finalpart = pathparts.back();
+    pathparts.pop_back();
+
+    // Create subtables as necessary.
+    for (const eng::string &subname : pathparts) {
+        LS.rawget(subtab, tab, subname);
+        if (LS.isnil(subtab)) {
+            LS.set(subtab, LuaNewTable);
+            LS.rawset(tab, subname, subtab);
+        }
+        assert(LS.istable(subtab));
+        LS.set(tab, subtab);
+    }
+
+    // Set the string value.
+    LS.rawset(tab, finalpart, value);
+    LS.result();
+    assert(stack_is_clear());
+}
+    
+void World::tangible_copy_global(int64_t id, const eng::string &path, const eng::string &global) {
+    lua_State *L = state();
+    LuaVar tangibles, tab, subtab, globtab, value;
+    LuaStack LS(L, tangibles, tab, subtab, globtab, value);
+
+    // Fetch the lua side of the tangible.
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(tab, tangibles, id);
+    assert(LS.istable(tab));
+    
+    // Split the path parts into the table names and final part.
+    util::StringVec pathparts = util::split(path, '.');
+    assert(pathparts.size() >= 1);
+    eng::string finalpart = pathparts.back();
+    pathparts.pop_back();
+
+    // Create subtables as necessary.
+    for (const eng::string &subname : pathparts) {
+        LS.rawget(subtab, tab, subname);
+        if (LS.isnil(subtab)) {
+            LS.set(subtab, LuaNewTable);
+            LS.rawset(tab, subname, subtab);
+        }
+        assert(LS.istable(subtab));
+        LS.set(tab, subtab);
+    }
+
+    // Copy the global value.
+    LS.getglobaltable(globtab);
+    LS.rawget(value, globtab, global);
+    LS.rawset(tab, finalpart, value);
+    LS.result();
+}
+
+void World::tangible_set_class(int64_t id, const eng::string &c) const {
+    LuaVar tangibles, tan, meta, sclass;
+    LuaStack LS(state(), tangibles, tan, meta, sclass);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(tan, tangibles, id);
+    assert(LS.istable(tan));
+    LS.getmetatable(meta, tan);
+    if (c == "") {
+        LS.set(sclass, LuaNil);
+    } else {
+        LS.makeclass(sclass, c);
+    }
+    LS.rawset(meta, "__index", sclass);
+    LS.result();
+}
+
+eng::string World::tangible_get_class(int64_t id) const {
+    LuaVar tangibles, tan, meta, sclass;
+    LuaStack LS(state(), tangibles, tan, meta, sclass);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(tan, tangibles, id);
+    assert(LS.istable(tan));
+    LS.getmetatable(meta, tan);
+    LS.rawget(sclass, meta, "__index");
+    eng::string result = LS.classname(sclass);
+    LS.result();
+    return result;
+}
+
+static bool worlds_identical(const UniqueWorld &w1, const UniqueWorld &w2) {
+    StreamBuffer sbw1, sbw2;
+    w1->serialize(&sbw1);
+    w2->serialize(&sbw2);
+    return sbw1.contents_equal(&sbw2);
+}
+
+LuaDefine(unittests_world1animdiff, "", "some unit tests") {
+    UniqueWorld m(new World(util::WORLD_TYPE_MASTER));
+    UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC));
+    UniqueWorld cs(new World(util::WORLD_TYPE_C_SYNC));
+    StreamBuffer sb;
+    util::IdVector ids = util::id_vector_create(123, 345);
+
+    // Create some tangibles, and add some animations.
+    m->tangible_make(0, 123, "somewhere", false);
+    m->tangible_make(0, 345, "somewhere", false);
+    m->tangible_walkto(123, 770, 3, 4);
+    m->tangible_walkto(345, 771, 6, 2);
+    LuaAssertStrEq(L, m->tangible_ids_debug_string(), "123,345");
+    LuaAssertStrEq(L, m->tangible_anim_debug_string(123), 
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=770 action=walkto x=3 y=4; ");
+    LuaAssertStrEq(L, m->tangible_anim_debug_string(345),
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=771 action=walkto x=6 y=2; ");
+
+    // Now difference transmit all that to the client.
+    ss->diff_visible(ids, m.get(), &sb);
+    cs->patch_visible(&sb, nullptr);
+    LuaAssertStrEq(L, ss->tangible_ids_debug_string(), "123,345");
+    LuaAssertStrEq(L, ss->tangible_anim_debug_string(123), 
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=770 action=walkto x=3 y=4; ");
+    LuaAssertStrEq(L, ss->tangible_anim_debug_string(345),
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=771 action=walkto x=6 y=2; ");
+    LuaAssert(L, worlds_identical(ss, cs));
+    
+    // Now add some more animation records to the master.
+    m->tangible_walkto(123, 772, 7, 3);
+    m->tangible_walkto(345, 773, 2, 5);
+    LuaAssertStrEq(L, m->tangible_anim_debug_string(123),
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=770 action=walkto x=3 y=4; "
+        "id=772 action=walkto x=7 y=3; ");
+    LuaAssertStrEq(L, m->tangible_anim_debug_string(345),
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=771 action=walkto x=6 y=2; "
+        "id=773 action=walkto x=2 y=5; ");
+
+    // Now difference transmit all that to the client again.
+    ss->diff_visible(ids, m.get(), &sb);
+    cs->patch_visible(&sb, nullptr);
+    LuaAssertStrEq(L, ss->tangible_anim_debug_string(123),
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=770 action=walkto x=3 y=4; "
+        "id=772 action=walkto x=7 y=3; ");
+    LuaAssertStrEq(L, ss->tangible_anim_debug_string(345),
+        "id=0 action= plane=somewhere x=0 y=0 z=0 facing=0 graphic=; "
+        "id=771 action=walkto x=6 y=2; "
+        "id=773 action=walkto x=2 y=5; ");
+    LuaAssert(L, worlds_identical(ss, cs));
+
+    // Delete tangible 345.
+    m->tangible_delete(345);
+    LuaAssertStrEq(L, m->tangible_ids_debug_string(), "123");
+
+    // And difference transmit
+    ss->diff_visible(ids, m.get(), &sb);
+    cs->patch_visible(&sb, nullptr);
+    LuaAssertStrEq(L, ss->tangible_ids_debug_string(), "123");
+    LuaAssert(L, worlds_identical(ss, cs));
+
+    return 0;
+}
+
+LuaDefine(unittests_world2pairtab, "", "some unit tests") {
+    UniqueWorld m(new World(util::WORLD_TYPE_MASTER));
+    UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC));
+    StreamBuffer sb;
+    int ncreate;
+
+    // Create a master model containing some general tables, and
+    // some specialty tables (not numberable).
+    m->tangible_make(0, 123, "somewhere", false);
+    m->tangible_set_string(123, "inventory.TID", "inventory");
+    m->tangible_set_string(123, "transactions.TID", "transactions");
+    m->tangible_set_string(123, "skills.TID", "skills");
+    m->tangible_set_string(123, "skills.leet.TID", "skills.leet");
+    m->tangible_set_string(123, "inventory.cplx.TID", "inventory.cplx");
+    m->tangible_copy_global(123, "math", "math");
+    m->tangible_copy_global(123, "gltab", "_G");
+
+    // Now we're going to create a synchronous model that's similar to, but not
+    // exactly the same as that master model. 
+    ss->tangible_make(0, 123, "somewhere", false);
+    ss->tangible_set_string(123, "inventory.TID", "inventory");
+    ss->tangible_set_string(123, "skills.TID", "skills");
+    ss->tangible_set_string(123, "skills.crap.TID", "skills.crap");
+    ss->tangible_set_string(123, "skills.leet.TID", "skills.leet");
+    ss->tangible_set_string(123, "math.TID", "math");
+    ss->tangible_set_string(123, "gltab.TID", "gltab");
+
+    // Now we're going to test the numbering and pairing of tables.
+    // Only these tables should pair: inventory, skills, and skills.leet
+    ss->number_lua_tables(util::id_vector_create(123));
+    LuaAssertStrEq(L, ss->numbered_tables_debug_string(),
+        "gltab;inventory;math;skills;skills.crap;skills.leet;");
+    ss->pair_lua_tables(util::id_vector_create(123), m->state());
+    LuaAssertStrEq(L, ss->paired_tables_debug_string(m->state()),
+        "inventory=inventory;skills=skills;skills.leet=skills.leet;");
+
+    // Test the creation of new tables during difference transmission.
+    // The master world model above has two tables that couldn't be paired
+    // to the client: inventory.cplx, and transactions.  These two tables
+    // should be paired to new, created tables.
+    ncreate = m->number_remaining_tables(util::id_vector_create(123), m->state());
+    LuaAssert(L, ncreate == 2);
+    ss->create_new_tables(ncreate);
+    LuaAssertStrEq(L, ss->paired_tables_debug_string(m->state()),
+        "inventory=inventory;inventory.cplx=unknown;skills=skills;skills.leet=skills.leet;transactions=unknown;");
+    return 0;
+}
+
+LuaDefine(unittests_world3diffluatab, "", "some unit tests") {
+    UniqueWorld m(new World(util::WORLD_TYPE_MASTER));
+    UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC));
+    UniqueWorld cs(new World(util::WORLD_TYPE_C_SYNC));
+    StreamBuffer sb;
+
+    // Initialize all three models so that a tangible exists.
+    m->tangible_make(0, 123, "somewhere", false);
+    ss->tangible_make(0, 123, "somewhere", false);
+    cs->tangible_make(0, 123, "somewhere", false);
+    m->tangible_make(0, 345, "somewhere", false);
+    ss->tangible_make(0, 345, "somewhere", false);
+    cs->tangible_make(0, 345, "somewhere", false);
+
+    // Put some data into the master model.
+    m->tangible_set_string(123, "bacon", "crispy");
+    m->tangible_set_string(123, "inventory.gold", "wealthy");
+    m->tangible_set_string(123, "skills.hunting", "leet");
+    m->tangible_set_string(123, "skills.magic.fireball", "weak");
+    m->tangible_set_string(345, "inventory.gold", "poor");
+    m->tangible_set_string(345, "phone", "867-5309");
+
+    // The data in the master model should now look like this:
+    const char *expect_123 =
+        "{ "
+            "bacon='crispy', "
+            "inventory={ gold='wealthy' }, "
+            "skills={ "
+                "hunting='leet', "
+                "magic={ fireball='weak' } "
+            "} "
+        "}";
+    const char *expect_345 =
+        "{ "
+            "inventory={ gold='poor' }, "
+            "phone='867-5309' "
+        "}";
+    LuaAssertStrEq(L, m->tangible_pprint(123), expect_123);
+    LuaAssertStrEq(L, m->tangible_pprint(345), expect_345);
+
+    // Difference transmit.
+    ss->diff_luatabs(123, m.get(), &sb);
+    cs->patch_luatabs(&sb, nullptr);
+
+    // Verify that the data was transmitted.
+    LuaAssertStrEq(L, ss->tangible_pprint(123), expect_123);
+    LuaAssertStrEq(L, cs->tangible_pprint(123), expect_123);
+    LuaAssertStrEq(L, ss->tangible_pprint(345), expect_345);
+    LuaAssertStrEq(L, cs->tangible_pprint(345), expect_345);
+    LuaAssert(L, worlds_identical(ss, cs));
+
+    return 0;
+}
+
+LuaDefine(unittests_world4difftanclass, "", "some unit tests") {
+    UniqueWorld m(new World(util::WORLD_TYPE_MASTER));
+    UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC));
+    UniqueWorld cs(new World(util::WORLD_TYPE_C_SYNC));
+    StreamBuffer sb;
+
+    // Initialize all three models so that a tangible exists.
+    m->tangible_make(0, 123, "somewhere", false);
+    ss->tangible_make(0, 123, "somewhere", false);
+    cs->tangible_make(0, 123, "somewhere", false);
+
+    // Change the lua class of the tangible.
+    m->tangible_set_class(123, "chicken");
+    LuaAssertStrEq(L, m->tangible_get_class(123), "chicken");
+
+    // Difference transmit.
+    ss->diff_tanclass(123, m.get(), &sb);
+    cs->patch_tanclass(&sb, nullptr);
+
+    // Verify that the data was transmitted.
+    LuaAssertStrEq(L, ss->tangible_get_class(123), "chicken");
+    LuaAssertStrEq(L, cs->tangible_get_class(123), "chicken");
+    LuaAssert(L, worlds_identical(ss, cs));
+
+    // Change the class again.
+    m->tangible_set_class(123, "");
+    LuaAssertStrEq(L, m->tangible_get_class(123), "");
+
+    // Difference transmit.
+    ss->diff_tanclass(123, m.get(), &sb);
+    cs->patch_tanclass(&sb, nullptr);
+
+    // Verify that the data was transmitted.
+    LuaAssertStrEq(L, ss->tangible_get_class(123), "");
+    LuaAssertStrEq(L, cs->tangible_get_class(123), "");
+    LuaAssert(L, worlds_identical(ss, cs));
+
+    return 0;
+}

luprex/cpp/core/world.hpp

@@ -0,0 +1,567 @@
+
+#ifndef WORLD_HPP
+#define WORLD_HPP
+
+#include "wrap-set.hpp"
+#include "wrap-unordered-map.hpp"
+#include "wrap-map.hpp"
+
+#include <memory>
+#include <utility>
+
+#include "luastack.hpp"
+#include "planemap.hpp"
+#include "idalloc.hpp"
+#include "animqueue.hpp"
+#include "invocation.hpp"
+#include "streambuffer.hpp"
+#include "debugcollector.hpp"
+#include "printbuffer.hpp"
+#include "sched.hpp"
+#include "http.hpp"
+#include "source.hpp"
+#include "gui.hpp"
+#include "luasnap.hpp"
+
+class World;
+
+class Tangible : public eng::opnew {
+private:
+    friend class World;
+
+    // Serialize and deserialize
+    //
+    // The tangible's ID is not serialized.  When you serialize a tangible, you
+    // should probably serialize the ID separately.
+    //
+    // The Lua portion of the tangible is not serialized here.  Instead, the lua
+    // portion is serialized when you serialize the lua state as a whole.
+    //
+    // PlaneItem is not serialized.  The deserialize routine rebuilds the
+    // PlaneItem from the AnimQueue.
+    //
+    // World pointer is not serialized.
+    //
+    void serialize(StreamBuffer *sb);
+    void deserialize(StreamBuffer *sb);
+
+public:
+    // Always points back to the world model.
+    World *world_;
+
+    // Animation queue.
+    //
+    AnimQueue anim_queue_;
+
+    // Plane Item.
+    //
+    // The PlaneItem also contains this tangible's ID.
+    // To move this PlaneItem, update the anim_queue first, then call
+    // update_plane_item, which copies the data from the anim_queue.
+    //
+    PlaneItem plane_item_;
+
+    // Player ID pool
+    //
+    // This is present in every tangible, whether a player or not.
+    // However, the fifo is only enabled in logged-in players.
+    //
+    IdPlayerPool id_player_pool_;
+
+    // Print Buffer
+    //
+    // Stores the console output for this actor until it can be
+    // probed by the client. Most tangibles have empty printbuffers,
+    // which are stored as just a null pointer internally.
+    //
+    PrintBuffer print_buffer_;
+    
+    // constructor.
+    //
+    Tangible(World *w, int64_t id);
+
+    // Get the ID
+    //
+    int64_t id() const { return plane_item_.id(); }
+
+    void update_plane_item();
+    bool is_an_actor() { return (id_player_pool_.get_fifo_capacity() > 0); }
+    void configure_id_pool_for_actor() { id_player_pool_.set_fifo_capacity(3); id_player_pool_.refill(); }
+};
+
+using UniqueTangible = std::unique_ptr<Tangible>;
+
+class World : public eng::opnew {
+public:
+    using IdVector = util::IdVector;
+    using TanVector = eng::vector<const Tangible*>;
+    using Redirects = eng::map<int64_t, int64_t>;
+    const float RadiusVisibility = 100.0;
+    const float RadiusClose = 10.0; 
+
+    // Constructor.
+    //
+    // The constructor also calls 'lua_open' to create a new
+    // lua interpreter for this world model.
+    //
+    World(util::WorldType wt);
+
+    // Destructor.
+    //
+    // Not currently functional.
+    //
+    ~World();
+
+    // get_lua_state
+    //
+    // Get the lua interpreter associated with this world model.
+    //
+    lua_State *state() const { return lua_snap_.state(); }
+
+    // get_near
+    //
+    // Get a list of the tangibles that are near the player.  If 'exclude_nowhere' is
+    // true, exclude any tangibles on the nowhere plane (but still include the player himself).
+    // The unsorted version returns the tangibles in an unpredictable order.  If sorted
+    // is false, return them in an unpredictable order.
+    //
+    IdVector get_near(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player, bool sorted) const;
+    
+    // Make a tangible.
+    //
+    // You must provide a valid previously-unused ID.  If pushdb is true, pushes
+    // the tangible's database onto the lua stack. Otherwise, leaves the lua
+    // stack untouched.
+    //
+    Tangible *tangible_make(lua_State *L, int64_t id, const eng::string &plane, bool pushdb);
+
+    // Get a pointer to the specified tangible.
+    //
+    // If there's no such tangible, returns nullptr.
+    //
+    Tangible *tangible_get(int64_t id);
+    const Tangible *tangible_get(int64_t id) const;
+
+    // Get a pointer to the specified tangible.
+    //
+    // The value on the lua stack should be a valid lua tangible.  If not,
+    // a lua error is generated.
+    //
+    Tangible *tangible_get(const LuaStack &LS, LuaSlot slot);
+    
+    // Get pointers to many tangibles.
+    //
+    TanVector tangible_get_all(const IdVector &ids) const;
+
+    // Delete the specified tangible.
+    //
+    // If there's no such tangible, this is a no-op.
+    //
+    void tangible_delete(int64_t id);
+
+    // Create a login actor.
+    //
+    // Creates a tangible of class 'login' and returns its ID.
+    // This is used to create a temporary actor which is used during
+    // the login process.
+    //
+    int64_t create_login_actor();
+
+    // Fetch all redirects and clear the redirects table.
+    //
+    Redirects fetch_redirects();
+
+    // Probe an arbitrary lua expression.
+    //
+    // Any print-statements in the lua code are sent into
+    // a stringstream.  The return value of probe_lua is the string
+    // from the stringstream.  If the lua expression returns a
+    // value, that is also printed to the stringstream.
+    //
+    eng::string probe_lua(int64_t actor_id, const eng::string &lua);
+
+    // Probe the 'interface' function of the specified sprite.
+    //
+    void update_gui(int64_t actor_id, int64_t place_id, Gui *gui);
+
+    // Invoke an Invocation object.
+    //
+    // This is the primary dispatcher for all operations that mutate a world model.
+    // To mutate a world model, create an invocation, then invoke it.
+    //
+    // It is legal to mutate a world model without using 'Invoke', but
+    // only in authoritative world models.
+    //
+    void invoke(const Invocation &inv);
+    
+    // Get the PrintBuffer of the actor.
+    //
+    const PrintBuffer *get_printbuffer(int64_t actor_id);
+
+    // Update the source database from disk.
+    //
+    // Special case: if the source pointer is nullptr, does not update.
+    //
+    void update_source(const util::LuaSourcePtr &source);
+    void update_source(const util::LuaSourceVec &source);
+    
+    // Supply an HTTP response to an outstanding HTTP request.
+    //
+    void http_response(const HttpParser &response);
+    void http_responses(const HttpParserVec &responses);
+    
+    // Abort all HTTP requests.  This is typically used after
+    // reloading a world from a save-game.  The http requests that
+    // were in progress are long-since dead.
+    //
+    void abort_all_http_requests(int status_code, std::string_view error);
+
+    // Serve an HTTP query coming in from outside.
+    //
+    // Note: the lua code for the http_serve runs in a nonblocking
+    // context.  It must produce a result instantly.
+    //
+    HttpServerResponse http_serve(const HttpParser &request);
+    
+    // Run all unit tests.
+    //
+    void run_unittests();
+    
+    // fetch_global_pointer
+    //
+    // Given a lua state, fetch the world model associated with
+    // that lua state.
+    //
+    static World *fetch_global_pointer(lua_State *L);
+
+    // Check if the world is authoritative.
+    //
+    bool is_authoritative() const { return (world_type_ == util::WORLD_TYPE_MASTER) || (world_type_ == util::WORLD_TYPE_STANDALONE); }
+
+    // Get a table showing all outstanding HTTP requests.
+    //
+    const HttpClientRequestMap &http_requests() const { return http_requests_; }
+
+    // Serialize and deserialize.
+    //
+    void serialize(StreamBuffer *sb);
+    void deserialize(StreamBuffer *sb);
+
+    // Snapshot and rollback.
+    //
+    void snapshot();
+    void rollback();
+    bool snapshot_empty() { return snapshot_.empty(); }
+    
+    // Run any threads which according to the scheduler queue are ready.
+    //
+    void run_scheduled_threads();
+
+    // Check that the main thread has nothing on the stack
+    //
+    bool stack_is_clear() const { return lua_gettop(state()) == 0; }
+
+    // Set the lthread state.
+    //
+    // Whenever lua code is running, and ONLY when lua code is running,
+    // we store the following information in the world model:
+    //
+    //  * lthread_actor_id: current actor
+    //  * lthread_place_id: current place
+    //  * lthread_use_ppool: true if we should use the player ID pool.
+    //  * lthread_prints_: a stringstream which will collect 'print' statements.
+    //
+    // As soon as the lua code stops executing, these variables are
+    // cleared.
+    //
+    void clear_lthread_state();
+    void open_lthread_state(int64_t actor_id, int64_t place_id, int64_t thread_id, bool ppool, bool prints);
+    void close_lthread_state();
+
+    std::ostream *lthread_print_stream() const;
+
+    // Allocate a single ID.
+    //
+    // The rules are as follows:
+    //   * if lthread_use_ppool is false, uses the global pool.
+    //   * if lthread_actor_id is not a valid actor id, uses the global pool.
+    //   * otherwise, uses the player pool of lthread_actor_id.
+    //
+    int64_t alloc_id_predictable();
+    
+    // If we're in a probe, generate an error.
+    // If we're in a nonauthoritative model, do a nopredict yield.
+    // Otherwise, return.
+    void guard_blockable(lua_State *L, const char *fn);
+    
+    // If we're in a probe, return.
+    // If we're in a nonauthoritative model, do a nopredict yield.
+    // Otherwise, return.
+    void guard_nopredict(lua_State *L, const char *fn);
+
+private:
+    // Add a thread to the scheduler queue.
+    //
+    void schedule(int64_t clk, int64_t thid, int64_t plid);
+
+    // Store a pointer to a world model into a lua registry.
+    //
+    static void store_global_pointer(lua_State *L, World *w);
+
+    // Invoke a plan.
+    //
+    void invoke_plan(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data);
+
+    // Invoke a lua string.
+    //
+    void invoke_lua(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data);
+    
+    // Invoke the flush-prints operation.
+    //
+    void invoke_flush_prints(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data);
+
+    // Invoke the tick operation.
+    //
+    void invoke_tick(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data);
+    
+    // Invoke the lua_source operation.
+    //
+    void invoke_lua_source(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data);
+    
+public:
+    ////////////////////////////////////////////////////////////////////////////
+    //
+    // TESTING SUPPORT
+    //
+    // The following functions are not designed to be useful for production
+    // code, they're designed to be helpful for unit testing.
+    //
+    ////////////////////////////////////////////////////////////////////////////
+
+    // Add a 'walkto' animation to the specified tangible.
+    //
+    void tangible_walkto(int64_t id, int64_t animid, float x, float y);
+
+    // Get the tangible's animation queue as a debug string.
+    //
+    eng::string tangible_anim_debug_string(int64_t id) const;
+
+    // Get the tangible's ID Pool as a debug string.
+    //
+    eng::string tangible_id_pool_debug_string(int64_t id) const;
+
+    // Get a list of all existing tangibles as a comma-separated string.
+    //
+    eng::string tangible_ids_debug_string() const;
+
+    // Get a list of all tangibles near the target as a string.
+    //
+    eng::string tangibles_near_debug_string(int64_t actor, int64_t distance);
+    
+    // Shows the TID (table ID) of the tables that were numbered.
+    // TIDs are in alphabetical order.  Any table without a TID
+    // shows up as "unknown"
+    //
+    eng::string numbered_tables_debug_string() const;
+
+    // Paired tables debug string.  Shows TID=TID pairs, sorted alphabetically.
+    //
+    eng::string paired_tables_debug_string(lua_State *master) const;
+
+    // Store a string in the tangible's database.
+    //
+    void tangible_set_string(int64_t id, const eng::string &path, const eng::string &value);
+
+    // Copy a lua global variable into the tangible's database.
+    //
+    void tangible_copy_global(int64_t id, const eng::string &path, const eng::string &global);
+
+    // Pretty-print the entire tangible database and return it as a string.
+    //
+    eng::string tangible_pprint(int64_t id) const;
+
+    // Set the tangible's lua class.
+    //
+    void tangible_set_class(int64_t id, const eng::string &c) const;
+
+    // Get the tangible's lua class (returns empty string if none).
+    //
+    eng::string tangible_get_class(int64_t id) const;
+    
+
+public:
+    ///////////////////////////////////////////////////////////
+    //
+    // world-difftab: Nonrecursive table comparison
+    //
+    // These routines compare tables in the master lua to the corresponding
+    // tables in the synchronous lua.  This is a nonrecursive process, because
+    // the recursion has already been done during the table enumeration process.
+    //
+    ///////////////////////////////////////////////////////////
+
+    void patch_numbered_tables(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_numbered_tables(lua_State *master, StreamBuffer *sb);
+
+    void patch_tangible_databases(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_tangible_databases(const IdVector &basis, lua_State *master, StreamBuffer *sb);
+
+    void patch_tangible_classes(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_tangible_classes(const IdVector &basis, lua_State *master, StreamBuffer *sb);
+
+public:
+    ///////////////////////////////////////////////////////////
+    //
+    // Difference transmission
+    //
+    ///////////////////////////////////////////////////////////
+
+    util::IdVector get_visible_union(int64_t actor_id, World *master);
+
+    int64_t patch_actor(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_actor(int64_t actor_id, World *master, StreamBuffer *sb); 
+
+    void patch_visible(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_visible(const util::IdVector &ids, World *master, StreamBuffer *sb); 
+
+    void patch_luatabs(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_luatabs(int64_t actor_id, World *master, StreamBuffer *sb);
+
+    void patch_tanclass(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_tanclass(int64_t actor_id, World *master, StreamBuffer *sb);
+
+    void patch_source(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_source(World *master, StreamBuffer *sb);
+    
+    // This is the main entry point for difference transmission.
+    //
+    int64_t patch_everything(StreamBuffer *sb, DebugCollector *dbc);
+    void diff_everything(int64_t actor, World *master, StreamBuffer *sb);
+
+public:
+    ///////////////////////////////////////////////////////////
+    //
+    // world-pairtab: Numbering and pairing of lua tables.
+    //
+    // The following routines pair up tables in the synchronous
+    // model with tables in the master model, by assigning matching
+    // table numbers.  This is not one subroutine but several, because
+    // some of the steps happen on the server, some on the client,
+    // and so forth.
+    //
+    // The goal of these routines is to build these data structures:
+    //
+    // Table-to-number mapping is stored in registry.tnmap
+    // Number-to-table mapping is stored in registry.ntmap
+    //
+    ///////////////////////////////////////////////////////////
+
+    // In the synchronous models, number tables recursively.
+    //
+    // This is a simple recursive traversal, which numbers tables.
+    // This creates the initial ntmap in the synchronous models.
+    //
+    int number_lua_tables(const IdVector &basis);
+
+    // Pair tables in the master model to tables in the synch model.
+    //
+    // Recursively walk the master and synchronous model in parallel,
+    // copying table numbers from the synchronous ntmap into the master's ntmap.
+    //
+    void pair_lua_tables(const IdVector &basis, lua_State *master);
+
+    // Number previously unpaired tables in the master model.
+    //
+    // This finds every not-yet-numbered table in the master model,
+    // and appends these tables to the master's ntmap.  Once they're
+    // in the ntmap, they can be paired by simply creating new tables
+    // in the synchronous model.
+    //
+    int number_remaining_tables(const IdVector &basis, lua_State *master);
+
+    // Create new tables in the synchronous models.
+    //
+    // Creates new tables in the synchronous model and appends these
+    // new tables to the synchronous model's ntmap.
+    //
+    void create_new_tables(int n);
+
+    // Delete the table numbering.
+    //
+    // This simply removes registry.tnmap and registry.ntmap
+    //
+    void unnumber_lua_tables();
+
+private:
+    // Type of model
+    util::WorldType world_type_;
+
+    // A lua intepreter with snapshot function.
+    //
+    LuaSnap lua_snap_;
+    
+    // The Global ID Pool.
+    //
+    IdGlobalPool id_global_pool_;
+
+    // Source Database.
+    //
+    SourceDB source_db_;
+    PlaneMap plane_map_;
+
+    // Tangibles table.
+    //
+    eng::unordered_map<int64_t, UniqueTangible> tangibles_;
+
+    // Current time.
+    //
+    int64_t clock_;
+
+    // Thread schedule: must include every thread, except
+    // for the one currently-executing thread.
+    //
+    Schedule thread_sched_;
+
+    // Outstanding HTTP requests, indexed by request ID.
+    // Authoritative models only.
+    //
+    HttpClientRequestMap http_requests_;
+
+    // Serialized snapshot of world model.
+    //
+    StreamBuffer snapshot_;
+
+    // Redirects.
+    //
+    Redirects redirects_;
+
+    // lthread variables: see set_lthread_state for explanation.
+    //
+    int64_t lthread_actor_id_;
+    int64_t lthread_place_id_;
+    int64_t lthread_thread_id_;
+    int64_t lthread_use_ppool_;
+    std::unique_ptr<eng::ostringstream> lthread_prints_;
+
+    friend class Tangible;  
+    friend int lfn_tangible_animate(lua_State *L);
+    friend int lfn_tangible_build(lua_State *L);
+    friend int lfn_tangible_redirect(lua_State *L);
+    friend int lfn_tangible_actor(lua_State *L);
+    friend int lfn_tangible_place(lua_State *L);
+    friend int lfn_tangible_nopredict(lua_State *L);
+    friend int lfn_tangible_near(lua_State *L);
+    friend int lfn_tangible_scan(lua_State *L);
+    friend int lfn_tangible_find(lua_State *L);
+    friend int lfn_tangible_start(lua_State *L);
+    friend int lfn_math_random(lua_State *L);
+    friend int lfn_math_randomstate(lua_State *L);
+    friend int lfn_wait(lua_State *L);
+    friend int lfn_nopredict(lua_State *L);
+    friend int lfn_http_request(lua_State *L, const char *method);
+};
+
+using UniqueWorld = std::unique_ptr<World>;
+
+
+#endif // WORLD_HPP

luprex/cpp/drv/driver-common.cpp

@@ -0,0 +1,488 @@
+#define POLLVEC_SIZE (DRV_MAX_CHAN + 1)
+
+
+static void if_error_print_and_exit(const std::string_view str) {
+    if (!str.empty()) {
+        std::cerr << std::endl << "error: " << str << std::endl;
+        exit(1);
+    }
+}
+
+class Driver {
+   public:
+    enum ChanState {
+        CHAN_INACTIVE,
+        CHAN_PLAINTEXT,
+        CHAN_SSL_CONNECTING,
+        CHAN_SSL_ACCEPTING,
+        CHAN_SSL_READWRITE,
+    };
+    struct ChanInfo {
+        int chid;
+        SOCKET socket;
+        SSL *ssl;
+
+        ChanState state;
+        uint32_t nbytes;
+        const char *bytes;
+        bool ready_now;
+        bool ready_on_pollin;
+        bool ready_on_pollout;
+        bool ready_on_outgoing;
+        uint32_t last_write_nbytes;
+
+        bool marked_for_deletion() const { return state == CHAN_INACTIVE; }
+    };
+
+    EngineWrapper engw;
+    std::vector<ChanInfo> chans_;
+    std::map<int, SOCKET> listen_sockets_;
+    bool read_console_recently_;
+    std::unique_ptr<struct pollfd[]> pollvec_;
+    std::unique_ptr<char[]> chbuf_;
+
+    sslutil::UniqueCTX ssl_server_ctx_;
+    sslutil::UniqueCTX ssl_client_secure_ctx_;
+    sslutil::UniqueCTX ssl_client_insecure_ctx_;
+
+    void handle_listen_ports() {
+        uint32_t nports; const uint32_t *ports;
+        engw.get_listen_ports(&engw, &nports, &ports);
+        for (uint32_t i = 0; i < nports; i++) {
+            int port = ports[i];
+            if (listen_sockets_.find(port) == listen_sockets_.end()) {
+                std::string err;
+                SOCKET sock = listen_on_port(port, err);
+                if_error_print_and_exit(err);
+                assert(sock != INVALID_SOCKET);
+                listen_sockets_[port] = sock;
+            }
+        }
+    }
+
+    void handle_lua_source() {
+        if (engw.get_rescan_lua_source(&engw)) {
+            drvutil::ostringstream oss;
+            std::string err = drvutil::package_lua_source(".", &oss);
+            if_error_print_and_exit(err);
+            engw.play_set_lua_source(&engw, oss.size(), oss.c_str());
+        }
+    }
+
+    void close_channel(ChanInfo &chan, std::string_view err) {
+        // std::cerr << "Closing channel " << chan.chid << std::endl;
+        assert(chan.state != CHAN_INACTIVE);
+        // Close and release the SSL channel.
+        if (chan.ssl != nullptr) {
+            SSL_free(chan.ssl);
+            chan.ssl = nullptr;
+        }
+        // Close and release the socket.
+        assert(chan.socket != INVALID_SOCKET);
+        assert(socket_close(chan.socket) == 0);
+        chan.socket = INVALID_SOCKET;
+        // Close everything else.
+        engw.play_notify_close(&engw, chan.chid, err.size(), err.data());
+        chan.state = CHAN_INACTIVE;
+        chan.chid = -1;
+        chan.nbytes = 0;
+        chan.bytes = 0;
+        chan.ready_now = false;
+        chan.ready_on_pollin = false;
+        chan.ready_on_pollout = false;
+        chan.ready_on_outgoing = false;
+        chan.last_write_nbytes = 0;
+    }
+
+    void handle_console_output() {
+        while (true) {
+            uint32_t ndata; const char *data;
+            engw.get_outgoing(&engw, 0, &ndata, &data);
+            if (ndata == 0) break;
+            if (ndata > DRV_SHORTSTRING_SIZE) ndata = DRV_SHORTSTRING_SIZE;
+            int nwrote = console_write(data, ndata);
+            if (nwrote <= 0) break;
+            engw.play_sent_outgoing(&engw, 0, nwrote);
+        }
+    }
+
+    void handle_console_input() {
+        char buffer[256];
+        read_console_recently_ = false;
+        while (true) {
+            int nread = console_read(buffer, 256);
+            if (nread <= 0) break;
+            read_console_recently_ = true;
+            engw.play_recv_incoming(&engw, 0, nread, buffer);
+        }
+    }
+
+    void make_channel(SOCKET sock, int chid, SSL_CTX *ctx, ChanState state) {
+        ChanInfo newchan;
+        newchan.chid = chid;
+        newchan.socket = sock;
+        newchan.ssl = SSL_new(ctx);
+        newchan.state = state;
+        newchan.nbytes = 0;
+        newchan.bytes = 0;
+        newchan.ready_now = false;
+        newchan.ready_on_pollin = false;
+        newchan.ready_on_pollout = true;
+        newchan.ready_on_outgoing = false;
+        newchan.last_write_nbytes = 0;
+        SSL_set_fd(newchan.ssl, newchan.socket);
+        // SSL_set_msg_callback(newchan.ssl, SSL_trace);
+        // SSL_set_msg_callback_arg(newchan.ssl, BIO_new_fp(stderr,0));
+        chans_.push_back(newchan);
+    }
+
+    void handle_new_outgoing_sockets() {
+        uint32_t nchids; const uint32_t *chids;
+        engw.get_new_outgoing(&engw, &nchids, &chids);
+        for (uint32_t i = 0; i < nchids; i++) {
+            uint32_t chid = chids[i];
+            std::string err, cert, host, port;
+            const char *target = engw.get_target(&engw, chid);
+            drvutil::split_target(target, cert, host, port);
+            if (cert.empty() || host.empty() || port.empty()) {
+                std::string message = "invalid target: ";
+                message += target;
+                engw.play_notify_close(&engw, chid, message.size(), message.c_str());
+                continue;
+            }
+            SSL_CTX *ctx = nullptr;
+            if (cert == "cert") {
+                ctx = ssl_client_secure_ctx_.get();
+            } else if (cert == "nocert") {
+                ctx = ssl_client_insecure_ctx_.get();
+            } else {
+                std::string message = "invalid cert rule: ";
+                message += target;
+                engw.play_notify_close(&engw, chid, message.size(), message.c_str());
+                continue;
+            }
+            SOCKET sock = open_connection(host.c_str(), port.c_str(), err);
+            if (sock == INVALID_SOCKET) {
+                engw.play_notify_close(&engw, chid, err.size(), err.c_str());
+                continue;
+            }
+            // std::cerr << "Opening channel " << chid << std::endl;
+            make_channel(sock, chid, ctx, CHAN_SSL_CONNECTING);
+        }
+        engw.play_clear_new_outgoing(&engw);
+    }
+
+    void accept_connection(int port, SOCKET sock) {
+        std::string err;
+        SOCKET socket = accept_on_socket(sock, err);
+        if_error_print_and_exit(err);
+        if (socket != INVALID_SOCKET) {
+            uint32_t chid = engw.play_notify_accept(&engw, port);
+            // std::cerr << "Accepted channel " << chid << std::endl;
+            make_channel(socket, chid, ssl_server_ctx_.get(), CHAN_SSL_ACCEPTING);
+        }
+    }
+
+    void advance_plaintext(ChanInfo &chan) {
+        std::string err;
+
+        // Try to write plaintext to the channel.
+        uint32_t ndata; const char *data;
+        engw.get_outgoing(&engw, chan.chid, &ndata, &data);
+        if (ndata > 0) {
+            int sbytes = ndata;
+            if (sbytes > DRV_SHORTSTRING_SIZE) sbytes = DRV_SHORTSTRING_SIZE;
+            int wbytes = socket_send(chan.socket, data, sbytes, err);
+            if (wbytes < 0) {
+                close_channel(chan, err.c_str());
+            } else {
+                engw.play_sent_outgoing(&engw, chan.chid, wbytes);
+            }
+        }
+
+        // Try to read plaintext from the channel.
+        // Someday, find a way to avoid this copy.
+        int nrecv = socket_recv(chan.socket, chbuf_.get(), DRV_SHORTSTRING_SIZE, err);
+        if (nrecv < 0) {
+            close_channel(chan, err.c_str());
+        } else {
+            engw.play_recv_incoming(&engw, chan.chid, nrecv, chbuf_.get());
+        }
+
+        // Update the ready-flags for next time.
+        chan.ready_on_outgoing = true;
+        chan.ready_on_pollin = true;
+    }
+
+    void process_ssl_error(ChanInfo &chan, int retval) {
+        int error = SSL_get_error(chan.ssl, retval);
+        // std::cerr << "SSL error code = " << error << "  ";
+        if (error == SSL_ERROR_WANT_READ) {
+            chan.ready_on_pollin = true;
+        } else if (error == SSL_ERROR_WANT_WRITE) {
+            chan.ready_on_pollout = true;
+        } else {
+            std::string error = sslutil::error_string();
+            if (error == "") error = "unknown error";
+            close_channel(chan, error);
+        }
+    }
+
+    void advance_ssl_connecting(ChanInfo &chan) {
+        // std::cerr << "In advance_ssl_connecting" << std::endl;
+        int retval = SSL_connect(chan.ssl);
+        if (retval == 1) {
+            // Connection successful.
+            chan.state = CHAN_SSL_READWRITE;
+            chan.ready_now = true;
+        } else {
+            // std::cerr << "ssl_connect_error";
+            process_ssl_error(chan, retval);
+        }
+    }
+
+    void advance_ssl_accepting(ChanInfo &chan) {
+        // std::cerr << "In advance_ssl_accepting" << std::endl;
+        int retval = SSL_accept(chan.ssl);
+        if (retval == 1) {
+            // Connection successful.
+            chan.state = CHAN_SSL_READWRITE;
+            chan.ready_now = true;
+        } else {
+            process_ssl_error(chan, retval);
+        }
+    }
+
+    void advance_ssl_readwrite(ChanInfo &chan) {
+        // std::cerr << "In advance_ssl_readwrite" << std::endl;
+        // Try to read data.
+        int read_result = SSL_read(chan.ssl, chbuf_.get(), DRV_SHORTSTRING_SIZE);
+        if (read_result > 0) {
+            engw.play_recv_incoming(&engw, chan.chid, read_result, chbuf_.get());
+            chan.ready_now = true;
+        } else {
+            process_ssl_error(chan, read_result);
+            if (chan.state == CHAN_INACTIVE) return;
+        }
+
+        // Try to write data.
+        uint32_t wbytes;
+        if (chan.last_write_nbytes > 0) {
+            wbytes = chan.last_write_nbytes;
+            assert(wbytes < chan.nbytes);
+        } else {
+            wbytes = chan.nbytes;
+            if (wbytes > 65536) wbytes = 65536;
+        }
+        if (wbytes > 0) {
+            int write_result = SSL_write(chan.ssl, chan.bytes, wbytes);
+            if (write_result > 0) {
+                engw.play_sent_outgoing(&engw, chan.chid, write_result);
+                chan.last_write_nbytes = 0;
+                chan.ready_on_outgoing = true;
+            } else {
+                chan.last_write_nbytes = wbytes;
+                process_ssl_error(chan, write_result);
+                if (chan.state == CHAN_INACTIVE) return;
+            }
+        } else {
+            chan.ready_on_outgoing = true;
+        }
+        // std::cerr << "rpi=" << chan.ready_on_pollin << ".rpo=" <<
+        // chan.ready_on_pollout << ".rn=" << chan.ready_now << ".rog=" <<
+        // chan.ready_on_outgoing << "  ";
+    }
+
+    void advance_channel(ChanInfo &chan) {
+        sslutil::clear_all_errors();
+        switch (chan.state) {
+            case CHAN_PLAINTEXT:
+                advance_plaintext(chan);
+                break;
+            case CHAN_SSL_CONNECTING:
+                advance_ssl_connecting(chan);
+                break;
+            case CHAN_SSL_ACCEPTING:
+                advance_ssl_accepting(chan);
+                break;
+            case CHAN_SSL_READWRITE:
+                advance_ssl_readwrite(chan);
+                break;
+            default:
+                assert(false);
+                break;
+        }
+    }
+
+    void handle_socket_input_output() {
+        std::string err;
+        int mstimeout = read_console_recently_ ? 100 : 1000;
+
+        // Peek output buffers and determine channel release flags.
+        bool any_released = false;
+        for (ChanInfo &chan : chans_) {
+            engw.get_outgoing(&engw, chan.chid, &chan.nbytes, &chan.bytes);
+            if (chan.nbytes == 0) {
+                if (engw.get_channel_released(&engw, chan.chid)) {
+                    close_channel(chan, "");
+                    any_released = true;
+                }
+            }
+        }
+
+        // Delete any released channels
+        if (any_released) {
+            drvutil::remove_marked_items(chans_);
+        }
+
+        // Construct the struct pollfd vector.
+        int pollsize = 0;
+        for (const auto &p : listen_sockets_) {
+            struct pollfd &pfd = pollvec_[pollsize++];
+            pfd.fd = p.second;
+            pfd.events = POLLIN;
+            pfd.revents = 0;
+        }
+        for (const ChanInfo &chan : chans_) {
+            struct pollfd &pfd = pollvec_[pollsize++];
+            assert(chan.socket != INVALID_SOCKET);
+            pfd.fd = chan.socket;
+            pfd.events = 0;
+            pfd.revents = 0;
+            if (chan.ready_now) mstimeout = 0;
+            if (chan.ready_on_pollin) pfd.events |= POLLIN;
+            if (chan.ready_on_pollout) pfd.events |= POLLOUT;
+            if (chan.ready_on_outgoing && (chan.nbytes > 0))
+                pfd.events |= POLLOUT;
+            // std::cerr << "evt=" << pfd.events << ".nb=" << chan.nbytes <<
+            // std::endl;
+        }
+
+        // Do the poll.
+        socket_poll(pollvec_.get(), pollsize, mstimeout, err);
+        if_error_print_and_exit(err);
+
+        // Check listening sockets.
+        int index = 0;
+        for (auto &p : listen_sockets_) {
+            struct pollfd &pfd = pollvec_[index++];
+            if (pfd.revents & (POLLIN | POLLERR)) {
+                accept_connection(p.first, p.second);
+            }
+        }
+
+        // Advance channels where possible.
+        for (ChanInfo &chan : chans_) {
+            struct pollfd &pfd = pollvec_[index++];
+            bool pollin = ((pfd.revents & POLLIN) != 0);
+            bool pollout = ((pfd.revents & POLLOUT) != 0);
+            bool pollerr = ((pfd.revents & (POLLERR | POLLHUP)) != 0);
+            if (chan.ready_now || pollerr ||
+                (chan.ready_on_pollin && pollin) ||
+                (chan.ready_on_pollout && pollout) ||
+                (chan.ready_on_outgoing && (chan.nbytes > 0) && pollout)) {
+                chan.ready_now = false;
+                chan.ready_on_pollin = false;
+                chan.ready_on_pollout = false;
+                chan.ready_on_outgoing = false;
+                advance_channel(chan);
+            }
+            chan.nbytes = 0;
+            chan.bytes = 0;
+        }
+
+        // Delete any newly-inactive channels
+        drvutil::remove_marked_items(chans_);
+    }
+
+    int replay_logfile(const char *fn, bool verbose) {
+        engw.replay_initialize(&engw, fn);
+        if_error_print_and_exit(engw.error);
+        while (engw.rlog) {
+            engw.replay_step(&engw);
+        }
+        if_error_print_and_exit(engw.error);
+        return 0;
+    }
+
+    int drive(int argc, char *argv[]) {
+        // Remove the program name from argv.
+        std::string program = argv[0];
+        argc -= 1;
+        argv += 1;
+
+        // Load the DLL and gain access to its functions.
+        call_init_engine_wrapper(&engw);
+
+        // If argv contains "replay <filename>", do a replay,
+        // and then skip everything else.
+        if (argc >= 1) {
+            std::string cmd(argv[0]);
+            if ((cmd == "replay") || (cmd == "vreplay")) {
+                if (argc != 2) {
+                    std::cerr << "usage: " << program << " replay <filename>"
+                              << std::endl;
+                    return 1;
+                }
+                return replay_logfile(argv[1], cmd == "vreplay");
+            }
+        }
+
+        // If argv contains "record <filename>", start recording,
+        // and remove the "record <filename>" from argv.
+        std::string replaylogfn;
+        if (argc >= 1) {
+            std::string cmd = argv[0];
+            if (cmd == "record") {
+                if (argc < 2) {
+                    std::cerr << "The 'record' command must be followed by a filename" << std::endl;
+                    return 1;
+                }
+                replaylogfn = argv[1];
+                argc -= 2;
+                argv += 2;
+            }
+        }
+
+        // Initialize state variables.
+        read_console_recently_ = false;
+        chbuf_.reset(new char[DRV_SHORTSTRING_SIZE]);
+        pollvec_.reset(new struct pollfd[POLLVEC_SIZE]);
+
+        ssl_server_ctx_.reset(sslutil::new_context(SSL_VERIFY_NONE));
+        ssl_client_secure_ctx_.reset(sslutil::new_context(SSL_VERIFY_PEER));
+        ssl_client_insecure_ctx_.reset(sslutil::new_context(SSL_VERIFY_NONE));
+        ssl_load_certificate_authorities(ssl_client_secure_ctx_.get());
+        sslutil::ctx_load_dummy_cert(ssl_server_ctx_.get());
+
+        // Read the initial lua source code.
+        drvutil::ostringstream srcpak;
+        std::string srcpakerr = drvutil::package_lua_source(".", &srcpak);
+        if_error_print_and_exit(srcpakerr);
+
+        // Initialize the engine.
+        engw.play_initialize(&engw, argc, argv, srcpak.size(), srcpak.c_str(), replaylogfn.c_str());
+        if_error_print_and_exit(engw.error);
+
+        // Set up listening ports.
+        handle_listen_ports();
+
+        // Main loop.
+        while (!engw.get_stop_driver(&engw)) {
+            handle_lua_source();
+            handle_console_output();
+            handle_new_outgoing_sockets();
+            handle_socket_input_output();
+            handle_console_input();
+            handle_console_output();
+            engw.play_invoke_event_update(&engw, drvutil::get_monotonic_clock());
+        }
+
+        // Cleanup
+        engw.release(&engw);
+        for (ChanInfo &chan : chans_) {
+            close_channel(chan, "");
+        }
+        return 0;
+    }
+};

luprex/cpp/drv/driver-linux.cpp

@@ -0,0 +1,260 @@
+
+
+#include "drvutil.hpp"
+#include "sslutil.hpp"
+#include "../core/enginewrapper.hpp"
+
+#include <iostream>
+#include <cstdio>
+#include <cstring>
+#include <cassert>
+#include <map>
+#include <vector>
+#include <string>
+
+#include <poll.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <termios.h>
+#include <unistd.h>
+#include <sys/select.h>
+#include <sys/poll.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <sys/personality.h>
+#include <netdb.h>
+#include <malloc.h>
+#include <dlfcn.h>
+
+using SOCKET=int;
+const int INVALID_SOCKET = -1;
+
+struct termios orig_termios;
+
+
+void set_nonblocking(int fd) {
+    int flags = fcntl(fd, F_GETFL, 0);
+    assert(flags != -1);
+    int status = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+    assert(status != -1);
+}
+
+static void disable_tty_raw() {
+    tcsetattr(0, TCSAFLUSH, &orig_termios);
+}
+
+static void enable_tty_raw() {
+    int status = tcgetattr(0, &orig_termios);
+    assert(status >= 0);
+    atexit(disable_tty_raw);
+    struct termios raw = orig_termios;
+    raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
+    raw.c_lflag &= ~(ECHO | ICANON);
+    raw.c_oflag |= OPOST;
+    raw.c_cc[VMIN] = 0;
+    raw.c_cc[VTIME] = 0;
+    status = tcsetattr(0, TCSAFLUSH, &raw);
+    assert(status >= 0);
+}
+
+static SOCKET open_connection(const char *host, const char *port, std::string &err) {
+    struct addrinfo *addrs = nullptr;
+    struct addrinfo *goodaddr = nullptr;
+    struct addrinfo hints;
+    SOCKET sock = INVALID_SOCKET;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_INET;
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_protocol = IPPROTO_TCP;
+    hints.ai_flags = AI_NUMERICSERV;
+    
+    err.clear();
+    int status = getaddrinfo(host, port, &hints, &addrs);
+    if (status != 0) {
+        err = gai_strerror(status);
+        goto error_general;
+    }
+    if (addrs == nullptr) {
+        err = "no such host found";
+        goto error_general;
+    }
+    goodaddr = addrs;
+    assert(goodaddr->ai_family == AF_INET);
+    assert(goodaddr->ai_socktype == SOCK_STREAM);
+    assert(goodaddr->ai_protocol == IPPROTO_TCP);
+    sock = socket(goodaddr->ai_family, goodaddr->ai_socktype, goodaddr->ai_protocol);
+    if (sock <= 0) goto error_errno;
+
+    set_nonblocking(sock);
+
+    status = connect(sock, goodaddr->ai_addr, goodaddr->ai_addrlen);
+    if ((status != 0) && (errno != EINPROGRESS)) goto error_errno;
+
+    freeaddrinfo(addrs);
+    return sock;
+
+error_errno:
+    err = drvutil::strerror_str(errno);
+error_general:
+    if (sock != INVALID_SOCKET) close(sock);
+    if (addrs != nullptr) freeaddrinfo(addrs);
+    return INVALID_SOCKET;
+}
+
+static SOCKET listen_on_port(int port, std::string &err) {
+    int status, enable;
+    err.clear();
+
+    SOCKET sock = socket(AF_INET, SOCK_STREAM, 0);
+    if (sock <= 0) goto error_errno;
+
+    enable = 1;
+    status = setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
+    if (status != 0) goto error_errno;
+    
+    struct sockaddr_in server;
+    server.sin_family = AF_INET;
+    server.sin_addr.s_addr = INADDR_ANY;
+    server.sin_port = htons(port);
+
+    status = bind(sock, (struct sockaddr *)&server, sizeof(server));
+    if (status != 0) goto error_errno;
+    
+    status = listen(sock, 10);
+    if (status != 0) goto error_errno;
+
+    set_nonblocking(sock);
+    return sock;
+
+error_errno:
+    err = drvutil::strerror_str(errno);
+    if (sock >= 0) close(sock);
+    return INVALID_SOCKET;
+}
+
+static SOCKET accept_on_socket(SOCKET listen_socket, std::string &err) {
+    err.clear();
+    SOCKET chsock = accept(listen_socket, nullptr, nullptr);
+    if (chsock >= 0) {
+        set_nonblocking(chsock);
+        return chsock;
+    } else {
+        if ((errno != EAGAIN) && (errno != EWOULDBLOCK) && (errno != ECONNABORTED)) {
+            err = drvutil::strerror_str(errno);
+        }
+        return INVALID_SOCKET;
+    }
+}
+
+// the return values for socket_send and socket_recv are:
+//
+//  positive: sent or received bytes successfully
+//  zero: would block
+//  negative: channel closed, possibly cleanly or possibly with error
+//
+static int socket_send(SOCKET socket, const char *bytes, int nbytes, std::string &err) {
+    err.clear();
+    int wbytes = send(socket, bytes, nbytes, 0);
+    if (wbytes < 0) {
+        if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
+            return 0;
+        } else {
+            err = drvutil::strerror_str(errno);
+            return -1;
+        }
+    } else {
+        return wbytes;
+    }
+}
+
+static int socket_recv(SOCKET socket, char *bytes, int nbytes, std::string &err) {
+    err.clear();
+    int nrecv = recv(socket, bytes, nbytes, 0);
+    if (nrecv < 0) {
+        if ((errno == EWOULDBLOCK) || (errno == EAGAIN)) {
+            err = drvutil::strerror_str(errno);
+            return -1;
+        } else {
+            return 0;
+        }
+    } else if (nrecv == 0) {
+        return -1;
+    } else {
+        return nrecv;
+    }
+}
+
+static int socket_close(SOCKET socket) {
+    return close(socket);
+}
+
+static int socket_poll(struct pollfd *pollvec, int pollcount, int mstimeout, std::string &err) {
+    // socket_poll is implicitly expected to also poll stdin,
+    // if the OS allows that.  Linux does, so we add stdin to the
+    // poll vector.  The poll vector is required to have at
+    // least one free space in order to do this.
+    pollvec[pollcount].fd = 0;
+    pollvec[pollcount].events = POLLIN;
+    pollcount += 1;
+
+    // Do the poll.
+    int status = poll(pollvec, pollcount, mstimeout);
+    if (status < 0) {
+        err = drvutil::strerror_str(errno);
+        return -1;
+    }
+    return 0;
+}
+
+static int console_write(const char *bytes, int nbytes) {
+    return write(1, bytes, nbytes);
+}
+
+static int console_read(char *bytes, int nbytes) {
+    return read(0, bytes, nbytes);
+}
+
+// Load the DLL if it's not already loaded. Stores
+// the handle in a global variable.
+static void load_engine_dll() {
+    // Not actually implemented yet.  Currently, the engine
+    // is linked right into the executable.
+}
+
+static void call_init_engine_wrapper(EngineWrapper *w) {
+    load_engine_dll();
+    using InitFn = void (*)(EngineWrapper *);
+    InitFn initfn = (InitFn)dlsym(RTLD_DEFAULT, "init_engine_wrapper");
+    assert(initfn != nullptr);
+    initfn(w);
+}
+
+static void ssl_load_certificate_authorities(SSL_CTX *ctx) {
+    assert(SSL_CTX_set_default_verify_paths(ctx) == 1);
+}
+
+static void disable_randomization(int argc, char *argv[]) {
+   const int old_personality = personality(ADDR_NO_RANDOMIZE);
+   if (!(old_personality & ADDR_NO_RANDOMIZE)) {
+       const int new_personality = personality(ADDR_NO_RANDOMIZE);
+       if (new_personality & ADDR_NO_RANDOMIZE) {
+           execv(argv[0], argv);
+       }
+   }
+}
+
+#include "driver-common.cpp"
+
+
+int main(int argc, char **argv)
+{
+    disable_randomization(argc, argv);
+    enable_tty_raw();
+    assert(OPENSSL_init_ssl(0, NULL) == 1);
+    sslutil::clear_all_errors();
+    Driver driver;
+    return driver.drive(argc, argv);
+}
+

luprex/cpp/drv/driver-mingw.cpp

@@ -0,0 +1,279 @@
+#define WINVER 0x0600
+#define _WIN32_WINNT 0x0600
+
+#include "drvutil.hpp"
+#include "sslutil.hpp"
+#include "../cpp/enginewrapper.hpp"
+
+#include <iostream>
+#include <cstdio>
+#include <cstring>
+#include <cassert>
+#include <filesystem>
+
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#include <synchapi.h>
+#include <sysinfoapi.h>
+#include <windows.h>
+#include <openssl/ssl.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/bio.h>
+#include <openssl/pem.h>
+
+// OpenSSL requires plain ascii pathnames.  Returns empty string
+// if the path cannot be converted to plain ascii.
+std::string path_to_plain_ascii(const std::filesystem::path &path) {
+	std::wstring s = path.native();
+	for (wchar_t c : s) {
+		if ((c < 1) || (c > 127)) return "";
+	}
+	std::ostringstream oss;
+	for (wchar_t c : s) {
+		oss << ((char)c);
+	}
+	return oss.str();
+}
+	
+static void set_nonblocking(SOCKET sock) {
+    u_long mode = 1;  // 1 to enable non-blocking socket
+    int status = ioctlsocket(sock, FIONBIO, &mode);
+    assert(status == 0);
+}
+
+static PADDRINFOA find_good_addr(PADDRINFOA addrinfo) {
+    for (PADDRINFOA addr = addrinfo; addr != nullptr; addr = addr->ai_next) {
+        if (addr->ai_family == AF_INET) {
+            return addr;
+        }
+    }
+    return nullptr;
+}
+
+static SOCKET open_connection(const char *host, const char *port, std::string &err) {
+    PADDRINFOA addrs = nullptr;
+    PADDRINFOA goodaddr = nullptr;
+    SOCKET sock = INVALID_SOCKET;
+    
+    err.clear();
+    int status = getaddrinfo(host, port, nullptr, &addrs);
+    while (status == WSATRY_AGAIN) {
+        status = getaddrinfo(host, port, nullptr, &addrs);
+    }
+    if (status == WSAHOST_NOT_FOUND) {
+        err = "host not found";
+        goto error;
+    }
+    if (status != 0) {
+        err = "DNS resolution malfunction";
+        goto error;
+    }
+    goodaddr = find_good_addr(addrs);
+    if (goodaddr == nullptr) {
+        err = "host not an internet host";
+        goto error;
+    }
+    sock = socket(goodaddr->ai_family, SOCK_STREAM, IPPROTO_TCP);
+    if (sock == INVALID_SOCKET) {
+        err = "could not create a socket";
+        goto error;
+    }
+    set_nonblocking(sock);
+    status = connect(sock, goodaddr->ai_addr, goodaddr->ai_addrlen);
+    if (status != 0) {
+        int errcode = WSAGetLastError();
+        if (errcode != WSAEWOULDBLOCK) {
+            err = "connect failure";
+            goto error;
+        }
+    }
+    freeaddrinfo(addrs);
+    return sock;
+    
+error:
+    if (sock != INVALID_SOCKET) closesocket(sock);
+    if (addrs != nullptr) freeaddrinfo(addrs);
+    return SOCKET_ERROR;
+}
+
+SOCKET listen_on_port(int port, std::string &err) {
+    int status;
+    err.clear();
+    SOCKET sock = socket(AF_INET, SOCK_STREAM, 0);
+    if (sock == INVALID_SOCKET) {
+        err = "could not create a socket";
+        goto error;
+    }
+
+    struct sockaddr_in server;
+    server.sin_family = AF_INET;
+    server.sin_addr.s_addr = INADDR_ANY;
+    server.sin_port = htons(port);
+
+    status = bind(sock, (struct sockaddr *)&server, sizeof(server));
+    if (status < 0) {
+        err = "could not bind port";
+        goto error;
+    }
+    status = listen(sock, 10);
+    if (status < 0) {
+        err = "could not listen on socket";
+        goto error;
+    }
+    set_nonblocking(sock);
+    std::cerr << "listening socket is " << sock << std::endl;
+    return sock;
+
+error:
+    if (sock != INVALID_SOCKET) closesocket(sock);
+    return SOCKET_ERROR;
+}
+
+static SOCKET accept_on_socket(SOCKET listen_socket, std::string &err) {
+    SOCKET chsock = accept(listen_socket, nullptr, nullptr);
+    if (chsock != INVALID_SOCKET) {
+        set_nonblocking(chsock);
+        return chsock;
+    } else {
+        int errcode = WSAGetLastError();
+        if ((errcode == WSAEWOULDBLOCK) || (errcode == WSAECONNRESET)) {
+            return INVALID_SOCKET;
+        } else {
+            err = "accept failed";
+            return INVALID_SOCKET;
+        }
+    }
+}
+
+static int socket_send(SOCKET socket, const char *bytes, int nbytes, std::string &err) {
+    err.clear();
+    int wbytes = send(socket, bytes, nbytes, 0);
+    if (wbytes == SOCKET_ERROR) {
+        int errcode = WSAGetLastError();
+        if (errcode == WSAEWOULDBLOCK) {
+            return 0;
+        } else {
+            err = "send failure";
+            return -1;
+        }
+    } else {
+        assert(wbytes > 0);
+        return wbytes;
+    }
+}
+
+static int socket_recv(SOCKET socket, char *bytes, int nbytes, std::string &err) {
+    err.clear();
+    int nrecv = recv(socket, bytes, nbytes, 0);
+    if (nrecv < 0) {
+        int errcode = WSAGetLastError();
+        if (errcode == WSAEWOULDBLOCK) {
+            return 0;
+        } else {
+            err = "recv failure";
+            return -1;
+        }
+    } else if (nrecv == 0) {
+        return -1;
+    } else {
+        return nrecv;
+    }
+}
+
+static int socket_close(SOCKET socket) {
+    return closesocket(socket);
+}
+
+static int socket_poll(struct pollfd *pollvec, int pollcount, int mstimeout, std::string &err) {
+    if (pollcount == 0) {
+        if (mstimeout > 0) Sleep(mstimeout);
+        return 0;
+    }
+    int status = WSAPoll(pollvec, pollcount, mstimeout);
+    if (status < 0) {
+        err = strerror_str(WSAGetLastError());
+        return -1;
+    }
+    return status;
+}
+
+static void init_winsock() {
+    WSADATA data;
+    int errcode = WSAStartup(2, &data);
+    if (errcode != 0) {
+        fprintf(stderr, "Winsock didn't initalize, error %d", errcode);
+        exit(1);
+    }
+}
+
+static int console_write(const char *bytes, int nbytes) {
+    if (nbytes == 0) return 0;
+    HANDLE hstdout = GetStdHandle(STD_OUTPUT_HANDLE);
+    assert(hstdout != INVALID_HANDLE_VALUE);
+    DWORD nwrote;
+    if (nbytes > 10000) nbytes = 10000;
+    assert(WriteConsoleA(hstdout, bytes, nbytes, &nwrote, nullptr));
+    assert(nwrote > 0);
+    return nwrote;
+}
+
+static int console_read(char *bytes, int nbytes) {
+    HANDLE hstdin = GetStdHandle(STD_INPUT_HANDLE);
+    assert(hstdin != INVALID_HANDLE_VALUE);
+    INPUT_RECORD inrecords[512];
+    DWORD nread, nevents;
+    int nascii = 0;
+    if (GetNumberOfConsoleInputEvents(hstdin, &nevents)) {
+        if (int(nevents) > nbytes) nevents = nbytes;
+        ReadConsoleInputA(hstdin, inrecords, nevents, &nread);
+        for (int i = 0; i < int(nread); i++) {
+            const INPUT_RECORD &inr = inrecords[i];
+            if (inr.EventType != KEY_EVENT) continue;
+            const KEY_EVENT_RECORD &key = inr.Event.KeyEvent;
+            if (!key.bKeyDown) continue;
+            char c = key.uChar.AsciiChar;
+            bytes[nascii++] = c;
+        }
+        return nascii;
+    } else {
+        return 0;
+    }
+}
+
+static void ssl_load_certificate_authorities(SSL_CTX *ctx) {
+    HCERTSTORE hStore = CertOpenSystemStoreW(0, L"ROOT");
+    PCCERT_CONTEXT pContext = NULL;
+    X509 *x509;
+    X509_STORE *store = SSL_CTX_get_cert_store(ctx);
+
+    if (!hStore) {
+        fprintf(stderr, "Cannot open system certificate store.\n");
+        exit(1);
+    }
+
+    while ((pContext = CertEnumCertificatesInStore(hStore, pContext))) {
+        const unsigned char *encoded_cert = pContext->pbCertEncoded; 
+        x509 = d2i_X509(NULL, &encoded_cert, pContext->cbCertEncoded);
+        if (x509) {
+            X509_STORE_add_cert(store, x509);
+            X509_free(x509);
+        }
+    }
+
+    CertCloseStore(hStore, 0);
+}
+
+#include "driver-common.cpp"
+
+int main(int argc, char **argv)
+{
+    init_winsock();
+    OPENSSL_init_ssl(0, NULL);
+    SourceDB::register_lua_builtins();
+    Driver driver;
+    return driver.drive(argc, argv);
+}
+

luprex/cpp/drv/drvutil.cpp

@@ -0,0 +1,269 @@
+
+#include "drvutil.hpp"
+
+#include <string_view>
+#include <vector>
+#include <cassert>
+#include <sstream>
+#include <fstream>
+#include <string.h>
+#include <iostream>
+
+namespace drvutil {
+
+
+inline static bool ascii_isspace(char c) {
+    return (c==' ')||(c=='\t')||(c=='\r')||(c=='\n')||(c=='\f')||(c=='\v');
+}
+
+std::string_view trim(std::string_view v) {
+    while ((!v.empty()) && (ascii_isspace(v.front()))) {
+        v.remove_prefix(1);
+    }
+    while ((!v.empty()) && (ascii_isspace(v.back()))) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+static std::string_view read_to_line(std::string_view &source) {
+    size_t pos = source.find('\n');
+    std::string_view result;
+    if (pos == std::string_view::npos) {
+        result = source;
+        source = std::string_view();
+    } else {
+        result = source.substr(0, pos);
+        source = source.substr(pos + 1);
+    }
+    if ((!result.empty()) && (result.back() == '\r')) {
+        result.remove_suffix(1);
+    }
+    return result;
+}
+
+std::vector<std::string_view> split_view(std::string_view v, char sep) {
+    std::vector<std::string_view> result;
+    while (true) {
+        size_t pos = v.find(sep);
+        if (pos == std::string_view::npos) break;
+        result.push_back(v.substr(0, pos));
+        v = v.substr(pos + 1);
+    }
+    result.push_back(v);
+    return result;
+}
+
+void split_target(std::string_view target, std::string &cert, std::string &host, std::string &port) {
+    std::vector<std::string_view> split = split_view(target, ':');
+    if (split.size() != 3) {
+        cert.clear(); host.clear(); port.clear();
+        return;
+    }
+    if (split[0].empty() || split[1].empty() || split[2].empty()) {
+        cert.clear(); host.clear(); port.clear();
+        return;
+    }
+    cert = std::string(split[0]);
+    host = std::string(split[1]);
+    port = std::string(split[2]);
+}
+
+
+static std::vector<std::string> parse_control_lst(std::string_view ctrl) {
+    std::vector<std::string> result;
+    while (!ctrl.empty()) {
+        std::string_view line = read_to_line(ctrl);
+        std::string_view trimmed = trim(line);
+        if ((trimmed.size() > 0) && (trimmed[0] != '#')) {
+            result.emplace_back(trimmed);
+        }
+    }
+    return result;
+}
+
+// Read a source file into a string.
+//
+static std::string read_file(const char *fn, std::string &err) {
+    std::ifstream t(fn);
+    if (t.fail()) {
+        err = std::string("Could not open ") + fn;
+        return "";
+    }
+    t.seekg(0, std::ios::end);
+    size_t size = t.tellg();
+    std::string result(size, ' ');
+    t.seekg(0);
+    t.read(&result[0], size); 
+    if ((t.fail()) || (size_t(t.tellg()) != size)) {
+        err = std::string("Could not read ") + fn;
+        return "";
+    }
+    err = "";
+    return result;
+}
+
+// This encoding can be read by StreamBuffer::read_uint32.
+//
+static void sbwrite_uint32(std::ostream *s, uint32_t v) {
+    s->write((const char *)&v, 4);
+}
+
+// This encoding can be read by StreamBuffer::read_uint64.
+//
+static void sbwrite_uint64(std::ostream *s, uint64_t v) {
+    s->write((const char *)&v, 8);
+}
+
+// This encoding can be read by StreamBuffer::read_string.
+//
+static void sbwrite_string(std::ostream *s, std::string_view sv) {
+    s->put(0xFF);
+    sbwrite_uint64(s, sv.size());
+    s->write(sv.data(), sv.size());
+}
+
+// This encoding can be read by StreamBuffer::read_string.
+//
+static bool sbwrite_file(std::ostream *s, const char *fn) {
+    s->put(0xFF);
+    uint64_t pos1 = s->tellp();
+    sbwrite_uint64(s, 0);
+    uint64_t pos2 = s->tellp();
+    std::ifstream t(fn);
+    if (t.fail()) {
+        return false;
+    }
+    *s << t.rdbuf();
+    if (t.fail()) {
+        return false;
+    }
+    uint64_t pos3 = s->tellp();
+    s->seekp(pos1);
+    sbwrite_uint64(s, pos3 - pos2);
+    s->seekp(pos3);
+    return true;
+}
+
+std::string package_lua_source(const std::string &base, std::ostream *s) {
+    std::string err;
+    std::string cfn = base + "/lua/control.lst";
+    std::string ctrl = read_file(cfn.c_str(), err);
+    if (!err.empty()) {
+        return err;
+    }
+
+    std::vector<std::string> names = parse_control_lst(ctrl);
+    sbwrite_uint32(s, names.size());
+    for (int i = 0; i < int(names.size()); i++) {
+        sbwrite_string(s, names[i]);
+    }
+    for (int i = 0; i < int(names.size()); i++) {
+        std::string lfn = base + "/lua/" + names[i];
+        if (!sbwrite_file(s, lfn.c_str())) {
+            return std::string("Cannot read source file: ") + lfn;
+        }
+    }
+    return "";
+}
+
+// strerror has to be the most overcomplicated function imaginable. The simple
+// version, 'strerror', is not thread-safe, and the improved versions are all
+// incompatible from OS to OS.  Even different versions of linux aren't
+// compatible.  A lot of conditional compilation is needed.
+
+#if defined(__linux__)
+
+inline static void strerror_helper(int status, int errnum, char errbuf[256]) {
+    if (status != 0) {
+        snprintf(errbuf, 256, "unknown errno %d", errnum);
+    }
+}
+
+inline static void strerror_helper(const char *result, int errnum, char errbuf[256]) {
+    if (result != errbuf) {
+        snprintf(errbuf, 256, "%s", result);
+    }
+}
+
+void strerror_safe(int errnum, char errbuf[256]) {
+    auto rval = strerror_r(errnum, errbuf, 256);
+    strerror_helper(rval, errnum, errbuf);
+}
+
+#elif defined(_WIN32)
+
+void strerror_safe(int errnum, char errbuf[256]) {
+    int status = strerror_s(errbuf, 256, errnum);
+    if (status != 0) {
+        snprintf(errbuf, 256, "unknown errno %d", errnum);
+    }
+);
+
+#endif
+
+std::string strerror_str(int errnum) {
+    char buf[256];
+    strerror_safe(errnum, buf);
+    return buf;
+}
+
+// The monotonic clock is required to start at zero at initialization time,
+// advance steadily, and never go backwards. It is okay, however, if it is a
+// little inaccurate, or if it drifts a little over time.
+
+#if defined(__linux__) 
+
+    class MonoClock {
+    private:
+        struct timespec base_;
+    public:
+        MonoClock() {
+            int status = clock_gettime(CLOCK_MONOTONIC, &base_);
+            assert(status == 0);
+        }
+        double get() {
+            struct timespec t;
+            int status = clock_gettime(CLOCK_MONOTONIC, &t);
+            assert(status == 0);
+            double tv_sec = t.tv_sec - base_.tv_sec;
+            double tv_nsec = t.tv_nsec - base_.tv_nsec;
+            return tv_sec + (tv_nsec * 1.0E-9);
+        }
+    };
+
+#elif defined(_WIN32)
+
+    class MonoClock {
+    public:
+        double freq_;
+        LONGLONG base_;
+        inline LONGLONG qpc() {
+            LARGE_INTEGER x;
+            BOOL status = QueryPerformanceCounter(&x);
+            assert(status != 0);
+            return x.QuadPart;
+        }
+        MonoClock() {
+            LARGE_INTEGER x;
+            BOOL status = QueryPerformanceFrequency(&x);
+            assert(status != 0);
+            freq_ = 1.0 / double(x.QuadPart);
+            base_ = qpc();
+        }
+        double get() {
+            return (qpc() - base) * freq_;
+        }
+    };
+
+#else
+    #error "Only support __linux__ or _WIN32"
+#endif
+
+
+static MonoClock monoclock;
+double get_monotonic_clock() {
+    return monoclock.get();
+}
+
+} // namespace drv

luprex/cpp/drv/drvutil.hpp

@@ -0,0 +1,99 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// DRIVER_UTIL
+//
+////////////////////////////////////////////////////////////////////////////////
+
+
+#ifndef DRVUTIL_HPP
+#define DRVUTIL_HPP
+
+#include <vector>
+#include <string>
+#include <memory>
+#include <string_view>
+#include <ostream>
+#include <sstream>
+#include <algorithm>
+
+namespace drvutil {
+
+// Read the lua source from disk into an ostringstream.
+//
+// To pass the lua source into the DLL, here is what you do: Construct an
+// ostringstream.  Use package_lua_source to package all the lua source into
+// the ostringstream.  Fetch the packaged source code using ostringstream::str.
+// Pass the packaged source code into drv_set_lua_source.
+//
+// The DLL must then decode the source package.  Here is how it does that:
+// It creates a StreamBuffer from the packaged up source.  Then it must
+// call these StreamBuffer methods:
+//
+//   - read the number of source files using read_uint32.
+//   - for each file, read the filename using read_string.
+//   - for each file, read the contents using read_string.
+//
+// If package_lua_source encounters an error reading the source code, then it
+// returns an error message.  In this case, the ostream contains garbage.  If
+// there is no error, returns the empty string.
+//
+std::string package_lua_source(const std::string &base, std::ostream *oss);
+
+// Parse a target designation.
+//
+// A target consists of 'cert::host::port'.
+//
+void split_target(std::string_view target, std::string &cert, std::string &host, std::string &port);
+
+// Get a system error message, in an OS-independent manner.
+//
+// These versions of strerror is thread-safe, and it never fails
+// to put a message into the buffer.
+//
+void strerror_safe(int errnum, char result[256]);
+std::string strerror_str(int errnum);
+
+// Get the amount of time elapsed since program start.
+//
+// This is guaranteed to be monotonically increasing.  It is not
+// guaranteed to be accurate.  Error could gradually accumulate over
+// time.
+//
+double get_monotonic_clock();
+
+// drvutil::ostringstream
+//
+// This is a variant of ostringstream in which it is possible
+// to get the contents without copying.   To get the contents
+// without copying, use oss.size() and oss.c_str()
+//
+class ostringstream : public std::ostringstream {
+    class rstringbuf : public std::stringbuf {
+    public:
+        char *eback() { return std::streambuf::eback(); }
+    };
+    rstringbuf rsbuf_;
+public:
+    ostringstream() {
+        std::basic_ostream<char>::rdbuf(&rsbuf_);
+    }
+    size_t size() {
+        return tellp();
+    }
+    const char *c_str() {
+        return rsbuf_.eback();
+    }
+};
+
+// Remove items from a vector that are marked for deletion.
+//
+template<class T>
+void remove_marked_items(T &vec) {
+    auto iter = std::partition(vec.begin(), vec.end(), [] (const auto &x) { return !x.marked_for_deletion(); });
+    vec.erase(iter, vec.end());
+}
+
+
+} // namespace drvutil
+
+#endif // DRVUTIL_HPP

luprex/cpp/drv/sslutil.cpp

@@ -0,0 +1,211 @@
+#include "drvutil.hpp"
+#include "sslutil.hpp"
+#include <iostream>
+#include <cassert>
+#include <vector>
+#include <filesystem>
+
+namespace sslutil {
+
+const char *dummy_cert = 
+    "-----BEGIN CERTIFICATE-----\n"
+    "MIIDezCCAmOgAwIBAgIUajKmxrLMr9zBMlphrTJU5qKG8FgwDQYJKoZIhvcNAQEL\n"
+    "BQAwTDELMAkGA1UEBhMCVVMxFTATBgNVBAgMDFBlbm5zeWx2YW5pYTESMBAGA1UE\n"
+    "CgwJbG9jYWxob3N0MRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMjIwMzIyMTczMzA4\n"
+    "WhgPMjEyMjAyMjYxNzMzMDhaMEwxCzAJBgNVBAYTAlVTMRUwEwYDVQQIDAxQZW5u\n"
+    "c3lsdmFuaWExEjAQBgNVBAoMCWxvY2FsaG9zdDESMBAGA1UEAwwJbG9jYWxob3N0\n"
+    "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5OWIaKqYae4nPxvu5EP3\n"
+    "VilcjApYcMT4+2ypfQoB6PEep5lwguA929rNsTKnhGsEiQAZ0eZPEZN7VhUwf/hz\n"
+    "26jIyTT43ELkt6k97wwSZSXuT65RpSiemwEs6g2mMwzpgP6nv+yam4HjE9AKiHGN\n"
+    "YeTV72Nw1EN70t6IjIf4jsJRXqDJkUx5sSSD6j0WBTOhzozIDgZHTDwiLhatE66m\n"
+    "SNoD8oWC0PscbUgOJkFpbaCAS8RJmpsdgkTFae2rzL9cOFLGw6OgV/BV1J1s0ks8\n"
+    "+veoMMtIO6fese+OZ+DyQbuGaoaltZUXzY6QjD5l34m2mGplelT7BrpcqJTBHwmh\n"
+    "CwIDAQABo1MwUTAdBgNVHQ4EFgQUXQM5TVfJ9gpUXg8fZ8yfuUVcBP8wHwYDVR0j\n"
+    "BBgwFoAUXQM5TVfJ9gpUXg8fZ8yfuUVcBP8wDwYDVR0TAQH/BAUwAwEB/zANBgkq\n"
+    "hkiG9w0BAQsFAAOCAQEAqYX/ZGv0Qh/xdXppjnqojm8mH0giDW4tvwMqHcW3YRa3\n"
+    "9J2yYot+rHjU5g4n6HEmWDBE0eqLz9n3Y3fkFzT8RWZwBaST965CgsfGofyuA2hC\n"
+    "Ddn4Am3B5tTPmi8WWRZg8amhpGVD/mwkoVFIK0M337b1aZUJYPE+Kc9WetSL2KqB\n"
+    "EhqSQpkAWhVadzP85dq2T9EDjAvhlFTFlDEBx1GDUcc8M0KQ9NEvLT7LgoUcbMiT\n"
+    "PerlSZQTB0crchXTRSERgiwu80r7D6STn/RcPL9Fg5PkA94/d87jGbmV4sxSRsvM\n"
+    "z+DnJGjHrV1J/jHPrnVvVLpigBlGno3C5O/sRw3gcQ==\n"
+    "-----END CERTIFICATE-----\n";
+
+const char *dummy_key =
+    "-----BEGIN PRIVATE KEY-----\n"
+    "MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDk5Yhoqphp7ic/\n"
+    "G+7kQ/dWKVyMClhwxPj7bKl9CgHo8R6nmXCC4D3b2s2xMqeEawSJABnR5k8Rk3tW\n"
+    "FTB/+HPbqMjJNPjcQuS3qT3vDBJlJe5PrlGlKJ6bASzqDaYzDOmA/qe/7JqbgeMT\n"
+    "0AqIcY1h5NXvY3DUQ3vS3oiMh/iOwlFeoMmRTHmxJIPqPRYFM6HOjMgOBkdMPCIu\n"
+    "Fq0TrqZI2gPyhYLQ+xxtSA4mQWltoIBLxEmamx2CRMVp7avMv1w4UsbDo6BX8FXU\n"
+    "nWzSSzz696gwy0g7p96x745n4PJBu4ZqhqW1lRfNjpCMPmXfibaYamV6VPsGulyo\n"
+    "lMEfCaELAgMBAAECggEBAJa1AiFX4U4tva1xqNKmZV1XklWqIhzts7lnDBkF08gZ\n"
+    "qcNT5Z5mIpR09eVropwvEidZ56Yp63l5D0XYYbyAS1gfQ0QnGot7h7fdOKgB3MK4\n"
+    "PLY94gfKPNN17KqWHg2SvNNv1+cn04v78xUCb0zy5tHDp5Acexdm70ohtupARElJ\n"
+    "LSHdS7ebsqZUFXbbM3BpPEsQLi3PrzNs1DrKkZ3rR6eMGrsDqExXx8/foi9aZKsd\n"
+    "BGM2/kcTJ5aY6NhSv5iqO1oK46sbMrjVW/bYNsOyl0eFjwTRahn+Zhp/JMewZYeu\n"
+    "715g6kzbZNwEzBLgrhNPF6E2ycEr/C6z5bE78g5QCkECgYEA8s07UUY25bjYiWWy\n"
+    "W38pT7d/OXBSyKnq16N6MjVahl29r7nezFiDeLhLC0QiwXu/+qyxVZkB95MMGZXS\n"
+    "AsaKFNis3AJ6eR4SYyhpSScYKNvlKIiW37TtR4FDcy7y5LL6tFpiDDIGH3LuyWNo\n"
+    "d76142MBpv5aStnLGYU3pcZj43sCgYEA8VbNM4nqgSCQcbnHYjvsgphEMNSaoVie\n"
+    "xob2uigXdV6Te0ayoUFBnVNKVsRhk+sswuTV4k1pK/On+USVl2tQ16tcaVMjTfSD\n"
+    "HLYTJLmt6s4DcywWj5dfkbDoe5PulGXNZE960qXmOC62Lf0VMRwJ5x4FBRvGTjKC\n"
+    "zvekI2/kO7ECgYEAhBGeclb/BXXGUvY+TgadMf9d9KBkZ0IFu8Xwcd8TnoLe6vbv\n"
+    "ebery75zE228egIWKwREcYsIxuH1cvVLhrb35N73J7UxaTAyUD1rB598RL1XqPSj\n"
+    "HIwNhReK2NxwwnWYaQHA02FiczjRKjooWPojdcwk2fEArDZLg1YzLrj7HIECgYEA\n"
+    "htdx1Y8ESFtyeShMv5UtoxYCW6oeL3H9XH0CE6bc3IYYLvOkULbOO2HTEkGtJ2Fp\n"
+    "5AbJfiS0U4tS2dI5Jp4eUDH9cxexjRfFvd/5ODbKdnver5X9kQMJsbQ/YPSZg66R\n"
+    "oK9Lt7Bbvh5TScSy93psCgba1SzckspkDdGNkwMsaTECgYEAnFWaxormLUpXQRLs\n"
+    "tKzMMHgVnHlsHiqXH432zmT2fpGZHYoWbsGuQjjrHGnSiu3QbDhnzM6y/T2GRs6z\n"
+    "zHteIo/tzIyxg4MvJGJ9qANA7HoiKBdQ7G/I/NLJIyWAjj+e7/hgzKFcf+dpjpDq\n"
+    "HcKc9a4WXhC7yu79e5BnKWltHXY=\n"
+    "-----END PRIVATE KEY-----\n";
+
+std::string error_string() {
+    // Get the last code.
+    int code = 0;
+    while (true) {
+        int icode = ERR_get_error();
+        if (icode == 0) break;
+        code = icode;
+    }
+
+    // Fetch and clear errno.
+    int terrno = errno;
+    errno = 0;
+    
+    if (code != 0) {
+        const char *rc = ERR_reason_error_string(code);
+        if (rc != nullptr) {
+            return rc;
+        } else {
+            return drvutil::strerror_str(ERR_GET_REASON(code));
+        }
+    } else if (terrno != 0) {
+        return drvutil::strerror_str(terrno);
+    } else {
+        return "";
+    }
+}
+
+std::string path_to_plain_ascii(const std::filesystem::path &path) {
+	std::string s = path.native();
+	for (char c : s) {
+		if ((c < 1) || (c > 127)) return "";
+	}
+	return s;
+}
+
+void clear_all_errors() {
+    ERR_clear_error();
+    errno = 0;
+}
+
+SSL_CTX *new_context(int verify) {
+    SSL_CTX *ctx = SSL_CTX_new(TLS_method());
+    SSL_CTX_set_mode(ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+    SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
+    SSL_CTX_set_verify(ctx, verify, nullptr);
+    return ctx;
+}
+
+static int ctx_use_certificate_str(SSL_CTX *ctx, const char *str) {
+    UniqueBIO bio(BIO_new(BIO_s_mem()));
+    BIO_puts(bio.get(), str);
+    UniqueX509 certificate(PEM_read_bio_X509(bio.get(), NULL, NULL, NULL));
+    return SSL_CTX_use_certificate(ctx, certificate.get());
+}
+
+static int ctx_use_privatekey_str(SSL_CTX *ctx, const char *str) {
+    UniqueBIO bio(BIO_new(BIO_s_mem()));
+    BIO_puts(bio.get(), str);
+    UniquePKEY pkey(PEM_read_bio_PrivateKey(bio.get(), NULL, NULL, NULL));
+    return SSL_CTX_use_PrivateKey(ctx, pkey.get());
+}
+
+void ctx_load_dummy_cert(SSL_CTX *ctx) {
+    ERR_clear_error();
+    if (ctx_use_certificate_str(ctx, dummy_cert) <= 0) {
+        ERR_print_errors_fp(stderr);
+        exit(1);
+    }
+    if (ctx_use_privatekey_str(ctx, dummy_key) <= 0) {
+        ERR_print_errors_fp(stderr);
+        exit(1);
+    }
+}
+
+static int count_certificates(const char *fn) {
+    static char null_passwd;
+    ErrClearErrorOnExit ece;
+    UniqueBIO bio(BIO_new(BIO_s_file()));
+    assert(bio != nullptr);
+    if (BIO_read_filename(bio.get(), fn) <= 0) {
+        std::cerr << "Cannot open file: " << fn << std::endl;
+        exit(1);
+    }
+    int total = 0;
+    while (true) {
+        UniqueX509 x(PEM_read_bio_X509_AUX(bio.get(), nullptr, nullptr, &null_passwd));
+        if (x == nullptr) break;
+        total += 1;
+    }
+    return total;
+}
+
+static bool contains_privatekey(const char *fn) {
+    static char null_passwd;
+    ErrClearErrorOnExit ece;
+    UniqueBIO bio(BIO_new(BIO_s_file()));
+    assert(bio != nullptr);
+    if (BIO_read_filename(bio.get(), fn) <= 0) {
+        std::cerr << "Cannot open file: " << fn << std::endl;
+        exit(1);
+    }
+    UniquePKEY k(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, &null_passwd));
+    return k != nullptr;
+}
+    
+void ctx_load_cert_from_directory(SSL_CTX *ctx, const std::string &dir) {
+    std::vector<std::string> key_paths;
+    std::vector<std::string> cert_paths;
+
+    for (const auto & entry : std::filesystem::directory_iterator(dir)) {
+		std::string fn = path_to_plain_ascii(entry.path());
+		if (fn.empty()) {
+			std::cerr << "Ignoring file with non-ascii filename: " << entry.path() << std::endl;
+		} else {
+			if (count_certificates(fn.c_str()) >= 1) {
+				cert_paths.push_back(fn);
+			}
+			if (contains_privatekey(fn.c_str())) {
+				key_paths.push_back(fn);
+			}
+		}
+    }
+
+    if (cert_paths.size() > 1) {
+        std::cerr << "Directory contains multiple certs: " << dir << std::endl;
+        exit(1);
+    }
+    if (key_paths.size() > 1) {
+        std::cerr << "Directory contains multiple keys: " << dir << std::endl;
+        exit(1);
+    }
+    if (cert_paths.empty()) {
+        std::cerr << "Directory doesn't contain a cert: " << dir << std::endl;
+        exit(1);
+    }
+    if (key_paths.empty()) {
+        std::cerr << "Directory doesn't contain a key: " << dir << std::endl;
+        exit(1);
+    }
+
+    int status;
+    status = SSL_CTX_use_PrivateKey_file(ctx, key_paths[0].c_str(), SSL_FILETYPE_PEM);
+    assert(status == 1);
+    status = SSL_CTX_use_certificate_chain_file(ctx, cert_paths[0].c_str());
+    assert(status == 1);
+}
+
+} // namespace sslutil
+

luprex/cpp/drv/sslutil.hpp

@@ -0,0 +1,61 @@
+#ifndef SSLUTIL_HPP
+#define SSLUTIL_HPP
+
+#include "drvutil.hpp"
+
+#include <openssl/ssl.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/bio.h>
+#include <openssl/pem.h>
+#include <openssl/conf.h>
+
+#include <memory>
+
+namespace sslutil {
+
+struct SSL_Deleter {
+    void operator()(SSL *ssl) { SSL_free(ssl); }
+};
+
+struct CTX_Deleter {
+    void operator()(SSL_CTX *ctx) { SSL_CTX_free(ctx); }
+};
+
+struct BIO_Deleter {
+    void operator()(BIO *bio) { BIO_free(bio); }
+};
+
+struct X509_Deleter {
+    void operator()(X509 *x) { X509_free(x); }
+};
+
+struct PKEY_Deleter {
+    void operator()(EVP_PKEY *p) { EVP_PKEY_free(p); }
+};
+
+using UniqueSSL = std::unique_ptr<SSL, SSL_Deleter>;
+using UniqueCTX = std::unique_ptr<SSL_CTX, CTX_Deleter>;
+using UniqueBIO = std::unique_ptr<BIO, BIO_Deleter>;
+using UniqueX509 = std::unique_ptr<X509, X509_Deleter>;
+using UniquePKEY = std::unique_ptr<EVP_PKEY, PKEY_Deleter>;
+
+struct ErrClearErrorOnExit {
+    ~ErrClearErrorOnExit() {
+        ERR_clear_error();
+    }
+};
+
+// Return the OpenSSL error as a string.
+std::string error_string();
+void clear_all_errors();
+SSL_CTX *new_context(int verify);
+void ctx_load_dummy_cert(SSL_CTX *ctx);
+void ctx_load_cert_from_directory(SSL_CTX *ctx, const std::string &dir);
+
+} // namespace sslutil
+
+#endif // SSLUTIL_HPP
+

luprex/cpp/wrap/mkstub.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python3
+
+import sys
+
+base=sys.argv[1]
+ubase=base.upper()
+dash=base.replace("_", "-")
+
+with open(f"wrap-{dash}.hpp", "w") as f:
+    print(f"#ifndef WRAP_{ubase}_HPP", file=f)
+    print(f"#define WRAP_{ubase}_HPP", file=f)
+    print("", file=f)
+    print('#include "eng-malloc.hpp"', file=f)
+    print(f"#include <{base}>", file=f)
+    print("", file=f)
+    print("namespace eng {", file=f)
+    print("} // namespace eng", file=f)
+    print("", file=f)
+    print(f"#endif // WRAP_{ubase}_HPP", file=f)

luprex/cpp/wrap/wrap-bytell-hash-map.hpp

@@ -0,0 +1,14 @@
+#ifndef WRAP_BYTELL_HASH_MAP_HPP
+#define WRAP_BYTELL_HASH_MAP_HPP
+
+#include "eng-malloc.hpp"
+#include "bytell-hash-map.hpp"
+
+namespace eng {
+template<class K, class V, class H=std::hash<K>, class E=std::equal_to<K>>
+class bytell_hash_map : public ska::bytell_hash_map<K, V, H, E, eng::allocator<std::pair<const K, V>>>, public eng::opnew {
+    using ska::bytell_hash_map<K, V, H, E, eng::allocator<std::pair<const K, V>>>::bytell_hash_map;
+};
+} // namespace eng
+
+#endif // WRAP_BYTELL_HASH_MAP_HPP

luprex/cpp/wrap/wrap-deque.hpp

@@ -0,0 +1,14 @@
+#ifndef WRAP_DEQUE_HPP
+#define WRAP_DEQUE_HPP
+
+#include "eng-malloc.hpp"
+#include <deque>
+
+namespace eng {
+template<class T>
+class deque : public std::deque<T, eng::allocator<T>>, public eng::opnew {
+    using std::deque<T, eng::allocator<T>>::deque;
+};
+} // namespace eng
+
+#endif // WRAP_DEQUE_HPP

luprex/cpp/wrap/wrap-map.hpp

@@ -0,0 +1,14 @@
+#ifndef WRAP_MAP_HPP
+#define WRAP_MAP_HPP
+
+#include "eng-malloc.hpp"
+#include <map>
+
+namespace eng {
+template<class K, class V, class C=std::less<K>>
+class map : public std::map<K, V, C, eng::allocator<std::pair<const K, V>>>, eng::opnew {
+    using std::map<K, V, C, eng::allocator<std::pair<const K, V>>>::map;
+};
+} // namespace eng
+
+#endif // WRAP_MAP_HPP

luprex/cpp/wrap/wrap-set.hpp

@@ -0,0 +1,14 @@
+#ifndef WRAP_SET_HPP
+#define WRAP_SET_HPP
+
+#include "eng-malloc.hpp"
+#include <set>
+
+namespace eng {
+template<class K, class C=std::less<K>>
+class set : public std::set<K, C, eng::allocator<K>>, public eng::opnew {
+    using std::set<K, C, eng::allocator<K>>::set;
+};
+} // namespace eng
+
+#endif // WRAP_SET_HPP

luprex/cpp/wrap/wrap-sstream.hpp

@@ -0,0 +1,18 @@
+#ifndef WRAP_SSTREAM_HPP
+#define WRAP_SSTREAM_HPP
+
+#include "eng-malloc.hpp"
+#include "wrap-string.hpp"
+#include <sstream>
+#include <string_view>
+
+namespace eng {
+template<class C, class T=std::char_traits<C>>
+class basic_ostringstream : public std::basic_ostringstream<C, T, eng::allocator<C>>, public eng::opnew {
+    using underlying = std::basic_ostringstream<C, T, eng::allocator<C>>;
+    using underlying::basic_ostringstream;
+};
+using ostringstream = basic_ostringstream<char>;
+} // namespace eng
+
+#endif // WRAP_SSTREAM_HPP

luprex/cpp/wrap/wrap-string.hpp

@@ -0,0 +1,13 @@
+#ifndef WRAP_STRING_HPP
+#define WRAP_STRING_HPP
+
+#include "eng-malloc.hpp"
+#include <string>
+
+namespace eng {
+template<class C, class T=std::char_traits<C>>
+using basic_string = std::basic_string<C, T, eng::allocator<C>>;
+using string = basic_string<char>;
+} // namespace eng
+
+#endif // WRAP_STRING_HPP

luprex/cpp/wrap/wrap-unordered-map.hpp

@@ -0,0 +1,14 @@
+#ifndef WRAP_UNORDERED_MAP_HPP
+#define WRAP_UNORDERED_MAP_HPP
+
+#include "eng-malloc.hpp"
+#include <unordered_map>
+
+namespace eng {
+template<class K, class V, class H=std::hash<K>, class E=std::equal_to<K>>
+class unordered_map : public std::unordered_map<K, V, H, E, eng::allocator<std::pair<const K, V>>>, public eng::opnew {
+    using std::unordered_map<K, V, H, E, eng::allocator<std::pair<const K, V>>>::unordered_map;
+};
+} // namespace eng
+
+#endif // WRAP_UNORDERED_MAP_HPP

luprex/cpp/wrap/wrap-unordered-set.hpp

@@ -0,0 +1,14 @@
+#ifndef WRAP_UNORDERED_SET_HPP
+#define WRAP_UNORDERED_SET_HPP
+
+#include "eng-malloc.hpp"
+#include <unordered_set>
+
+namespace eng {
+template<class K, class H=std::hash<K>, class E=std::equal_to<K>>
+class unordered_set : public std::unordered_set<K, H, E, eng::allocator<K>>, public eng::opnew {
+    using std::unordered_set<K, H, E, eng::allocator<K>>::unordered_set;
+};
+} // namespace eng
+
+#endif // WRAP_UNORDERED_SET_HPP

luprex/cpp/wrap/wrap-vector.hpp

@@ -0,0 +1,14 @@
+#ifndef WRAP_VECTOR_HPP
+#define WRAP_VECTOR_HPP
+
+#include "eng-malloc.hpp"
+#include <vector>
+
+namespace eng {
+template<class T>
+class vector : public std::vector<T, eng::allocator<T>>, public eng::opnew {
+    using std::vector<T, eng::allocator<T>>::vector;
+};
+} // namespace eng
+
+#endif // WRAP_VECTOR_HPP