Lots of work on printbuffers

luprex/core/cpp/drivertests.cpp

@@ -1,5 +1,6 @@
 #include "drivertests.hpp"
 #include "drivenengine.hpp"
+#include "world.hpp"
 #include <iomanip>
 
 static void write_closed_message(Channel *ch, StreamBuffer *out) {
@@ -127,6 +128,21 @@ public:
     }
 };
 
+
+class RunUnitTests : public DrivenEngine {
+private:
+    UniqueWorld world_;
+
+    void event_init(int argc, char *argv[])
+    {
+        world_.reset(new World(util::WORLD_TYPE_STANDALONE));
+        world_->update_source(get_lua_source());
+        world_->run_unittests();
+        stop_driver();
+    }
+};
+
+
 UniqueDrivenEngine make_DriverListenTest() {
     return UniqueDrivenEngine(new DriverListenTest);
 }
@@ -143,4 +159,7 @@ UniqueDrivenEngine make_DriverPrintClockTest() {
     return UniqueDrivenEngine(new DriverPrintClockTest);
 }
 
+UniqueDrivenEngine make_RunUnitTests() {
+    return UniqueDrivenEngine(new RunUnitTests);
+}
 

luprex/core/cpp/drivertests.hpp

@@ -7,6 +7,7 @@ UniqueDrivenEngine make_DriverListenTest();
 UniqueDrivenEngine make_DriverWebServerTest();
 UniqueDrivenEngine make_DriverDNSFailTest();
 UniqueDrivenEngine make_DriverPrintClockTest();
+UniqueDrivenEngine make_RunUnitTests();
 
 #endif // DRIVERTESTS_HPP
 

luprex/core/cpp/main.cpp

@@ -19,6 +19,7 @@ static EngineMaker makers[] = {
     { "driverwebservertest", make_DriverWebServerTest },
     { "driverdnsfailtest", make_DriverDNSFailTest },
     { "driverprintclocktest", make_DriverPrintClockTest },
+    { "unittest", make_RunUnitTests },
     { nullptr, nullptr },
 };
 

luprex/core/cpp/printbuffer.cpp

@@ -2,11 +2,70 @@
 #include <algorithm>
 #include <sstream>
 
-PrintBuffer::PrintBuffer(util::WorldType wt) {
+struct PrintBufferCore {
-    world_type_ = wt;
+    // The most recent lines printed.
-    clear();
+    std::deque<std::string> lines_;
+
+    // Line number of the first line in the buffer.  From this, all other
+    // line numbers can be inferred.
+    int first_line_;
+
+    // Line number of the first unchecked line in the buffer.  All line
+    // numbers including this one and beyond are unchecked.
+    int first_unchecked_;
+
+    // Constructor.
+    PrintBufferCore() : first_line_(0), first_unchecked_(0) {}
+};
+
+static PrintBufferCore shared_core;
+
+PrintBuffer::PrintBuffer() {
+    core_ = &shared_core;
 }
 
+PrintBuffer::~PrintBuffer() {
+    if (core_ != &shared_core) delete core_;
+}
+
+int PrintBuffer::first_line() const {
+    return core_->first_line_;
+}
+
+int PrintBuffer::last_line() const {
+    return core_->first_line_ + core_->lines_.size();
+}
+
+int PrintBuffer::first_unchecked() const {
+    return core_->first_unchecked_;
+}
+
+bool PrintBuffer::never_printed() const {
+    return (core_->first_line_==0) && (core_->first_unchecked_==0) && (core_->lines_.size()==0);
+}
+
+const std::string &PrintBuffer::nth(int n) const {
+    return core_->lines_[n - core_->first_line_];
+}
+
+std::string PrintBuffer::debug_string() const {
+    std::ostringstream oss;
+    oss << core_->first_line_ << "," << core_->first_unchecked_ << ":";
+    for (int i = 0; i < int(core_->lines_.size()); i++) {
+        oss << core_->lines_[i] << ";";
+    }
+    return oss.str();
+}
+
+void PrintBuffer::clear() {
+    if (core_ != &shared_core) {
+        delete core_;
+        core_ = &shared_core;
+    }
+}
+
+
+
 static int first_line_len(const char *text, int len) {
     for (int i = 0; i < len; i++) {
         if (text[i] == '\n') return i;
@@ -14,58 +73,73 @@ static int first_line_len(const char *text, int len) {
     return len;
 }
 
-void PrintBuffer::add_line(const char *text, int len) {
+static void add_line(PrintBufferCore *core, const char *text, int len, bool auth) {
-    lines_.emplace_back(text, len);
+    assert(core != &shared_core);
-    if ((world_type_ == util::WORLD_TYPE_MASTER)||(world_type_ == util::WORLD_TYPE_STANDALONE)) {
+    core->lines_.emplace_back(text, len);
-        first_unchecked_ = first_line_ + int(lines_.size());
+    if (auth) {
+        core->first_unchecked_ = core->first_line_ + int(core->lines_.size());
     }
 }
 
-void PrintBuffer::add_string(const char *text, int len) {
+void PrintBuffer::add_string(const std::string &s, bool auth) {
+    const char *text = s.c_str();
+    int len = s.size();
+    if (core_ == &shared_core) core_ = new PrintBufferCore;
     while (true) {
         int fll = first_line_len(text, len);
         if (fll == len) {
             if (len > 0) {
-                add_line(text, len);
+                add_line(core_, text, len, auth);
             }
             return;
         } else {
-            add_line(text, fll);
+            add_line(core_, text, fll, auth);
             text += (fll + 1);
             len -= (fll + 1);
         }
     }
 }
 
-void PrintBuffer::add_string(const std::string &s) {
-    add_string(s.c_str(), s.size());
-}
-
 void PrintBuffer::discard_upto(int n) {
-    while ((!lines_.empty()) && (first_line_ < n)) {
+    if (core_ == &shared_core) return;
-        lines_.pop_front();
+    while ((!core_->lines_.empty()) && (core_->first_line_ < n)) {
-        first_line_ += 1;
+        core_->lines_.pop_front();
+        core_->first_line_ += 1;
     }
-    if (first_unchecked_ < first_line_) {
+    if (core_->first_unchecked_ < core_->first_line_) {
-        first_unchecked_ = first_line_;
+        core_->first_unchecked_ = core_->first_line_;
     }
 }
 
-void PrintBuffer::clear() {
+void PrintBuffer::serialize(StreamBuffer *sb) const {
-    first_line_ = 0;
+    if (never_printed()) {
-    first_unchecked_ = 0;
+        sb->write_bool(true);
-    lines_.clear();
+    } else {
+        sb->write_bool(false);
+        sb->write_uint32(core_->first_line_);
+        sb->write_uint32(core_->first_unchecked_);
+        sb->write_uint32(core_->lines_.size());
+        for (const std::string &s : core_->lines_) {
+            sb->write_string(s);
+        }
+    }
 }
 
-std::string PrintBuffer::debug_string() const {
+void PrintBuffer::deserialize(StreamBuffer *sb) {
-    std::ostringstream oss;
+    bool never_printed = sb->read_bool();
-    oss << first_line_ << "," << first_unchecked_ << ":";
+    if (never_printed) {
-    for (int i = 0; i < int(lines_.size()); i++) {
+        clear();
-        oss << lines_[i] << ";";
+    } else {
+        if (core_ == &shared_core) core_ = new PrintBufferCore;
+        core_->first_line_ = sb->read_uint32();
+        core_->first_unchecked_ = sb->read_uint32();
+        int nlines = sb->read_uint32();
+        core_->lines_.clear();
+        for (int i = 0; i < nlines; i++) {
+            core_->lines_.push_back(sb->read_string());
+        }
     }
-    return oss.str();
 }
-
 
 void PrintBuffer::diff(const PrintBuffer &auth, StreamBuffer *sb) const {
     // Currently, the implementation is simple.  The synchronous model discards
@@ -74,26 +148,27 @@ void PrintBuffer::diff(const PrintBuffer &auth, StreamBuffer *sb) const {
     // just transmission, regardless of what was in the synchronous model.  I
     // think that's okay for the text console.
     // Ask the client to discard anything that precedes auth_first.
-    sb->write_int32(auth.first_line_);
+    sb->write_int32(auth.first_line());
     sb->write_int32(auth.last_line());
-    for (int i = first_unchecked_; i < auth.last_line(); i++) {
+    for (int i = core_->first_unchecked_; i < auth.last_line(); i++) {
         std::string line;
-        if (i >= auth.first_line_) line = auth.nth(i);
+        if (i >= auth.first_line()) line = auth.nth(i);
         sb->write_string(line);
     }
 }
 
 void PrintBuffer::patch(StreamBuffer *sb) {
+    if (core_ == &shared_core) core_ = new PrintBufferCore;
     int auth_first = sb->read_int32();
     int auth_last = sb->read_int32();
-    lines_.resize(first_unchecked_ - first_line_);
+    core_->lines_.resize(core_->first_unchecked_ - core_->first_line_);
-    for (int i = first_unchecked_; i < auth_last; i++) {
+    for (int i = core_->first_unchecked_; i < auth_last; i++) {
-        lines_.emplace_back(sb->read_string());
+        core_->lines_.emplace_back(sb->read_string());
     }
-    first_unchecked_ = first_line_ + lines_.size();
+    core_->first_unchecked_ = core_->first_line_ + core_->lines_.size();
     discard_upto(auth_first);
-    if (first_line_ < auth_first) {
+    if (core_->first_line_ < auth_first) {
-        first_line_ = auth_first;
+        core_->first_line_ = auth_first;
     }
 }
 
@@ -114,39 +189,59 @@ Invocation PrintChanneler::invocation(int64_t actor_id) {
 }
 
 LuaDefine(unittests_printbuffer, "c") {
-    PrintBuffer pbm(util::WORLD_TYPE_MASTER);
+    PrintBuffer pbm;
-    PrintBuffer pbs(util::WORLD_TYPE_S_SYNC);
+    PrintBuffer pbs;
     StreamBuffer sb;
 
+    // Test authoritative add_string and discard_upto.
     LuaAssertStrEq(L, pbm.debug_string(), "0,0:");
-    pbm.add_string("foo\nbar\nbaz\n");
+    pbm.add_string("foo\nbar\nbaz\n", true);
     LuaAssertStrEq(L, pbm.debug_string(), "0,3:foo;bar;baz;");
-    pbm.add_string("a\nb\nc");
+    pbm.add_string("a\nb\nc", true);
     LuaAssertStrEq(L, pbm.debug_string(), "0,6:foo;bar;baz;a;b;c;");
     pbm.discard_upto(2);
     LuaAssertStrEq(L, pbm.debug_string(), "2,6:baz;a;b;c;");
     pbm.discard_upto(8);
     LuaAssertStrEq(L, pbm.debug_string(), "6,6:");
 
+    // Test nonauthoritative add_string and discard_upto.
     LuaAssertStrEq(L, pbs.debug_string(), "0,0:");
-    pbs.add_string("foo\nbar\nbaz\n");
+    pbs.add_string("foo\nbar\nbaz\n", false);
     LuaAssertStrEq(L, pbs.debug_string(), "0,0:foo;bar;baz;");
-    pbs.add_string("a\nb\nc");
+    pbs.add_string("a\nb\nc", false);
     LuaAssertStrEq(L, pbs.debug_string(), "0,0:foo;bar;baz;a;b;c;");
     pbs.discard_upto(2);
     LuaAssertStrEq(L, pbs.debug_string(), "2,2:baz;a;b;c;");
     pbs.discard_upto(8);
     LuaAssertStrEq(L, pbs.debug_string(), "6,6:");
 
+    // Test serialization and deserialization of a nonempty printbuffer.
+    pbm.clear();
+    sb.clear();
+    pbm.add_string("boy\nhowdy\nyeah\n", true);
+    pbm.add_string("baby\nget\ndown\n", false);
+    LuaAssertStrEq(L, pbm.debug_string(), "0,3:boy;howdy;yeah;baby;get;down;");
+    pbm.serialize(&sb);
+    pbs.deserialize(&sb);
+    LuaAssertStrEq(L, pbs.debug_string(), "0,3:boy;howdy;yeah;baby;get;down;");
+
+    // Test serialization and deserialization of an empty printbuffer.
+    pbm.clear();
+    sb.clear();
+    LuaAssert(L, pbm.never_printed());
+    pbm.serialize(&sb);
+    pbs.deserialize(&sb);
+    LuaAssert(L, pbs.never_printed());
+    
     pbm.clear();
     pbs.clear();
     sb.clear();
-    pbm.add_string("foo\nbar\n");
+    pbm.add_string("foo\nbar\n", true);
     pbs.diff(pbm, &sb);
     pbs.patch(&sb);
     LuaAssertStrEq(L, pbs.debug_string(), "0,2:foo;bar;");
     pbm.clear();
-    pbm.add_string("foo\nyow\nding\ndong\n");
+    pbm.add_string("foo\nyow\nding\ndong\n", true);
     pbs.diff(pbm, &sb);
     pbs.patch(&sb);
     LuaAssertStrEq(L, pbs.debug_string(), "0,4:foo;bar;ding;dong;");
@@ -155,12 +250,12 @@ LuaDefine(unittests_printbuffer, "c") {
     pbs.diff(pbm, &sb);
     pbs.patch(&sb);
     LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;");
-    pbs.add_string("boy\nhowdy\n");
+    pbs.add_string("boy\nhowdy\n", false);
     LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;boy;howdy;");
     pbs.diff(pbm, &sb);
     pbs.patch(&sb);
     LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;");
-    pbs.add_string("boy\nhowdy\nyeah\nbaby\nget\ndown\n");
+    pbs.add_string("boy\nhowdy\nyeah\nbaby\nget\ndown\n", false);
     LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;boy;howdy;yeah;baby;get;down;");
     pbs.discard_upto(5);
     LuaAssertStrEq(L, pbs.debug_string(), "5,5:howdy;yeah;baby;get;down;");
@@ -168,5 +263,6 @@ LuaDefine(unittests_printbuffer, "c") {
     pbs.patch(&sb);
     LuaAssertStrEq(L, pbs.debug_string(), "5,5:");
     
+
     return 0;
 }

luprex/core/cpp/printbuffer.hpp

@@ -9,8 +9,8 @@
 //
 // Class PrintBuffer is a buffer for storing the output of the print statement.
 // It is part of the actor tangible.  When a lua thread calls 'print', the print
-// goes into stringstream lthread_console.  When the thread stops or yields, the
+// goes into stringstream lthread_prints.  When the thread stops or yields, the
-// contents of lthread_console are converted to lines and transferred to the
+// contents of lthread_prints are converted to lines and transferred to the
 // PrintBuffer of the actor. From there, it gets difference transmitted, and the
 // client can probe it.
 //
@@ -19,12 +19,12 @@
 // model and the synchronous model.  When the thread finishes, the PrintBuffer
 // in the actor in the master model will contain this:
 //
-//    * Block 0: Whose woods these are I think I know.  [authoritative]
+//    * Line 0: Whose woods these are I think I know.  [authoritative]
-//    * Block 1: His house is in the village though;    [authoritative]
+//    * Line 1: His house is in the village though;    [authoritative]
-//    * Block 2: He will not see me stopping here       [authoritative]
+//    * Line 2: He will not see me stopping here       [authoritative]
-//    * Block 3: To watch his woods fill up with snow.  [authoritative]
+//    * Line 3: To watch his woods fill up with snow.  [authoritative]
-//    * Block 4: My little horse must think it queer    [authoritative]
+//    * Line 4: My little horse must think it queer    [authoritative]
-//    * Block 5: To stop without a farmhouse near.      [authoritative]
+//    * Line 5: To stop without a farmhouse near.      [authoritative]
 //
 // Note that the buffer stores line numbers, which start from zero the moment
 // the player logs in.  In the master model, all lines are always authoritative
@@ -63,6 +63,14 @@
 // processed. Therefore, the client must be prepared that it might see some
 // redundant lines for a little while.
 //
+// MEMORY EFFICIENCY.
+//
+// Every tangible will contain a printbuffer.  To avoid memory waste, the
+// implementation of PrintBuffer stores everything inside a dynamically
+// allocated "core" struct.  All printbuffers that contain nothing share a
+// common empty core.  That way, the empty printbuffers that will exist in 99% of
+// all tangibles will take up very little space.
+//
 ////////////////////////////////////////////////////////////////////////////////
 
 
@@ -77,66 +85,56 @@
 #include <memory>
 #include <ostream>
 
+struct PrintBufferCore;
+
 class PrintBuffer {
 private:
-    // The most recent lines printed.
+    PrintBufferCore *core_;
-    std::deque<std::string> lines_;
-
-    // Line number of the first line in the buffer.  From this, all other
-    // line numbers can be inferred.
-    int first_line_;
-
-    // Line number of the first unchecked line in the buffer.  All line
-    // numbers including this one and beyond are unchecked.
-    int first_unchecked_;
-    
-    // The world type of the enclosing model.  This is used to determine
-    // whether add_string increments the n_checked counter or not.
-    util::WorldType world_type_;
-
-    // Add a line of text (internal).  Line is expected to NOT contain
-    // a newline (or any other weird control characters).
-    void add_line(const char *line, int len);
-    
-
-
 
 public:
-    PrintBuffer(util::WorldType wt);
+    PrintBuffer();
+    ~PrintBuffer();
 
     // Get the current first line.
-    int first_line() const { return first_line_; }
+    int first_line() const;
     
     // Return the line number beyond the end of the buffer.
-    int last_line() const { return first_line_ + int(lines_.size()); }
+    int last_line() const;
     
     // Get the current first unchecked line.
-    int first_unchecked() const { return first_unchecked_; }
+    // Guaranteed to be between first_line and last_line inclusive.
+    int first_unchecked() const;
+    
+    // Return true if the printbuffer is in the initial state.
+    // Note: if you clear the buffer, it's back in the initial state.
+    bool never_printed() const;
 
     // Get the specified line number.
-    const std::string &nth(int n) const { return lines_[n - first_line_]; }
+    const std::string &nth(int n) const;
+    
+    // Print the entire contents of the buffer to a string (for unit testing).
+    std::string debug_string() const;
+
+    // Clear the buffer
+    void clear();
 
     // Add a string.  If the string doesn't end in a newline, a newline
     // is added.  The string is broken into lines, and the lines are added
     // to the PrintBuffer.
-    void add_string(const char *text, int len);
+    void add_string(const std::string &s, bool auth);
-    void add_string(const std::string &s);
 
     // Discard lines up to but not including line N.
     void discard_upto(int n);
 
+    // Serialization and deserialization
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
     // Difference transmission
     void diff(const PrintBuffer &auth, StreamBuffer *sb) const;
     void patch(StreamBuffer *sb);
-
-    // Clear the buffer (for unit testing)
-    void clear();
-
-    // Print the entire contents of the buffer to a string (for unit testing).
-    std::string debug_string() const;
 };
 
-using UniquePrintBuffer = std::unique_ptr<PrintBuffer>;
 
 class PrintChanneler {
 private:

luprex/core/cpp/util.cpp

@@ -238,6 +238,10 @@ double distance_squared(double x1, double y1, double x2, double y2) {
     return dx*dx + dy*dy;
 }
 
+bool world_type_authoritative(util::WorldType wt) {
+    return (wt == WORLD_TYPE_MASTER) || (wt == WORLD_TYPE_STANDALONE);
+}
+
 static std::string get_file_contents(const std::string &fn) {
     std::ifstream fs(fn);
     std::stringstream buffer;

luprex/core/cpp/util.hpp

@@ -94,6 +94,9 @@ std::string trim(std::string s);
 // Calculate distance between two points
 double distance_squared(double x1, double y1, double x2, double y2);
 
+// Return true if a world type is authoritative.
+bool world_type_authoritative(util::WorldType wt);
+
 // Remove nullptrs from a vector of unique pointers.
 template<class T>
 void remove_nullptrs(std::vector<std::unique_ptr<T>> &vec) {

luprex/core/cpp/world-core.cpp

@@ -90,15 +90,16 @@ void Tangible::update_plane_item() {
 void Tangible::serialize(StreamBuffer *sb) {
     anim_queue_.serialize(sb);
     id_player_pool_.serialize(sb);
+    print_buffer_.serialize(sb);
 }
 
 void Tangible::deserialize(StreamBuffer *sb) {
     anim_queue_.deserialize(sb);
     id_player_pool_.deserialize(sb);
+    print_buffer_.deserialize(sb);
     update_plane_item();
 }
 
-
 Tangible *World::tangible_get(int64_t id) {
     auto iter = tangibles_.find(id);
     if (iter == tangibles_.end()) {
@@ -266,7 +267,7 @@ int64_t World::create_login_actor() {
     LS.rawset(mt, "__index", classtab);
     LS.result();
     tan->configure_id_pool_for_actor();
-    tan->print_buffer_.reset(new PrintBuffer(world_type_));
+    tan->print_buffer_.clear();
     assert(stack_is_clear());
     return tan->id();
 }
@@ -381,10 +382,7 @@ void World::invoke_flush_prints(int64_t actor_id, int64_t place_id, const std::s
     if (tactor == nullptr) {
         return;
     }
-    if (tactor->print_buffer_ == nullptr) {
+    tactor->print_buffer_.discard_upto(line);
-        return;
-    }
-    tactor->print_buffer_->discard_upto(line);
     assert(stack_is_clear());
 }
 
@@ -628,7 +626,7 @@ int64_t World::alloc_id_predictable() {
 const PrintBuffer *World::get_printbuffer(int64_t actor_id) {
     Tangible *actor = tangible_get(actor_id);
     if (actor != nullptr) {
-        return actor->print_buffer_.get();
+        return &actor->print_buffer_;
     }
     return nullptr;
 }
@@ -658,13 +656,10 @@ void World::close_lthread_state() {
     // send the output to std::cerr.
     if (lthread_prints_ != nullptr) {
         const std::string &output = lthread_prints_->str();
-        PrintBuffer *pbuffer = nullptr;
         Tangible *actor = tangible_get(lthread_actor_id_);
         if (actor != nullptr) {
-            pbuffer = actor->print_buffer_.get();
+            bool auth = util::world_type_authoritative(world_type_);
-        }
+            actor->print_buffer_.add_string(output, auth);
-        if (pbuffer != nullptr) {
-            pbuffer->add_string(output);
         }
     }
     // Now clean up everything.

luprex/core/cpp/world-diffxmit.cpp

@@ -14,9 +14,11 @@ int64_t World::patch_actor(StreamBuffer *sb) {
         s_actor = tangible_make(nullptr, actor_id, "", false);
         s_actor->id_player_pool_.deserialize(sb);
         s_actor->anim_queue_.deserialize(sb);
+        s_actor->print_buffer_.deserialize(sb);
     } else {
         s_actor->id_player_pool_.patch(sb);
         s_actor->anim_queue_.patch(sb);
+        s_actor->print_buffer_.patch(sb);
     }
     s_actor->update_plane_item();
     return actor_id;
@@ -35,9 +37,11 @@ void World::diff_actor(int64_t actor_id, World *master, StreamBuffer *xsb) {
     if (s_actor == nullptr) {
         m_actor->id_player_pool_.serialize(&tsb);
         m_actor->anim_queue_.serialize(&tsb);
+        m_actor->print_buffer_.serialize(&tsb);
     } else {
         s_actor->id_player_pool_.diff(m_actor->id_player_pool_, &tsb);
         s_actor->anim_queue_.diff(m_actor->anim_queue_, &tsb);
+        s_actor->print_buffer_.diff(m_actor->print_buffer_, &tsb);
     }
 
     // Forward to client, and apply to server-synchronous.

luprex/core/cpp/world.hpp

@@ -68,7 +68,7 @@ public:
     // In non-actors, this is a null pointer.  Stores the console
     // output for this actor until it can be probed by the client.
     //
-    UniquePrintBuffer print_buffer_;
+    PrintBuffer print_buffer_;
     
     // constructor.
     //