talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lots of work on printbuffers

Browse Source
This commit is contained in:
Josh Yelon
2021-11-14 15:57:18 -05:00
parent c1c0b02926
commit f933f451ad
10 changed files with 229 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
luprex/core/cpp/drivertests.cpp
View File

@@ -1,5 +1,6 @@
#include "drivertests.hpp"
#include "drivenengine.hpp"
#include "world.hpp"
#include <iomanip>
static void write_closed_message(Channel *ch, StreamBuffer *out) {
@@ -127,6 +128,21 @@ public:
}
};
class RunUnitTests : public DrivenEngine {
private:
UniqueWorld world_;
void event_init(int argc, char *argv[])
{
world_.reset(new World(util::WORLD_TYPE_STANDALONE));
world_->update_source(get_lua_source());
world_->run_unittests();
stop_driver();
}
};
UniqueDrivenEngine make_DriverListenTest() {
return UniqueDrivenEngine(new DriverListenTest);
}
@@ -143,4 +159,7 @@ UniqueDrivenEngine make_DriverPrintClockTest() {
return UniqueDrivenEngine(new DriverPrintClockTest);
}
UniqueDrivenEngine make_RunUnitTests() {
return UniqueDrivenEngine(new RunUnitTests);
}

1
luprex/core/cpp/drivertests.hpp
View File

@@ -7,6 +7,7 @@ UniqueDrivenEngine make_DriverListenTest();
UniqueDrivenEngine make_DriverWebServerTest();
UniqueDrivenEngine make_DriverDNSFailTest();
UniqueDrivenEngine make_DriverPrintClockTest();
UniqueDrivenEngine make_RunUnitTests();
#endif // DRIVERTESTS_HPP

1
luprex/core/cpp/main.cpp
View File

@@ -19,6 +19,7 @@ static EngineMaker makers[] = {
{ "driverwebservertest", make_DriverWebServerTest },
{ "driverdnsfailtest", make_DriverDNSFailTest },
{ "driverprintclocktest", make_DriverPrintClockTest },
{ "unittest", make_RunUnitTests },
{ nullptr, nullptr },
};

196
luprex/core/cpp/printbuffer.cpp
View File

@@ -2,11 +2,70 @@
#include <algorithm>
#include <sstream>
PrintBuffer::PrintBuffer(util::WorldType wt) {
world_type_ = wt;
clear();
struct PrintBufferCore {
// The most recent lines printed.
std::deque<std::string> lines_;
// Line number of the first line in the buffer. From this, all other
// line numbers can be inferred.
int first_line_;
// Line number of the first unchecked line in the buffer. All line
// numbers including this one and beyond are unchecked.
int first_unchecked_;
// Constructor.
PrintBufferCore() : first_line_(0), first_unchecked_(0) {}
};
static PrintBufferCore shared_core;
PrintBuffer::PrintBuffer() {
core_ = &shared_core;
}
PrintBuffer::~PrintBuffer() {
if (core_ != &shared_core) delete core_;
}
int PrintBuffer::first_line() const {
return core_->first_line_;
}
int PrintBuffer::last_line() const {
return core_->first_line_ + core_->lines_.size();
}
int PrintBuffer::first_unchecked() const {
return core_->first_unchecked_;
}
bool PrintBuffer::never_printed() const {
return (core_->first_line_==0) && (core_->first_unchecked_==0) && (core_->lines_.size()==0);
}
const std::string &PrintBuffer::nth(int n) const {
return core_->lines_[n - core_->first_line_];
}
std::string PrintBuffer::debug_string() const {
std::ostringstream oss;
oss << core_->first_line_ << "," << core_->first_unchecked_ << ":";
for (int i = 0; i < int(core_->lines_.size()); i++) {
oss << core_->lines_[i] << ";";
}
return oss.str();
}
void PrintBuffer::clear() {
if (core_ != &shared_core) {
delete core_;
core_ = &shared_core;
}
}
static int first_line_len(const char *text, int len) {
for (int i = 0; i < len; i++) {
if (text[i] == '\n') return i;
@@ -14,58 +73,73 @@ static int first_line_len(const char *text, int len) {
return len;
}
void PrintBuffer::add_line(const char *text, int len) {
lines_.emplace_back(text, len);
if ((world_type_ == util::WORLD_TYPE_MASTER)||(world_type_ == util::WORLD_TYPE_STANDALONE)) {
first_unchecked_ = first_line_ + int(lines_.size());
static void add_line(PrintBufferCore *core, const char *text, int len, bool auth) {
assert(core != &shared_core);
core->lines_.emplace_back(text, len);
if (auth) {
core->first_unchecked_ = core->first_line_ + int(core->lines_.size());
}
}
void PrintBuffer::add_string(const char *text, int len) {
void PrintBuffer::add_string(const std::string &s, bool auth) {
const char *text = s.c_str();
int len = s.size();
if (core_ == &shared_core) core_ = new PrintBufferCore;
while (true) {
int fll = first_line_len(text, len);
if (fll == len) {
if (len > 0) {
add_line(text, len);
add_line(core_, text, len, auth);
}
return;
} else {
add_line(text, fll);
add_line(core_, text, fll, auth);
text += (fll + 1);
len -= (fll + 1);
}
}
}
void PrintBuffer::add_string(const std::string &s) {
add_string(s.c_str(), s.size());
}
void PrintBuffer::discard_upto(int n) {
while ((!lines_.empty()) && (first_line_ < n)) {
lines_.pop_front();
first_line_ += 1;
if (core_ == &shared_core) return;
while ((!core_->lines_.empty()) && (core_->first_line_ < n)) {
core_->lines_.pop_front();
core_->first_line_ += 1;
}
if (first_unchecked_ < first_line_) {
first_unchecked_ = first_line_;
if (core_->first_unchecked_ < core_->first_line_) {
core_->first_unchecked_ = core_->first_line_;
}
}
void PrintBuffer::clear() {
first_line_ = 0;
first_unchecked_ = 0;
lines_.clear();
}
std::string PrintBuffer::debug_string() const {
std::ostringstream oss;
oss << first_line_ << "," << first_unchecked_ << ":";
for (int i = 0; i < int(lines_.size()); i++) {
oss << lines_[i] << ";";
void PrintBuffer::serialize(StreamBuffer *sb) const {
if (never_printed()) {
sb->write_bool(true);
} else {
sb->write_bool(false);
sb->write_uint32(core_->first_line_);
sb->write_uint32(core_->first_unchecked_);
sb->write_uint32(core_->lines_.size());
for (const std::string &s : core_->lines_) {
sb->write_string(s);
}
}
return oss.str();
}
void PrintBuffer::deserialize(StreamBuffer *sb) {
bool never_printed = sb->read_bool();
if (never_printed) {
clear();
} else {
if (core_ == &shared_core) core_ = new PrintBufferCore;
core_->first_line_ = sb->read_uint32();
core_->first_unchecked_ = sb->read_uint32();
int nlines = sb->read_uint32();
core_->lines_.clear();
for (int i = 0; i < nlines; i++) {
core_->lines_.push_back(sb->read_string());
}
}
}
void PrintBuffer::diff(const PrintBuffer &auth, StreamBuffer *sb) const {
// Currently, the implementation is simple. The synchronous model discards
@@ -74,26 +148,27 @@ void PrintBuffer::diff(const PrintBuffer &auth, StreamBuffer *sb) const {
// just transmission, regardless of what was in the synchronous model. I
// think that's okay for the text console.
// Ask the client to discard anything that precedes auth_first.
sb->write_int32(auth.first_line_);
sb->write_int32(auth.first_line());
sb->write_int32(auth.last_line());
for (int i = first_unchecked_; i < auth.last_line(); i++) {
for (int i = core_->first_unchecked_; i < auth.last_line(); i++) {
std::string line;
if (i >= auth.first_line_) line = auth.nth(i);
if (i >= auth.first_line()) line = auth.nth(i);
sb->write_string(line);
}
}
void PrintBuffer::patch(StreamBuffer *sb) {
if (core_ == &shared_core) core_ = new PrintBufferCore;
int auth_first = sb->read_int32();
int auth_last = sb->read_int32();
lines_.resize(first_unchecked_ - first_line_);
for (int i = first_unchecked_; i < auth_last; i++) {
lines_.emplace_back(sb->read_string());
core_->lines_.resize(core_->first_unchecked_ - core_->first_line_);
for (int i = core_->first_unchecked_; i < auth_last; i++) {
core_->lines_.emplace_back(sb->read_string());
}
first_unchecked_ = first_line_ + lines_.size();
core_->first_unchecked_ = core_->first_line_ + core_->lines_.size();
discard_upto(auth_first);
if (first_line_ < auth_first) {
first_line_ = auth_first;
if (core_->first_line_ < auth_first) {
core_->first_line_ = auth_first;
}
}
@@ -114,39 +189,59 @@ Invocation PrintChanneler::invocation(int64_t actor_id) {
}
LuaDefine(unittests_printbuffer, "c") {
PrintBuffer pbm(util::WORLD_TYPE_MASTER);
PrintBuffer pbs(util::WORLD_TYPE_S_SYNC);
PrintBuffer pbm;
PrintBuffer pbs;
StreamBuffer sb;
// Test authoritative add_string and discard_upto.
LuaAssertStrEq(L, pbm.debug_string(), "0,0:");
pbm.add_string("foo\nbar\nbaz\n");
pbm.add_string("foo\nbar\nbaz\n", true);
LuaAssertStrEq(L, pbm.debug_string(), "0,3:foo;bar;baz;");
pbm.add_string("a\nb\nc");
pbm.add_string("a\nb\nc", true);
LuaAssertStrEq(L, pbm.debug_string(), "0,6:foo;bar;baz;a;b;c;");
pbm.discard_upto(2);
LuaAssertStrEq(L, pbm.debug_string(), "2,6:baz;a;b;c;");
pbm.discard_upto(8);
LuaAssertStrEq(L, pbm.debug_string(), "6,6:");
// Test nonauthoritative add_string and discard_upto.
LuaAssertStrEq(L, pbs.debug_string(), "0,0:");
pbs.add_string("foo\nbar\nbaz\n");
pbs.add_string("foo\nbar\nbaz\n", false);
LuaAssertStrEq(L, pbs.debug_string(), "0,0:foo;bar;baz;");
pbs.add_string("a\nb\nc");
pbs.add_string("a\nb\nc", false);
LuaAssertStrEq(L, pbs.debug_string(), "0,0:foo;bar;baz;a;b;c;");
pbs.discard_upto(2);
LuaAssertStrEq(L, pbs.debug_string(), "2,2:baz;a;b;c;");
pbs.discard_upto(8);
LuaAssertStrEq(L, pbs.debug_string(), "6,6:");
// Test serialization and deserialization of a nonempty printbuffer.
pbm.clear();
sb.clear();
pbm.add_string("boy\nhowdy\nyeah\n", true);
pbm.add_string("baby\nget\ndown\n", false);
LuaAssertStrEq(L, pbm.debug_string(), "0,3:boy;howdy;yeah;baby;get;down;");
pbm.serialize(&sb);
pbs.deserialize(&sb);
LuaAssertStrEq(L, pbs.debug_string(), "0,3:boy;howdy;yeah;baby;get;down;");
// Test serialization and deserialization of an empty printbuffer.
pbm.clear();
sb.clear();
LuaAssert(L, pbm.never_printed());
pbm.serialize(&sb);
pbs.deserialize(&sb);
LuaAssert(L, pbs.never_printed());
pbm.clear();
pbs.clear();
sb.clear();
pbm.add_string("foo\nbar\n");
pbm.add_string("foo\nbar\n", true);
pbs.diff(pbm, &sb);
pbs.patch(&sb);
LuaAssertStrEq(L, pbs.debug_string(), "0,2:foo;bar;");
pbm.clear();
pbm.add_string("foo\nyow\nding\ndong\n");
pbm.add_string("foo\nyow\nding\ndong\n", true);
pbs.diff(pbm, &sb);
pbs.patch(&sb);
LuaAssertStrEq(L, pbs.debug_string(), "0,4:foo;bar;ding;dong;");
@@ -155,12 +250,12 @@ LuaDefine(unittests_printbuffer, "c") {
pbs.diff(pbm, &sb);
pbs.patch(&sb);
LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;");
pbs.add_string("boy\nhowdy\n");
pbs.add_string("boy\nhowdy\n", false);
LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;boy;howdy;");
pbs.diff(pbm, &sb);
pbs.patch(&sb);
LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;");
pbs.add_string("boy\nhowdy\nyeah\nbaby\nget\ndown\n");
pbs.add_string("boy\nhowdy\nyeah\nbaby\nget\ndown\n", false);
LuaAssertStrEq(L, pbs.debug_string(), "2,4:ding;dong;boy;howdy;yeah;baby;get;down;");
pbs.discard_upto(5);
LuaAssertStrEq(L, pbs.debug_string(), "5,5:howdy;yeah;baby;get;down;");
@@ -168,5 +263,6 @@ LuaDefine(unittests_printbuffer, "c") {
pbs.patch(&sb);
LuaAssertStrEq(L, pbs.debug_string(), "5,5:");
return 0;
}

88
luprex/core/cpp/printbuffer.hpp
View File

@@ -9,8 +9,8 @@
//
// Class PrintBuffer is a buffer for storing the output of the print statement.
// It is part of the actor tangible. When a lua thread calls 'print', the print
// goes into stringstream lthread_console. When the thread stops or yields, the
// contents of lthread_console are converted to lines and transferred to the
// goes into stringstream lthread_prints. When the thread stops or yields, the
// contents of lthread_prints are converted to lines and transferred to the
// PrintBuffer of the actor. From there, it gets difference transmitted, and the
// client can probe it.
//
@@ -19,12 +19,12 @@
// model and the synchronous model. When the thread finishes, the PrintBuffer
// in the actor in the master model will contain this:
//
// * Block 0: Whose woods these are I think I know. [authoritative]
// * Block 1: His house is in the village though; [authoritative]
// * Block 2: He will not see me stopping here [authoritative]
// * Block 3: To watch his woods fill up with snow. [authoritative]
// * Block 4: My little horse must think it queer [authoritative]
// * Block 5: To stop without a farmhouse near. [authoritative]
// * Line 0: Whose woods these are I think I know. [authoritative]
// * Line 1: His house is in the village though; [authoritative]
// * Line 2: He will not see me stopping here [authoritative]
// * Line 3: To watch his woods fill up with snow. [authoritative]
// * Line 4: My little horse must think it queer [authoritative]
// * Line 5: To stop without a farmhouse near. [authoritative]
//
// Note that the buffer stores line numbers, which start from zero the moment
// the player logs in. In the master model, all lines are always authoritative
@@ -63,6 +63,14 @@
// processed. Therefore, the client must be prepared that it might see some
// redundant lines for a little while.
//
// MEMORY EFFICIENCY.
//
// Every tangible will contain a printbuffer. To avoid memory waste, the
// implementation of PrintBuffer stores everything inside a dynamically
// allocated "core" struct. All printbuffers that contain nothing share a
// common empty core. That way, the empty printbuffers that will exist in 99% of
// all tangibles will take up very little space.
//
////////////////////////////////////////////////////////////////////////////////
@@ -77,66 +85,56 @@
#include <memory>
#include <ostream>
struct PrintBufferCore;
class PrintBuffer {
private:
// The most recent lines printed.
std::deque<std::string> lines_;
// Line number of the first line in the buffer. From this, all other
// line numbers can be inferred.
int first_line_;
// Line number of the first unchecked line in the buffer. All line
// numbers including this one and beyond are unchecked.
int first_unchecked_;
// The world type of the enclosing model. This is used to determine
// whether add_string increments the n_checked counter or not.
util::WorldType world_type_;
// Add a line of text (internal). Line is expected to NOT contain
// a newline (or any other weird control characters).
void add_line(const char *line, int len);
PrintBufferCore *core_;
public:
PrintBuffer(util::WorldType wt);
PrintBuffer();
~PrintBuffer();
// Get the current first line.
int first_line() const { return first_line_; }
int first_line() const;
// Return the line number beyond the end of the buffer.
int last_line() const { return first_line_ + int(lines_.size()); }
int last_line() const;
// Get the current first unchecked line.
int first_unchecked() const { return first_unchecked_; }
// Guaranteed to be between first_line and last_line inclusive.
int first_unchecked() const;
// Return true if the printbuffer is in the initial state.
// Note: if you clear the buffer, it's back in the initial state.
bool never_printed() const;
// Get the specified line number.
const std::string &nth(int n) const { return lines_[n - first_line_]; }
const std::string &nth(int n) const;
// Print the entire contents of the buffer to a string (for unit testing).
std::string debug_string() const;
// Clear the buffer
void clear();
// Add a string. If the string doesn't end in a newline, a newline
// is added. The string is broken into lines, and the lines are added
// to the PrintBuffer.
void add_string(const char *text, int len);
void add_string(const std::string &s);
void add_string(const std::string &s, bool auth);
// Discard lines up to but not including line N.
void discard_upto(int n);
// Serialization and deserialization
void serialize(StreamBuffer *sb) const;
void deserialize(StreamBuffer *sb);
// Difference transmission
void diff(const PrintBuffer &auth, StreamBuffer *sb) const;
void patch(StreamBuffer *sb);
// Clear the buffer (for unit testing)
void clear();
// Print the entire contents of the buffer to a string (for unit testing).
std::string debug_string() const;
};
using UniquePrintBuffer = std::unique_ptr<PrintBuffer>;
class PrintChanneler {
private:

4
luprex/core/cpp/util.cpp
View File

@@ -238,6 +238,10 @@ double distance_squared(double x1, double y1, double x2, double y2) {
return dx*dx + dy*dy;
}
bool world_type_authoritative(util::WorldType wt) {
return (wt == WORLD_TYPE_MASTER) || (wt == WORLD_TYPE_STANDALONE);
}
static std::string get_file_contents(const std::string &fn) {
std::ifstream fs(fn);
std::stringstream buffer;

3
luprex/core/cpp/util.hpp
View File

@@ -94,6 +94,9 @@ std::string trim(std::string s);
// Calculate distance between two points
double distance_squared(double x1, double y1, double x2, double y2);
// Return true if a world type is authoritative.
bool world_type_authoritative(util::WorldType wt);
// Remove nullptrs from a vector of unique pointers.
template<class T>
void remove_nullptrs(std::vector<std::unique_ptr<T>> &vec) {

19
luprex/core/cpp/world-core.cpp
View File

@@ -90,15 +90,16 @@ void Tangible::update_plane_item() {
void Tangible::serialize(StreamBuffer *sb) {
anim_queue_.serialize(sb);
id_player_pool_.serialize(sb);
print_buffer_.serialize(sb);
}
void Tangible::deserialize(StreamBuffer *sb) {
anim_queue_.deserialize(sb);
id_player_pool_.deserialize(sb);
print_buffer_.deserialize(sb);
update_plane_item();
}
Tangible *World::tangible_get(int64_t id) {
auto iter = tangibles_.find(id);
if (iter == tangibles_.end()) {
@@ -266,7 +267,7 @@ int64_t World::create_login_actor() {
LS.rawset(mt, "__index", classtab);
LS.result();
tan->configure_id_pool_for_actor();
tan->print_buffer_.reset(new PrintBuffer(world_type_));
tan->print_buffer_.clear();
assert(stack_is_clear());
return tan->id();
}
@@ -381,10 +382,7 @@ void World::invoke_flush_prints(int64_t actor_id, int64_t place_id, const std::s
if (tactor == nullptr) {
return;
}
if (tactor->print_buffer_ == nullptr) {
return;
}
tactor->print_buffer_->discard_upto(line);
tactor->print_buffer_.discard_upto(line);
assert(stack_is_clear());
}
@@ -628,7 +626,7 @@ int64_t World::alloc_id_predictable() {
const PrintBuffer *World::get_printbuffer(int64_t actor_id) {
Tangible *actor = tangible_get(actor_id);
if (actor != nullptr) {
return actor->print_buffer_.get();
return &actor->print_buffer_;
}
return nullptr;
}
@@ -658,13 +656,10 @@ void World::close_lthread_state() {
// send the output to std::cerr.
if (lthread_prints_ != nullptr) {
const std::string &output = lthread_prints_->str();
PrintBuffer *pbuffer = nullptr;
Tangible *actor = tangible_get(lthread_actor_id_);
if (actor != nullptr) {
pbuffer = actor->print_buffer_.get();
}
if (pbuffer != nullptr) {
pbuffer->add_string(output);
bool auth = util::world_type_authoritative(world_type_);
actor->print_buffer_.add_string(output, auth);
}
}
// Now clean up everything.

4
luprex/core/cpp/world-diffxmit.cpp
View File

@@ -14,9 +14,11 @@ int64_t World::patch_actor(StreamBuffer *sb) {
s_actor = tangible_make(nullptr, actor_id, "", false);
s_actor->id_player_pool_.deserialize(sb);
s_actor->anim_queue_.deserialize(sb);
s_actor->print_buffer_.deserialize(sb);
} else {
s_actor->id_player_pool_.patch(sb);
s_actor->anim_queue_.patch(sb);
s_actor->print_buffer_.patch(sb);
}
s_actor->update_plane_item();
return actor_id;
@@ -35,9 +37,11 @@ void World::diff_actor(int64_t actor_id, World *master, StreamBuffer *xsb) {
if (s_actor == nullptr) {
m_actor->id_player_pool_.serialize(&tsb);
m_actor->anim_queue_.serialize(&tsb);
m_actor->print_buffer_.serialize(&tsb);
} else {
s_actor->id_player_pool_.diff(m_actor->id_player_pool_, &tsb);
s_actor->anim_queue_.diff(m_actor->anim_queue_, &tsb);
s_actor->print_buffer_.diff(m_actor->print_buffer_, &tsb);
}
// Forward to client, and apply to server-synchronous.

2
luprex/core/cpp/world.hpp
View File

@@ -68,7 +68,7 @@ public:
// In non-actors, this is a null pointer. Stores the console
// output for this actor until it can be probed by the client.
//
UniquePrintBuffer print_buffer_;
PrintBuffer print_buffer_;
// constructor.
//