talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

difference transmission of source databases done

Browse Source
This commit is contained in:
Josh Yelon
2021-09-23 12:40:21 -04:00
parent 6fbe700107
commit 6490db43fc
8 changed files with 320 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
luprex/core/cpp/luastack.cpp
View File

@@ -263,11 +263,18 @@ std::string LuaStack::classname(LuaSlot tab) {
lua_pushstring(L_, "__class");
lua_rawget(L_, tab);
if (lua_type(L_, -1) == LUA_TSTRING) {
size_t len;
const char *s = lua_tolstring(L_, -1, &len);
result = std::string(s, len);
lua_pushglobaltable(L_); // cname table
lua_pushvalue(L_, -2); // cname table cname
lua_rawget(L_, -2); // cname table ctab
if (lua_rawequal(L_, -1, tab)) {
size_t len;
const char *s = lua_tolstring(L_, -3, &len);
result = std::string(s, len);
}
lua_pop(L_, 3);
} else {
lua_pop(L_, 1);
}
lua_pop(L_, 1);
}
return result;
}

284
luprex/core/cpp/source.cpp
View File

@@ -13,6 +13,7 @@
#include "traceback.hpp"
#include "table.hpp"
#include "source.hpp"
#include "luasnap.hpp"
// Read control.lst
//
@@ -40,27 +41,16 @@ LuaDefine(source_makeclass, "f") {
return LS.result();
}
LuaDefine(source_isclass, "f") {
LuaDefine(source_classname, "f") {
LuaArg table;
LuaVar classname, globtab, classtab;
LuaRet result;
LuaStack LS(L, table, classname, globtab, classtab, result);
if (!LS.istable(table)) {
LS.set(result, false);
return LS.result();
LuaStack LS(L, table, result);
std::string rstr = LS.classname(table);
if (rstr == "") {
LS.set(result, LuaNil);
} else {
LS.set(result, rstr);
}
LS.rawget(classname, table, "__class");
if (!LS.isstring(classname)) {
LS.set(result, false);
return LS.result();
}
LS.getglobaltable(globtab);
LS.rawget(classtab, globtab, classname);
if (!LS.rawequal(classtab, table)) {
LS.set(result, false);
return LS.result();
}
LS.set(result, true);
return LS.result();
}
@@ -91,6 +81,7 @@ static void load_builtin_class(lua_State *L, const char *name, lua_CFunction fun
while (LS.next(sourcetab, key, value) != 0) {
LS.rawset(classtab, key, value);
}
LS.result();
}
static void source_install_builtins(lua_State *L) {
@@ -113,12 +104,36 @@ static void source_install_builtins(lua_State *L) {
LS.makeclass(stringclass, "string");
LS.set(nullstring, "");
LS.setmetatable(nullstring, stringclass);
LS.result();
}
static void get_info_table(LuaStack &LS, LuaSlot db, LuaSlot info, const std::string &fn) {
LS.rawget(info, db, fn);
if (!LS.istable(info)) {
LS.set(info, LuaNewTable);
LS.rawset(db, fn, info);
}
LS.rawset(info, "name", fn);
}
static void calculate_loadresult(LuaStack &LS0, LuaSlot info, const std::string &fn, const std::string &code) {
LuaVar loadresult;
LuaStack LS(LS0.state(), loadresult);
if (code == "") {
LS.rawset(info, "loadresult", "missing or empty source file");
} else {
std::string chunk = "=" + fn;
luaL_loadbuffer(LS.state(), code.c_str(), code.size(), chunk.c_str());
lua_replace(LS.state(), loadresult.index());
LS.rawset(info, "loadresult", loadresult);
}
}
static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot info) {
lua_State *L = LS0.state();
LuaVar fingerprint, null, loadresult;
LuaStack LS(L, fingerprint, null, loadresult);
LuaVar fingerprint;
LuaStack LS(L, fingerprint);
// Get the existing info table from the source DB.
if (LS.istable(source)) {
@@ -131,7 +146,7 @@ static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot
}
// If the file modification is wrong, update
// these fields: code, fingerprint, closure, error
// these fields: code, hash, fingerprint, closure, error
// Otherwise, update nothing.
std::string cfn = LS.ckstring(fn);
LS.rawget(fingerprint, info, "fingerprint");
@@ -141,23 +156,143 @@ static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot
}
// std::cerr << "Probing " << cfn << std::endl;
std::string new_fingerprint = util::get_file_fingerprint("lua/" + cfn);
LS.set(null, LuaNil);
if ((old_fingerprint == "") || (old_fingerprint != new_fingerprint)) {
std::cerr << "Rereading " << cfn << std::endl;
std::string ccode = util::get_file_contents("lua/" + cfn);
std::string code;
if (new_fingerprint != "") code = util::get_file_contents("lua/" + cfn);
LS.rawset(info, "name", fn);
LS.rawset(info, "fingerprint", new_fingerprint);
LS.rawset(info, "code", ccode);
if ((new_fingerprint == "")||(ccode == "")) {
LS.rawset(info, "loadresult", "cannot read source file");
LS.rawset(info, "code", code);
LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
calculate_loadresult(LS, info, cfn, code);
}
LS.result();
}
void SourceDB::diff(const SourceDB &auth, StreamBuffer *sb) {
LuaVar sdb, sfn, sinfo, shash, sseq;
LuaVar mdb, mfn, minfo, mhash, mseq, mcode;
LuaStack SLS(lua_state_, sdb, sfn, sinfo, shash, sseq);
LuaStack MLS(auth.lua_state_, mdb, mfn, minfo, mhash, mseq, mcode);
sb->write_int32(0);
int wc_after = sb->total_writes();
int nupdates = 0;
// Fetch the two source databases.
SLS.rawget(sdb, LuaRegistry, "sourcedb");
MLS.rawget(mdb, LuaRegistry, "sourcedb");
// Loop over the master database.
MLS.set(mfn, LuaNil);
while (MLS.next(mdb, mfn, minfo) != 0) {
std::string fn = MLS.ckstring(mfn);
assert(MLS.istable(minfo));
MLS.rawget(mseq, minfo, "sequence");
MLS.rawget(mhash, minfo, "hash");
bool diffhash = true;
bool diffseq = true;
SLS.set(sfn, fn);
SLS.rawget(sinfo, sdb, sfn);
if (SLS.istable(sinfo)) {
SLS.rawget(shash, sinfo, "hash");
SLS.rawget(sseq, sinfo, "sequence");
diffhash = MLS.ckstring(mhash) != SLS.ckstring(shash);
diffseq = MLS.ckinteger(mseq) != SLS.ckinteger(sseq);
}
if (diffhash || diffseq) {
sb->write_string(MLS.ckstring(mfn));
sb->write_int32(MLS.ckinteger(mseq));
if (diffhash) {
MLS.rawget(mcode, minfo, "code");
sb->write_string(MLS.ckstring(mcode));
} else {
sb->write_string("\001");
}
nupdates += 1;
}
}
// Loop over synch database.
SLS.set(sfn, LuaNil);
while (SLS.next(sdb, sfn, sinfo) != 0) {
std::string fn = SLS.ckstring(sfn);
assert(SLS.istable(sinfo));
MLS.set(mfn, fn);
MLS.rawget(minfo, mdb, mfn);
if (!MLS.istable(minfo)) {
sb->write_string(SLS.ckstring(sfn));
sb->write_int32(-1);
sb->write_string("");
nupdates += 1;
}
}
sb->overwrite_int32(wc_after, nupdates);
MLS.result();
SLS.result();
}
void SourceDB::patch(StreamBuffer *sb) {
lua_State *L = lua_state_;
LuaVar db, info;
LuaStack LS(L, db, info);
LS.rawget(db, LuaRegistry, "sourcedb");
int nupdates = sb->read_int32();
for (int i = 0; i < nupdates; i++) {
std::string fn = sb->read_string();
int sequence = sb->read_int32();
std::string code = sb->read_string();
if (sequence < 0) {
LS.rawset(db, fn, LuaNil);
} else {
std::string chunk = "=" + cfn;
luaL_loadbuffer(L, ccode.c_str(), ccode.size(), chunk.c_str());
lua_replace(L, loadresult.index());
LS.rawset(info, "loadresult", loadresult);
get_info_table(LS, db, info, fn);
LS.rawset(info, "sequence", sequence);
if (code != "\001") {
LS.rawset(info, "code", code);
LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
calculate_loadresult(LS, info, fn, code);
}
}
}
LS.result();
if (nupdates > 0) rebuild(false);
}
void SourceDB::set(const std::string &fn, const std::string &code, int sequence) {
lua_State *L = lua_state_;
LuaVar db, info;
LuaStack LS(L, db, info);
LS.rawget(db, LuaRegistry, "sourcedb");
get_info_table(LS, db, info, fn);
LS.rawset(info, "sequence", sequence);
LS.rawset(info, "code", code);
LS.rawset(info, "fingerprint", "");
LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
calculate_loadresult(LS, info, fn, code);
LS.result();
}
std::string SourceDB::get(const std::string &fn) {
lua_State *L = lua_state_;
LuaVar db, info, code, sequence, loadresult;
LuaStack LS(L, db, info, code, sequence, loadresult);
LS.rawget(db, LuaRegistry, "sourcedb");
LS.rawget(info, db, fn);
if (!LS.istable(info)) {
LS.result();
return "<nonexistent>";
}
LS.rawget(code, info, "code");
LS.rawget(sequence, info, "sequence");
LS.rawget(loadresult, info, "loadresult");
std::string ccode = LS.ckstring(code);
int seqno = LS.ckint(sequence);
std::string cloadresult;
if (LS.isfunction(loadresult)) {
cloadresult = "<function>";
} else {
cloadresult = LS.ckstring(loadresult);
}
std::ostringstream oss;
oss << seqno << ":" << ccode << ":" << cloadresult;
LS.result();
return oss.str();
}
void SourceDB::update() {
@@ -250,7 +385,7 @@ static void source_load_cfunctions(lua_State *L) {
// Run all the closures from the source database.
//
static void source_load_lfunctions(lua_State *L) {
static void source_load_lfunctions(lua_State *L, bool print_errors) {
LuaRet errors;
LuaVar sourcedb, key, info, seq, closure, err;
LuaStack LS(L, sourcedb, errors, key, info, seq, closure, err);
@@ -278,7 +413,9 @@ static void source_load_lfunctions(lua_State *L) {
// If there's already an error in the sourcedb, collect it.
if (!LS.isfunction(closure)) {
errss << LS.ckstring(closure) << "\n";
if (print_errors) {
errss << LS.ckstring(closure) << "\n";
}
continue;
}
@@ -293,14 +430,14 @@ static void source_load_lfunctions(lua_State *L) {
LS.result();
}
void SourceDB::rebuild() {
void SourceDB::rebuild(bool print_errors) {
lua_State *L = lua_state_;
LuaVar errs;
LuaStack LS(L, errs);
source_clear_globals(L);
source_install_builtins(L);
source_load_cfunctions(L);
source_load_lfunctions(L);
source_load_lfunctions(L, print_errors);
lua_replace(L, errs.index());
std::string errstr = LS.ckstring(errs);
std::cerr << errstr;
@@ -356,6 +493,7 @@ void SourceDB::init(lua_State *L) {
LS.getglobaltable(globtab);
LS.rawget(persist, LuaRegistry, "persist");
LS.rawget(unpersist, LuaRegistry, "unpersist");
LS.rawset(LuaRegistry, "sourcedb", LuaNewTable);
LS.set(classname, LuaNil);
while (LS.next(globtab, classname, classtab) != 0) {
if (LS.isstring(classname) && LS.istable(classtab)) {
@@ -394,3 +532,81 @@ LuaDefine(coroutine_getnextid, "c") {
LS.set(lid, lua_getnextid(CO));
return LS.result();
}
LuaDefine(unittests_sourcedb, "c") {
LuaSnap msnap;
LuaSnap ssnap;
SourceDB mdb;
SourceDB sdb;
mdb.init(msnap.state());
sdb.init(ssnap.state());
StreamBuffer sb;
// Create a code database using 'set'.
mdb.set("foo", "function foo() print('foo') end", 1);
mdb.set("bar", "function bar() print('bar') end", 2);
mdb.set("zoo", "funcjdshja mxooso yowza!", 3);
// Verify that get works.
LuaAssertStrEq(L, mdb.get("foo"), "1:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, mdb.get("bar"), "2:function bar() print('bar') end:<function>");
LuaAssertStrEq(L, mdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, mdb.get("baz"), "<nonexistent>");
// Difference transmit.
sdb.diff(mdb, &sb);
// There should still be nothing in the sdb.
LuaAssertStrEq(L, sdb.get("foo"), "<nonexistent>");
LuaAssertStrEq(L, sdb.get("bar"), "<nonexistent>");
LuaAssertStrEq(L, sdb.get("zoo"), "<nonexistent>");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
// Apply the diffs.
sdb.patch(&sb);
// Everything should now be copied to sdb.
LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, sdb.get("bar"), "2:function bar() print('bar') end:<function>");
LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
// Make a single change to the code.
mdb.set("bar", "function bar1() print('bar1') end", 2);
// Diff and patch
sdb.diff(mdb, &sb);
sdb.patch(&sb);
// Verify that it's been updated.
LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:<function>");
LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
// Make a single change to just a sequence number.
mdb.set("foo", "function foo() print('foo') end", 6);
// Diff
sdb.diff(mdb, &sb);
// The only thing we've updated is a single sequence number. Make
// sure the number of bytes transmitted is reasonable.
// 4 bytes for the count of changes
// 4 bytes for "foo"
// 4 bytes for the sequence number
// 2 bytes for unmodified code
// This verifies that the hash comparisons are working.
LuaAssert(L, sb.fill() == 14);
// Patch.
sdb.patch(&sb);
// Verify that it's been updated.
LuaAssertStrEq(L, sdb.get("foo"), "6:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:<function>");
LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
return 0;
}

22
luprex/core/cpp/source.hpp
View File

@@ -121,6 +121,7 @@
#define SOURCE_HPP
#include "luastack.hpp"
#include "streambuffer.hpp"
class SourceDB {
private:
@@ -143,10 +144,17 @@ public:
// Rebuild the lua environment: clear it out, then reinstall all the
// functions that should be there. See above for more information.
// If an error exists in any of the source files, or when loading any
// of the closures, the error is (currently) printed. We'll come up
// with better error handling later.
// of the closures, the error can optionally be printed or ignored.
//
void rebuild();
void rebuild(bool print_errors);
// Difference transmission.
//
// Note: The patch routine applies the differences to the source
// database, and if there are any changes, it does a source rebuild.
//
void diff(const SourceDB &auth, StreamBuffer *sb);
void patch(StreamBuffer *sb);
// run_unittests
//
@@ -154,6 +162,14 @@ public:
// are any errors, exits the program.
//
void run_unittests();
// Get/Set code (for unit testing).
//
// These functions are direct getters/setters for values in the source
// database. They are intended only for unit testing.
//
void set(const std::string &fn, const std::string &code, int sequence);
std::string get(const std::string &fn);
};
// The Lua 'makeclass' operator.

20
luprex/core/cpp/util.cpp
View File

@@ -105,11 +105,25 @@ IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2) {
return result;
}
HashValue hash_string(const std::string &s) {
uint64_t hash1 = 0;
uint64_t hash2 = 0;
SpookyHash::Hash128(s.c_str(), s.size(), &hash1, &hash2);
return util::HashValue(hash1, hash2);
}
HashValue hash_id_vector(const IdVector &idv) {
uint64_t hash1 = 0;
uint64_t hash2 = 0;
SpookyHash::Hash128(&idv[0], idv.size() * sizeof(int64_t), &hash1, &hash2);
return std::make_pair(hash1, hash2);
return util::HashValue(hash1, hash2);
}
std::string hash_to_hex(const HashValue &hv) {
std::ostringstream oss;
oss << std::hex << std::setw(16) << std::setfill('0') << hv.first;
oss << std::hex << std::setw(16) << std::setfill('0') << hv.second;
return oss.str();
}
StringVec split(const std::string &s, char sep) {
@@ -318,6 +332,10 @@ LuaDefine(unittests_util, "c") {
LuaAssert(L, xyza != xyzc);
LuaAssert(L, xyza.debug_string() == "(3,4,5)");
// Test hash_to_string
LuaAssertStrEq(L, util::hash_to_hex(util::HashValue(0x1234,0x789a)),
"0000000000001234000000000000789a");
return 0;
}

6
luprex/core/cpp/util.hpp
View File

@@ -39,9 +39,15 @@ std::string id_vector_debug_string(const IdVector &idv);
// Unions and sorts two ID vectors.
IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2);
// Get a 64-bit hashvalue for a string.
HashValue hash_string(const std::string &str);
// Get a 64-bit hashvalue for an ID vector.
HashValue hash_id_vector(const IdVector &idv);
// Convert a hash to a hexadecimal string.
std::string hash_to_hex(const HashValue &hash);
// Split a string into multiple strings
StringVec split(const std::string &s, char sep);

2
luprex/core/cpp/world-core.cpp
View File

@@ -63,7 +63,7 @@ World::World(util::WorldType wt) {
// Initialize the SourceDB. At this stage, the sourcedb is
// empty, so it's just populating the lua builtins.
source_db_.init(state());
source_db_.rebuild();
source_db_.rebuild(true);
LS.result();
assert (stack_is_clear());

10
luprex/core/cpp/world-diffxmit.cpp
View File

@@ -262,11 +262,20 @@ void World::diff_tanclass(int64_t actor_id, World *master, StreamBuffer *xsb) {
assert(tsb.at_eof());
}
void World::patch_source(StreamBuffer *sb) {
source_db_.patch(sb);
}
void World::diff_source(World *master, StreamBuffer *sb) {
source_db_.diff(master->source_db_, sb);
}
void World::patch_everything(StreamBuffer *sb) {
patch_actor(sb);
patch_visible(sb);
patch_luatabs(sb);
patch_tanclass(sb);
patch_source(sb);
}
void World::diff_everything(int64_t actor_id, World *master, StreamBuffer *sb) {
@@ -275,4 +284,5 @@ void World::diff_everything(int64_t actor_id, World *master, StreamBuffer *sb) {
diff_visible(visible, master, sb);
diff_luatabs(actor_id, master, sb);
diff_tanclass(actor_id, master, sb);
diff_source(master, sb);
}

5
luprex/core/cpp/world.hpp
View File

@@ -174,7 +174,7 @@ public:
// Update the source database from disk.
//
void update_source() { source_db_.update(); source_db_.rebuild(); }
void update_source() { source_db_.update(); source_db_.rebuild(true); }
// Run all unit tests.
//
@@ -308,6 +308,9 @@ public:
void patch_tanclass(StreamBuffer *sb);
void diff_tanclass(int64_t actor_id, World *master, StreamBuffer *sb);
void patch_source(StreamBuffer *sb);
void diff_source(World *master, StreamBuffer *sb);
// This is the main entry point for difference transmission.
//
void patch_everything(StreamBuffer *sb);