talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

difference transmission of source databases done

Browse Source
This commit is contained in:
Josh Yelon
2021-09-23 12:40:21 -04:00
parent 6fbe700107
commit 6490db43fc
8 changed files with 320 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
luprex/core/cpp/source.cpp
View File

@@ -13,6 +13,7 @@
#include "traceback.hpp"
#include "table.hpp"
#include "source.hpp"
#include "luasnap.hpp"
// Read control.lst
//
@@ -40,27 +41,16 @@ LuaDefine(source_makeclass, "f") {
return LS.result();
}
LuaDefine(source_isclass, "f") {
LuaDefine(source_classname, "f") {
LuaArg table;
LuaVar classname, globtab, classtab;
LuaRet result;
LuaStack LS(L, table, classname, globtab, classtab, result);
if (!LS.istable(table)) {
LS.set(result, false);
return LS.result();
LuaStack LS(L, table, result);
std::string rstr = LS.classname(table);
if (rstr == "") {
LS.set(result, LuaNil);
} else {
LS.set(result, rstr);
}
LS.rawget(classname, table, "__class");
if (!LS.isstring(classname)) {
LS.set(result, false);
return LS.result();
}
LS.getglobaltable(globtab);
LS.rawget(classtab, globtab, classname);
if (!LS.rawequal(classtab, table)) {
LS.set(result, false);
return LS.result();
}
LS.set(result, true);
return LS.result();
}
@@ -91,6 +81,7 @@ static void load_builtin_class(lua_State *L, const char *name, lua_CFunction fun
while (LS.next(sourcetab, key, value) != 0) {
LS.rawset(classtab, key, value);
}
LS.result();
}
static void source_install_builtins(lua_State *L) {
@@ -113,12 +104,36 @@ static void source_install_builtins(lua_State *L) {
LS.makeclass(stringclass, "string");
LS.set(nullstring, "");
LS.setmetatable(nullstring, stringclass);
LS.result();
}
static void get_info_table(LuaStack &LS, LuaSlot db, LuaSlot info, const std::string &fn) {
LS.rawget(info, db, fn);
if (!LS.istable(info)) {
LS.set(info, LuaNewTable);
LS.rawset(db, fn, info);
}
LS.rawset(info, "name", fn);
}
static void calculate_loadresult(LuaStack &LS0, LuaSlot info, const std::string &fn, const std::string &code) {
LuaVar loadresult;
LuaStack LS(LS0.state(), loadresult);
if (code == "") {
LS.rawset(info, "loadresult", "missing or empty source file");
} else {
std::string chunk = "=" + fn;
luaL_loadbuffer(LS.state(), code.c_str(), code.size(), chunk.c_str());
lua_replace(LS.state(), loadresult.index());
LS.rawset(info, "loadresult", loadresult);
}
}
static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot info) {
lua_State *L = LS0.state();
LuaVar fingerprint, null, loadresult;
LuaStack LS(L, fingerprint, null, loadresult);
LuaVar fingerprint;
LuaStack LS(L, fingerprint);
// Get the existing info table from the source DB.
if (LS.istable(source)) {
@@ -131,7 +146,7 @@ static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot
}
// If the file modification is wrong, update
// these fields: code, fingerprint, closure, error
// these fields: code, hash, fingerprint, closure, error
// Otherwise, update nothing.
std::string cfn = LS.ckstring(fn);
LS.rawget(fingerprint, info, "fingerprint");
@@ -141,23 +156,143 @@ static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot
}
// std::cerr << "Probing " << cfn << std::endl;
std::string new_fingerprint = util::get_file_fingerprint("lua/" + cfn);
LS.set(null, LuaNil);
if ((old_fingerprint == "") || (old_fingerprint != new_fingerprint)) {
std::cerr << "Rereading " << cfn << std::endl;
std::string ccode = util::get_file_contents("lua/" + cfn);
std::string code;
if (new_fingerprint != "") code = util::get_file_contents("lua/" + cfn);
LS.rawset(info, "name", fn);
LS.rawset(info, "fingerprint", new_fingerprint);
LS.rawset(info, "code", ccode);
if ((new_fingerprint == "")||(ccode == "")) {
LS.rawset(info, "loadresult", "cannot read source file");
LS.rawset(info, "code", code);
LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
calculate_loadresult(LS, info, cfn, code);
}
LS.result();
}
void SourceDB::diff(const SourceDB &auth, StreamBuffer *sb) {
LuaVar sdb, sfn, sinfo, shash, sseq;
LuaVar mdb, mfn, minfo, mhash, mseq, mcode;
LuaStack SLS(lua_state_, sdb, sfn, sinfo, shash, sseq);
LuaStack MLS(auth.lua_state_, mdb, mfn, minfo, mhash, mseq, mcode);
sb->write_int32(0);
int wc_after = sb->total_writes();
int nupdates = 0;
// Fetch the two source databases.
SLS.rawget(sdb, LuaRegistry, "sourcedb");
MLS.rawget(mdb, LuaRegistry, "sourcedb");
// Loop over the master database.
MLS.set(mfn, LuaNil);
while (MLS.next(mdb, mfn, minfo) != 0) {
std::string fn = MLS.ckstring(mfn);
assert(MLS.istable(minfo));
MLS.rawget(mseq, minfo, "sequence");
MLS.rawget(mhash, minfo, "hash");
bool diffhash = true;
bool diffseq = true;
SLS.set(sfn, fn);
SLS.rawget(sinfo, sdb, sfn);
if (SLS.istable(sinfo)) {
SLS.rawget(shash, sinfo, "hash");
SLS.rawget(sseq, sinfo, "sequence");
diffhash = MLS.ckstring(mhash) != SLS.ckstring(shash);
diffseq = MLS.ckinteger(mseq) != SLS.ckinteger(sseq);
}
if (diffhash || diffseq) {
sb->write_string(MLS.ckstring(mfn));
sb->write_int32(MLS.ckinteger(mseq));
if (diffhash) {
MLS.rawget(mcode, minfo, "code");
sb->write_string(MLS.ckstring(mcode));
} else {
sb->write_string("\001");
}
nupdates += 1;
}
}
// Loop over synch database.
SLS.set(sfn, LuaNil);
while (SLS.next(sdb, sfn, sinfo) != 0) {
std::string fn = SLS.ckstring(sfn);
assert(SLS.istable(sinfo));
MLS.set(mfn, fn);
MLS.rawget(minfo, mdb, mfn);
if (!MLS.istable(minfo)) {
sb->write_string(SLS.ckstring(sfn));
sb->write_int32(-1);
sb->write_string("");
nupdates += 1;
}
}
sb->overwrite_int32(wc_after, nupdates);
MLS.result();
SLS.result();
}
void SourceDB::patch(StreamBuffer *sb) {
lua_State *L = lua_state_;
LuaVar db, info;
LuaStack LS(L, db, info);
LS.rawget(db, LuaRegistry, "sourcedb");
int nupdates = sb->read_int32();
for (int i = 0; i < nupdates; i++) {
std::string fn = sb->read_string();
int sequence = sb->read_int32();
std::string code = sb->read_string();
if (sequence < 0) {
LS.rawset(db, fn, LuaNil);
} else {
std::string chunk = "=" + cfn;
luaL_loadbuffer(L, ccode.c_str(), ccode.size(), chunk.c_str());
lua_replace(L, loadresult.index());
LS.rawset(info, "loadresult", loadresult);
get_info_table(LS, db, info, fn);
LS.rawset(info, "sequence", sequence);
if (code != "\001") {
LS.rawset(info, "code", code);
LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
calculate_loadresult(LS, info, fn, code);
}
}
}
LS.result();
if (nupdates > 0) rebuild(false);
}
void SourceDB::set(const std::string &fn, const std::string &code, int sequence) {
lua_State *L = lua_state_;
LuaVar db, info;
LuaStack LS(L, db, info);
LS.rawget(db, LuaRegistry, "sourcedb");
get_info_table(LS, db, info, fn);
LS.rawset(info, "sequence", sequence);
LS.rawset(info, "code", code);
LS.rawset(info, "fingerprint", "");
LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code)));
calculate_loadresult(LS, info, fn, code);
LS.result();
}
std::string SourceDB::get(const std::string &fn) {
lua_State *L = lua_state_;
LuaVar db, info, code, sequence, loadresult;
LuaStack LS(L, db, info, code, sequence, loadresult);
LS.rawget(db, LuaRegistry, "sourcedb");
LS.rawget(info, db, fn);
if (!LS.istable(info)) {
LS.result();
return "<nonexistent>";
}
LS.rawget(code, info, "code");
LS.rawget(sequence, info, "sequence");
LS.rawget(loadresult, info, "loadresult");
std::string ccode = LS.ckstring(code);
int seqno = LS.ckint(sequence);
std::string cloadresult;
if (LS.isfunction(loadresult)) {
cloadresult = "<function>";
} else {
cloadresult = LS.ckstring(loadresult);
}
std::ostringstream oss;
oss << seqno << ":" << ccode << ":" << cloadresult;
LS.result();
return oss.str();
}
void SourceDB::update() {
@@ -250,7 +385,7 @@ static void source_load_cfunctions(lua_State *L) {
// Run all the closures from the source database.
//
static void source_load_lfunctions(lua_State *L) {
static void source_load_lfunctions(lua_State *L, bool print_errors) {
LuaRet errors;
LuaVar sourcedb, key, info, seq, closure, err;
LuaStack LS(L, sourcedb, errors, key, info, seq, closure, err);
@@ -278,7 +413,9 @@ static void source_load_lfunctions(lua_State *L) {
// If there's already an error in the sourcedb, collect it.
if (!LS.isfunction(closure)) {
errss << LS.ckstring(closure) << "\n";
if (print_errors) {
errss << LS.ckstring(closure) << "\n";
}
continue;
}
@@ -293,14 +430,14 @@ static void source_load_lfunctions(lua_State *L) {
LS.result();
}
void SourceDB::rebuild() {
void SourceDB::rebuild(bool print_errors) {
lua_State *L = lua_state_;
LuaVar errs;
LuaStack LS(L, errs);
source_clear_globals(L);
source_install_builtins(L);
source_load_cfunctions(L);
source_load_lfunctions(L);
source_load_lfunctions(L, print_errors);
lua_replace(L, errs.index());
std::string errstr = LS.ckstring(errs);
std::cerr << errstr;
@@ -356,6 +493,7 @@ void SourceDB::init(lua_State *L) {
LS.getglobaltable(globtab);
LS.rawget(persist, LuaRegistry, "persist");
LS.rawget(unpersist, LuaRegistry, "unpersist");
LS.rawset(LuaRegistry, "sourcedb", LuaNewTable);
LS.set(classname, LuaNil);
while (LS.next(globtab, classname, classtab) != 0) {
if (LS.isstring(classname) && LS.istable(classtab)) {
@@ -394,3 +532,81 @@ LuaDefine(coroutine_getnextid, "c") {
LS.set(lid, lua_getnextid(CO));
return LS.result();
}
LuaDefine(unittests_sourcedb, "c") {
LuaSnap msnap;
LuaSnap ssnap;
SourceDB mdb;
SourceDB sdb;
mdb.init(msnap.state());
sdb.init(ssnap.state());
StreamBuffer sb;
// Create a code database using 'set'.
mdb.set("foo", "function foo() print('foo') end", 1);
mdb.set("bar", "function bar() print('bar') end", 2);
mdb.set("zoo", "funcjdshja mxooso yowza!", 3);
// Verify that get works.
LuaAssertStrEq(L, mdb.get("foo"), "1:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, mdb.get("bar"), "2:function bar() print('bar') end:<function>");
LuaAssertStrEq(L, mdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, mdb.get("baz"), "<nonexistent>");
// Difference transmit.
sdb.diff(mdb, &sb);
// There should still be nothing in the sdb.
LuaAssertStrEq(L, sdb.get("foo"), "<nonexistent>");
LuaAssertStrEq(L, sdb.get("bar"), "<nonexistent>");
LuaAssertStrEq(L, sdb.get("zoo"), "<nonexistent>");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
// Apply the diffs.
sdb.patch(&sb);
// Everything should now be copied to sdb.
LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, sdb.get("bar"), "2:function bar() print('bar') end:<function>");
LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
// Make a single change to the code.
mdb.set("bar", "function bar1() print('bar1') end", 2);
// Diff and patch
sdb.diff(mdb, &sb);
sdb.patch(&sb);
// Verify that it's been updated.
LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:<function>");
LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
// Make a single change to just a sequence number.
mdb.set("foo", "function foo() print('foo') end", 6);
// Diff
sdb.diff(mdb, &sb);
// The only thing we've updated is a single sequence number. Make
// sure the number of bytes transmitted is reasonable.
// 4 bytes for the count of changes
// 4 bytes for "foo"
// 4 bytes for the sequence number
// 2 bytes for unmodified code
// This verifies that the hash comparisons are working.
LuaAssert(L, sb.fill() == 14);
// Patch.
sdb.patch(&sb);
// Verify that it's been updated.
LuaAssertStrEq(L, sdb.get("foo"), "6:function foo() print('foo') end:<function>");
LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:<function>");
LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'");
LuaAssertStrEq(L, sdb.get("baz"), "<nonexistent>");
return 0;
}