diff --git a/luprex/core/cpp/luastack.cpp b/luprex/core/cpp/luastack.cpp index 1ab1df49..b7245ac6 100644 --- a/luprex/core/cpp/luastack.cpp +++ b/luprex/core/cpp/luastack.cpp @@ -263,11 +263,18 @@ std::string LuaStack::classname(LuaSlot tab) { lua_pushstring(L_, "__class"); lua_rawget(L_, tab); if (lua_type(L_, -1) == LUA_TSTRING) { - size_t len; - const char *s = lua_tolstring(L_, -1, &len); - result = std::string(s, len); + lua_pushglobaltable(L_); // cname table + lua_pushvalue(L_, -2); // cname table cname + lua_rawget(L_, -2); // cname table ctab + if (lua_rawequal(L_, -1, tab)) { + size_t len; + const char *s = lua_tolstring(L_, -3, &len); + result = std::string(s, len); + } + lua_pop(L_, 3); + } else { + lua_pop(L_, 1); } - lua_pop(L_, 1); } return result; } diff --git a/luprex/core/cpp/source.cpp b/luprex/core/cpp/source.cpp index a4545a32..3aae50a8 100644 --- a/luprex/core/cpp/source.cpp +++ b/luprex/core/cpp/source.cpp @@ -13,6 +13,7 @@ #include "traceback.hpp" #include "table.hpp" #include "source.hpp" +#include "luasnap.hpp" // Read control.lst // @@ -40,27 +41,16 @@ LuaDefine(source_makeclass, "f") { return LS.result(); } -LuaDefine(source_isclass, "f") { +LuaDefine(source_classname, "f") { LuaArg table; - LuaVar classname, globtab, classtab; LuaRet result; - LuaStack LS(L, table, classname, globtab, classtab, result); - if (!LS.istable(table)) { - LS.set(result, false); - return LS.result(); + LuaStack LS(L, table, result); + std::string rstr = LS.classname(table); + if (rstr == "") { + LS.set(result, LuaNil); + } else { + LS.set(result, rstr); } - LS.rawget(classname, table, "__class"); - if (!LS.isstring(classname)) { - LS.set(result, false); - return LS.result(); - } - LS.getglobaltable(globtab); - LS.rawget(classtab, globtab, classname); - if (!LS.rawequal(classtab, table)) { - LS.set(result, false); - return LS.result(); - } - LS.set(result, true); return LS.result(); } @@ -91,6 +81,7 @@ static void load_builtin_class(lua_State *L, const char *name, lua_CFunction fun while (LS.next(sourcetab, key, value) != 0) { LS.rawset(classtab, key, value); } + LS.result(); } static void source_install_builtins(lua_State *L) { @@ -113,12 +104,36 @@ static void source_install_builtins(lua_State *L) { LS.makeclass(stringclass, "string"); LS.set(nullstring, ""); LS.setmetatable(nullstring, stringclass); + + LS.result(); +} + +static void get_info_table(LuaStack &LS, LuaSlot db, LuaSlot info, const std::string &fn) { + LS.rawget(info, db, fn); + if (!LS.istable(info)) { + LS.set(info, LuaNewTable); + LS.rawset(db, fn, info); + } + LS.rawset(info, "name", fn); +} + +static void calculate_loadresult(LuaStack &LS0, LuaSlot info, const std::string &fn, const std::string &code) { + LuaVar loadresult; + LuaStack LS(LS0.state(), loadresult); + if (code == "") { + LS.rawset(info, "loadresult", "missing or empty source file"); + } else { + std::string chunk = "=" + fn; + luaL_loadbuffer(LS.state(), code.c_str(), code.size(), chunk.c_str()); + lua_replace(LS.state(), loadresult.index()); + LS.rawset(info, "loadresult", loadresult); + } } static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot info) { lua_State *L = LS0.state(); - LuaVar fingerprint, null, loadresult; - LuaStack LS(L, fingerprint, null, loadresult); + LuaVar fingerprint; + LuaStack LS(L, fingerprint); // Get the existing info table from the source DB. if (LS.istable(source)) { @@ -131,7 +146,7 @@ static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot } // If the file modification is wrong, update - // these fields: code, fingerprint, closure, error + // these fields: code, hash, fingerprint, closure, error // Otherwise, update nothing. std::string cfn = LS.ckstring(fn); LS.rawget(fingerprint, info, "fingerprint"); @@ -141,23 +156,143 @@ static void source_updatefile(LuaStack &LS0, LuaSlot source, LuaSlot fn, LuaSlot } // std::cerr << "Probing " << cfn << std::endl; std::string new_fingerprint = util::get_file_fingerprint("lua/" + cfn); - LS.set(null, LuaNil); if ((old_fingerprint == "") || (old_fingerprint != new_fingerprint)) { std::cerr << "Rereading " << cfn << std::endl; - std::string ccode = util::get_file_contents("lua/" + cfn); + std::string code; + if (new_fingerprint != "") code = util::get_file_contents("lua/" + cfn); LS.rawset(info, "name", fn); LS.rawset(info, "fingerprint", new_fingerprint); - LS.rawset(info, "code", ccode); - if ((new_fingerprint == "")||(ccode == "")) { - LS.rawset(info, "loadresult", "cannot read source file"); + LS.rawset(info, "code", code); + LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code))); + calculate_loadresult(LS, info, cfn, code); + } + LS.result(); +} + +void SourceDB::diff(const SourceDB &auth, StreamBuffer *sb) { + LuaVar sdb, sfn, sinfo, shash, sseq; + LuaVar mdb, mfn, minfo, mhash, mseq, mcode; + LuaStack SLS(lua_state_, sdb, sfn, sinfo, shash, sseq); + LuaStack MLS(auth.lua_state_, mdb, mfn, minfo, mhash, mseq, mcode); + sb->write_int32(0); + int wc_after = sb->total_writes(); + int nupdates = 0; + // Fetch the two source databases. + SLS.rawget(sdb, LuaRegistry, "sourcedb"); + MLS.rawget(mdb, LuaRegistry, "sourcedb"); + // Loop over the master database. + MLS.set(mfn, LuaNil); + while (MLS.next(mdb, mfn, minfo) != 0) { + std::string fn = MLS.ckstring(mfn); + assert(MLS.istable(minfo)); + MLS.rawget(mseq, minfo, "sequence"); + MLS.rawget(mhash, minfo, "hash"); + bool diffhash = true; + bool diffseq = true; + SLS.set(sfn, fn); + SLS.rawget(sinfo, sdb, sfn); + if (SLS.istable(sinfo)) { + SLS.rawget(shash, sinfo, "hash"); + SLS.rawget(sseq, sinfo, "sequence"); + diffhash = MLS.ckstring(mhash) != SLS.ckstring(shash); + diffseq = MLS.ckinteger(mseq) != SLS.ckinteger(sseq); + } + if (diffhash || diffseq) { + sb->write_string(MLS.ckstring(mfn)); + sb->write_int32(MLS.ckinteger(mseq)); + if (diffhash) { + MLS.rawget(mcode, minfo, "code"); + sb->write_string(MLS.ckstring(mcode)); + } else { + sb->write_string("\001"); + } + nupdates += 1; + } + } + // Loop over synch database. + SLS.set(sfn, LuaNil); + while (SLS.next(sdb, sfn, sinfo) != 0) { + std::string fn = SLS.ckstring(sfn); + assert(SLS.istable(sinfo)); + MLS.set(mfn, fn); + MLS.rawget(minfo, mdb, mfn); + if (!MLS.istable(minfo)) { + sb->write_string(SLS.ckstring(sfn)); + sb->write_int32(-1); + sb->write_string(""); + nupdates += 1; + } + } + sb->overwrite_int32(wc_after, nupdates); + MLS.result(); + SLS.result(); +} + +void SourceDB::patch(StreamBuffer *sb) { + lua_State *L = lua_state_; + LuaVar db, info; + LuaStack LS(L, db, info); + LS.rawget(db, LuaRegistry, "sourcedb"); + int nupdates = sb->read_int32(); + for (int i = 0; i < nupdates; i++) { + std::string fn = sb->read_string(); + int sequence = sb->read_int32(); + std::string code = sb->read_string(); + if (sequence < 0) { + LS.rawset(db, fn, LuaNil); } else { - std::string chunk = "=" + cfn; - luaL_loadbuffer(L, ccode.c_str(), ccode.size(), chunk.c_str()); - lua_replace(L, loadresult.index()); - LS.rawset(info, "loadresult", loadresult); + get_info_table(LS, db, info, fn); + LS.rawset(info, "sequence", sequence); + if (code != "\001") { + LS.rawset(info, "code", code); + LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code))); + calculate_loadresult(LS, info, fn, code); + } } } LS.result(); + if (nupdates > 0) rebuild(false); +} + +void SourceDB::set(const std::string &fn, const std::string &code, int sequence) { + lua_State *L = lua_state_; + LuaVar db, info; + LuaStack LS(L, db, info); + LS.rawget(db, LuaRegistry, "sourcedb"); + get_info_table(LS, db, info, fn); + LS.rawset(info, "sequence", sequence); + LS.rawset(info, "code", code); + LS.rawset(info, "fingerprint", ""); + LS.rawset(info, "hash", util::hash_to_hex(util::hash_string(code))); + calculate_loadresult(LS, info, fn, code); + LS.result(); +} + +std::string SourceDB::get(const std::string &fn) { + lua_State *L = lua_state_; + LuaVar db, info, code, sequence, loadresult; + LuaStack LS(L, db, info, code, sequence, loadresult); + LS.rawget(db, LuaRegistry, "sourcedb"); + LS.rawget(info, db, fn); + if (!LS.istable(info)) { + LS.result(); + return ""; + } + LS.rawget(code, info, "code"); + LS.rawget(sequence, info, "sequence"); + LS.rawget(loadresult, info, "loadresult"); + std::string ccode = LS.ckstring(code); + int seqno = LS.ckint(sequence); + std::string cloadresult; + if (LS.isfunction(loadresult)) { + cloadresult = ""; + } else { + cloadresult = LS.ckstring(loadresult); + } + std::ostringstream oss; + oss << seqno << ":" << ccode << ":" << cloadresult; + LS.result(); + return oss.str(); } void SourceDB::update() { @@ -250,7 +385,7 @@ static void source_load_cfunctions(lua_State *L) { // Run all the closures from the source database. // -static void source_load_lfunctions(lua_State *L) { +static void source_load_lfunctions(lua_State *L, bool print_errors) { LuaRet errors; LuaVar sourcedb, key, info, seq, closure, err; LuaStack LS(L, sourcedb, errors, key, info, seq, closure, err); @@ -278,7 +413,9 @@ static void source_load_lfunctions(lua_State *L) { // If there's already an error in the sourcedb, collect it. if (!LS.isfunction(closure)) { - errss << LS.ckstring(closure) << "\n"; + if (print_errors) { + errss << LS.ckstring(closure) << "\n"; + } continue; } @@ -293,14 +430,14 @@ static void source_load_lfunctions(lua_State *L) { LS.result(); } -void SourceDB::rebuild() { +void SourceDB::rebuild(bool print_errors) { lua_State *L = lua_state_; LuaVar errs; LuaStack LS(L, errs); source_clear_globals(L); source_install_builtins(L); source_load_cfunctions(L); - source_load_lfunctions(L); + source_load_lfunctions(L, print_errors); lua_replace(L, errs.index()); std::string errstr = LS.ckstring(errs); std::cerr << errstr; @@ -356,6 +493,7 @@ void SourceDB::init(lua_State *L) { LS.getglobaltable(globtab); LS.rawget(persist, LuaRegistry, "persist"); LS.rawget(unpersist, LuaRegistry, "unpersist"); + LS.rawset(LuaRegistry, "sourcedb", LuaNewTable); LS.set(classname, LuaNil); while (LS.next(globtab, classname, classtab) != 0) { if (LS.isstring(classname) && LS.istable(classtab)) { @@ -394,3 +532,81 @@ LuaDefine(coroutine_getnextid, "c") { LS.set(lid, lua_getnextid(CO)); return LS.result(); } + +LuaDefine(unittests_sourcedb, "c") { + LuaSnap msnap; + LuaSnap ssnap; + SourceDB mdb; + SourceDB sdb; + mdb.init(msnap.state()); + sdb.init(ssnap.state()); + StreamBuffer sb; + + // Create a code database using 'set'. + mdb.set("foo", "function foo() print('foo') end", 1); + mdb.set("bar", "function bar() print('bar') end", 2); + mdb.set("zoo", "funcjdshja mxooso yowza!", 3); + + // Verify that get works. + LuaAssertStrEq(L, mdb.get("foo"), "1:function foo() print('foo') end:"); + LuaAssertStrEq(L, mdb.get("bar"), "2:function bar() print('bar') end:"); + LuaAssertStrEq(L, mdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'"); + LuaAssertStrEq(L, mdb.get("baz"), ""); + + // Difference transmit. + sdb.diff(mdb, &sb); + + // There should still be nothing in the sdb. + LuaAssertStrEq(L, sdb.get("foo"), ""); + LuaAssertStrEq(L, sdb.get("bar"), ""); + LuaAssertStrEq(L, sdb.get("zoo"), ""); + LuaAssertStrEq(L, sdb.get("baz"), ""); + + // Apply the diffs. + sdb.patch(&sb); + + // Everything should now be copied to sdb. + LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:"); + LuaAssertStrEq(L, sdb.get("bar"), "2:function bar() print('bar') end:"); + LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'"); + LuaAssertStrEq(L, sdb.get("baz"), ""); + + // Make a single change to the code. + mdb.set("bar", "function bar1() print('bar1') end", 2); + + // Diff and patch + sdb.diff(mdb, &sb); + sdb.patch(&sb); + + // Verify that it's been updated. + LuaAssertStrEq(L, sdb.get("foo"), "1:function foo() print('foo') end:"); + LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:"); + LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'"); + LuaAssertStrEq(L, sdb.get("baz"), ""); + + // Make a single change to just a sequence number. + mdb.set("foo", "function foo() print('foo') end", 6); + + // Diff + sdb.diff(mdb, &sb); + + // The only thing we've updated is a single sequence number. Make + // sure the number of bytes transmitted is reasonable. + // 4 bytes for the count of changes + // 4 bytes for "foo" + // 4 bytes for the sequence number + // 2 bytes for unmodified code + // This verifies that the hash comparisons are working. + LuaAssert(L, sb.fill() == 14); + + // Patch. + sdb.patch(&sb); + + // Verify that it's been updated. + LuaAssertStrEq(L, sdb.get("foo"), "6:function foo() print('foo') end:"); + LuaAssertStrEq(L, sdb.get("bar"), "2:function bar1() print('bar1') end:"); + LuaAssertStrEq(L, sdb.get("zoo"), "3:funcjdshja mxooso yowza!:zoo:1: syntax error near 'mxooso'"); + LuaAssertStrEq(L, sdb.get("baz"), ""); + + return 0; +} diff --git a/luprex/core/cpp/source.hpp b/luprex/core/cpp/source.hpp index 23ba12e6..35558546 100644 --- a/luprex/core/cpp/source.hpp +++ b/luprex/core/cpp/source.hpp @@ -121,6 +121,7 @@ #define SOURCE_HPP #include "luastack.hpp" +#include "streambuffer.hpp" class SourceDB { private: @@ -143,10 +144,17 @@ public: // Rebuild the lua environment: clear it out, then reinstall all the // functions that should be there. See above for more information. // If an error exists in any of the source files, or when loading any - // of the closures, the error is (currently) printed. We'll come up - // with better error handling later. + // of the closures, the error can optionally be printed or ignored. // - void rebuild(); + void rebuild(bool print_errors); + + // Difference transmission. + // + // Note: The patch routine applies the differences to the source + // database, and if there are any changes, it does a source rebuild. + // + void diff(const SourceDB &auth, StreamBuffer *sb); + void patch(StreamBuffer *sb); // run_unittests // @@ -154,6 +162,14 @@ public: // are any errors, exits the program. // void run_unittests(); + + // Get/Set code (for unit testing). + // + // These functions are direct getters/setters for values in the source + // database. They are intended only for unit testing. + // + void set(const std::string &fn, const std::string &code, int sequence); + std::string get(const std::string &fn); }; // The Lua 'makeclass' operator. diff --git a/luprex/core/cpp/util.cpp b/luprex/core/cpp/util.cpp index d109a5c6..6ea043cc 100644 --- a/luprex/core/cpp/util.cpp +++ b/luprex/core/cpp/util.cpp @@ -105,11 +105,25 @@ IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2) { return result; } +HashValue hash_string(const std::string &s) { + uint64_t hash1 = 0; + uint64_t hash2 = 0; + SpookyHash::Hash128(s.c_str(), s.size(), &hash1, &hash2); + return util::HashValue(hash1, hash2); +} + HashValue hash_id_vector(const IdVector &idv) { uint64_t hash1 = 0; uint64_t hash2 = 0; SpookyHash::Hash128(&idv[0], idv.size() * sizeof(int64_t), &hash1, &hash2); - return std::make_pair(hash1, hash2); + return util::HashValue(hash1, hash2); +} + +std::string hash_to_hex(const HashValue &hv) { + std::ostringstream oss; + oss << std::hex << std::setw(16) << std::setfill('0') << hv.first; + oss << std::hex << std::setw(16) << std::setfill('0') << hv.second; + return oss.str(); } StringVec split(const std::string &s, char sep) { @@ -318,6 +332,10 @@ LuaDefine(unittests_util, "c") { LuaAssert(L, xyza != xyzc); LuaAssert(L, xyza.debug_string() == "(3,4,5)"); + // Test hash_to_string + LuaAssertStrEq(L, util::hash_to_hex(util::HashValue(0x1234,0x789a)), + "0000000000001234000000000000789a"); + return 0; } diff --git a/luprex/core/cpp/util.hpp b/luprex/core/cpp/util.hpp index 7864bee3..17f2d687 100644 --- a/luprex/core/cpp/util.hpp +++ b/luprex/core/cpp/util.hpp @@ -39,9 +39,15 @@ std::string id_vector_debug_string(const IdVector &idv); // Unions and sorts two ID vectors. IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2); +// Get a 64-bit hashvalue for a string. +HashValue hash_string(const std::string &str); + // Get a 64-bit hashvalue for an ID vector. HashValue hash_id_vector(const IdVector &idv); +// Convert a hash to a hexadecimal string. +std::string hash_to_hex(const HashValue &hash); + // Split a string into multiple strings StringVec split(const std::string &s, char sep); diff --git a/luprex/core/cpp/world-core.cpp b/luprex/core/cpp/world-core.cpp index d28484b7..c6bb1835 100644 --- a/luprex/core/cpp/world-core.cpp +++ b/luprex/core/cpp/world-core.cpp @@ -63,7 +63,7 @@ World::World(util::WorldType wt) { // Initialize the SourceDB. At this stage, the sourcedb is // empty, so it's just populating the lua builtins. source_db_.init(state()); - source_db_.rebuild(); + source_db_.rebuild(true); LS.result(); assert (stack_is_clear()); diff --git a/luprex/core/cpp/world-diffxmit.cpp b/luprex/core/cpp/world-diffxmit.cpp index fbab6a29..958e67b3 100644 --- a/luprex/core/cpp/world-diffxmit.cpp +++ b/luprex/core/cpp/world-diffxmit.cpp @@ -262,11 +262,20 @@ void World::diff_tanclass(int64_t actor_id, World *master, StreamBuffer *xsb) { assert(tsb.at_eof()); } +void World::patch_source(StreamBuffer *sb) { + source_db_.patch(sb); +} + +void World::diff_source(World *master, StreamBuffer *sb) { + source_db_.diff(master->source_db_, sb); +} + void World::patch_everything(StreamBuffer *sb) { patch_actor(sb); patch_visible(sb); patch_luatabs(sb); patch_tanclass(sb); + patch_source(sb); } void World::diff_everything(int64_t actor_id, World *master, StreamBuffer *sb) { @@ -275,4 +284,5 @@ void World::diff_everything(int64_t actor_id, World *master, StreamBuffer *sb) { diff_visible(visible, master, sb); diff_luatabs(actor_id, master, sb); diff_tanclass(actor_id, master, sb); + diff_source(master, sb); } diff --git a/luprex/core/cpp/world.hpp b/luprex/core/cpp/world.hpp index bf0d2c1e..ffd8753f 100644 --- a/luprex/core/cpp/world.hpp +++ b/luprex/core/cpp/world.hpp @@ -174,7 +174,7 @@ public: // Update the source database from disk. // - void update_source() { source_db_.update(); source_db_.rebuild(); } + void update_source() { source_db_.update(); source_db_.rebuild(true); } // Run all unit tests. // @@ -308,6 +308,9 @@ public: void patch_tanclass(StreamBuffer *sb); void diff_tanclass(int64_t actor_id, World *master, StreamBuffer *sb); + void patch_source(StreamBuffer *sb); + void diff_source(World *master, StreamBuffer *sb); + // This is the main entry point for difference transmission. // void patch_everything(StreamBuffer *sb);