//////////////////////////////////////////////////////////////////// // // This file contains the code to compare the contents of tables. // The top level functions in this file are: // // World::diff_numbered_tables // World::diff_tangible_databases // World::patch_numbered_tables // World::patch_tangible_databases // // This file also contains all the support code needed to implement // this stuff, plus the unit tests (unittests_difftab). // //////////////////////////////////////////////////////////////////// #include "luastack.hpp" #include "streambuffer.hpp" #include "table.hpp" #include "world.hpp" // Given a table and an tnmap, return the table number of the table. // Returns zero if the table doesn't have a table number. // static int get_table_number(LuaCoreStack &MLS, LuaSlot mval, LuaSlot mtnmap) { lua_State *L = MLS.state(); lua_pushvalue(L, mval.index()); lua_rawget(L, mtnmap.index()); int result = 0; if (lua_type(L, -1) == LUA_TNUMBER) { result = lua_tointeger(L, -1); } lua_pop(L, 1); return result; } static bool equivalent_values(LuaCoreStack &MLS, LuaSlot mval, LuaSlot mtnmap, LuaCoreStack &SLS, LuaSlot sval, LuaSlot stnmap) { switch (MLS.xtype(mval)) { case LUA_TBOOLEAN: { if (SLS.type(sval) != LUA_TBOOLEAN) return false; return MLS.tryboolean(mval) == SLS.tryboolean(sval); } case LUA_TNUMBER: { if (SLS.type(sval) != LUA_TNUMBER) return false; return MLS.trynumber(mval) == SLS.trynumber(sval); } case LUA_TSTRING: { if (SLS.type(sval) != LUA_TSTRING) return false; return MLS.trystring(mval) == SLS.trystring(sval); } case LUA_TLIGHTUSERDATA: { if (SLS.type(sval) != LUA_TLIGHTUSERDATA) return false; return MLS.trytoken(mval) == SLS.trytoken(sval); } case LUA_TFUNCTION: { // Cannot really compare. Just return true if the types match. return SLS.type(sval) == MLS.type(mval); } case LUA_TT_GENERAL: { int midx = get_table_number(MLS, mval, mtnmap); if (midx == 0) { return SLS.isnil(sval); } int sidx = get_table_number(SLS, sval, stnmap); return midx == sidx; } case LUA_TT_CLASS: { if (SLS.xtype(sval) != LUA_TT_CLASS) return false; // What if it's an ill-formed class? return MLS.classname(mval) == SLS.classname(sval); } case LUA_TT_TANGIBLE: { if (SLS.xtype(sval) != LUA_TT_TANGIBLE) return false; // What if it's an ill-formed tangible? return MLS.tanid(mval) == SLS.tanid(sval); } case LUA_TT_GLOBALENV: { return (SLS.xtype(sval) == LUA_TT_GLOBALENV); } default: // We're forcing anything else to go to NIL, // and once it's NIL, we consider it 'equal'. return SLS.type(sval) == LUA_TNIL; } } static void transmit_value(LuaCoreStack &MLS, LuaSlot mval, LuaSlot mtnmap, StreamBuffer *sb) { switch (MLS.xtype(mval)) { case LUA_TBOOLEAN: { sb->write_uint8(LUA_TBOOLEAN); sb->write_bool(*MLS.tryboolean(mval)); return; } case LUA_TNUMBER: { sb->write_uint8(LUA_TNUMBER); sb->write_double(*MLS.trynumber(mval)); return; } case LUA_TSTRING: { sb->write_uint8(LUA_TSTRING); sb->write_string(*MLS.trystring(mval)); return; } case LUA_TLIGHTUSERDATA: { sb->write_uint8(LUA_TLIGHTUSERDATA); sb->write_uint64((*MLS.trytoken(mval)).value); return; } case LUA_TT_GENERAL: { int midx = get_table_number(MLS, mval, mtnmap); if (midx == 0) { sb->write_uint8(LUA_TNIL); } else { sb->write_uint8(LUA_TT_GENERAL); sb->write_uint32(midx); } return; } case LUA_TT_CLASS: { sb->write_uint8(LUA_TT_CLASS); sb->write_string(MLS.classname(mval)); return; } case LUA_TT_TANGIBLE: { sb->write_uint8(LUA_TT_TANGIBLE); sb->write_int64(MLS.tanid(mval)); return; } case LUA_TT_GLOBALENV: { sb->write_uint8(LUA_TT_GLOBALENV); return; } default: sb->write_uint8(LUA_TNIL); return; } } static bool diff_tables(LuaCoreStack &SLS0, LuaSlot stnmap, LuaSlot stab, LuaCoreStack &MLS0, LuaSlot mtnmap, LuaSlot mtab, bool cmeta, StreamBuffer *sb) { LuaVar skey, mkey, sval, mval, mnil; LuaExtStack SLS(SLS0.state(), skey, sval); LuaExtStack MLS(MLS0.state(), mkey, mval, mnil); assert(MLS.istable(mtab)); assert(SLS.istable(stab)); MLS.set(mnil, LuaNil); int nupdates = 0; sb->write_int32(0); int wc = sb->total_writes(); MLS.set(mkey, LuaNil); while (MLS.next(mtab, mkey, mval)) { if (!MLS.issortablekey(mkey)) continue; MLS.movesortablekey(mkey, SLS, skey); SLS.rawget(sval, stab, skey); if (!equivalent_values(MLS, mval, mtnmap, SLS, sval, stnmap)) { transmit_value(MLS, mkey, mtnmap, sb); transmit_value(MLS, mval, mtnmap, sb); nupdates += 1; } } SLS.set(skey, LuaNil); while (SLS.next(stab, skey, sval)) { if (!SLS.issortablekey(skey)) continue; SLS.movesortablekey(skey, MLS, mkey); MLS.rawget(mval, mtab, mkey); if (MLS.isnil(mval)) { transmit_value(MLS, mkey, mtnmap, sb); transmit_value(MLS, mval, mtnmap, sb); nupdates += 1; } } if (cmeta) { SLS.getmetatable(sval, stab); MLS.getmetatable(mval, mtab); if (!equivalent_values(MLS, mval, mtnmap, SLS, sval, stnmap)) { transmit_value(MLS, mnil, mtnmap, sb); transmit_value(MLS, mval, mtnmap, sb); nupdates += 1; } } sb->overwrite_int32(wc, nupdates); return (nupdates > 0); } static void set_transmitted_value(LuaCoreStack &LS, LuaSlot tangibles, LuaSlot ntmap, LuaSlot target, StreamBuffer *sb, const char *dbinfo, DebugCollector *dbc) { int kind = sb->read_uint8(); switch (kind) { case LUA_TBOOLEAN: { bool value = sb->read_bool(); DebugLine(dbc) << dbinfo << (value ? "true" : "false"); LS.set(target, value); return; } case LUA_TNUMBER: { double value = sb->read_double(); DebugLine(dbc) << dbinfo << value; LS.set(target, value); return; } case LUA_TSTRING: { eng::string value = sb->read_string(); DebugLine(dbc) << dbinfo << "'" << value << "'"; LS.set(target, value); return; } case LUA_TLIGHTUSERDATA: { LuaToken value(sb->read_uint64()); DebugLine(dbc) << dbinfo << "[" << value.str() << "]"; LS.set(target, value); return; } case LUA_TT_GENERAL: { int index = sb->read_int32(); DebugLine(dbc) << dbinfo << "table " << index; LS.rawget(target, ntmap, index); return; } case LUA_TT_CLASS: { eng::string value = sb->read_string(); DebugLine(dbc) << dbinfo << "class " << value; LS.makeclass(target, value); return; } case LUA_TT_TANGIBLE: { int64_t id = sb->read_int64(); DebugLine(dbc) << dbinfo << "tan " << id; LS.maketan(target, id); return; } case LUA_TT_GLOBALENV: { DebugLine(dbc) << dbinfo << "global env"; LS.getglobaltable(target); return; } case LUA_TNIL: { DebugLine(dbc) << dbinfo << "nil"; LS.set(target, LuaNil); return; } default: assert(false); // Should not get here. } } static void patch_table(LuaCoreStack &LS0, LuaSlot tangibles, LuaSlot ntmap, LuaSlot tab, StreamBuffer *sb, DebugCollector *dbc) { LuaVar key, val; LuaExtStack LS(LS0.state(), key, val); int ndiffs = sb->read_int32(); for (int i = 0; i < ndiffs; i++) { set_transmitted_value(LS, tangibles, ntmap, key, sb, "key=", dbc); set_transmitted_value(LS, tangibles, ntmap, val, sb, "val=", dbc); if (LS.isnil(key)) { LS.setmetatable(tab, val); } else { LS.rawset(tab, key, val); } } } void World::patch_numbered_tables(StreamBuffer *sb, DebugCollector *dbc) { lua_State *L = state(); LuaVar tangibles, ntmap, tab; LuaExtStack LS(L, tangibles, ntmap, tab); LS.rawget(tangibles, LuaRegistry, "tangibles"); LS.rawget(ntmap, LuaRegistry, "ntmap"); assert(LS.istable(tangibles)); assert(LS.istable(ntmap)); int nmodified = sb->read_int32(); for (int i = 0; i < nmodified; i++) { int index = sb->read_int32(); LS.rawget(tab, ntmap, index); assert(LS.istable(tab)); DebugHeader(dbc) << "Lua Table " << index << ":"; patch_table(LS, tangibles, ntmap, tab, sb, dbc); } } void World::diff_numbered_tables(lua_State *master, StreamBuffer *sb) { lua_State *synch = state(); LuaVar sntmap, mntmap, stnmap, mtnmap, stab, mtab; LuaExtStack SLS(synch, sntmap, stnmap, stab); LuaExtStack MLS(master, mntmap, mtnmap, mtab); SLS.rawget(sntmap, LuaRegistry, "ntmap"); MLS.rawget(mntmap, LuaRegistry, "ntmap"); SLS.rawget(stnmap, LuaRegistry, "tnmap"); MLS.rawget(mtnmap, LuaRegistry, "tnmap"); int m_ntables = MLS.rawlen(mntmap); int s_ntables = SLS.rawlen(sntmap); assert(m_ntables == s_ntables); sb->write_int32(0); int write_count_after = sb->total_writes(); int nmodified = 0; int s_top = lua_gettop(synch); int m_top = lua_gettop(master); for (int id = 1; id <= m_ntables; id++) { MLS.rawget(mtab, mntmap, id); if (MLS.istable(mtab)) { SLS.rawget(stab, sntmap, id); assert(SLS.istable(stab)); int tw = sb->total_writes(); sb->write_int32(id); nmodified += 1; if (!diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, true, sb)) { sb->unwrite_to(tw); nmodified -= 1; } } assert(lua_gettop(synch) == s_top); assert(lua_gettop(master) == m_top); } sb->overwrite_int32(write_count_after, nmodified); } void World::patch_tangible_databases(StreamBuffer *sb, DebugCollector *dbc) { lua_State *L = state(); LuaVar tangibles, ntmap, tab; LuaExtStack LS(L, tangibles, ntmap, tab); LS.rawget(tangibles, LuaRegistry, "tangibles"); LS.rawget(ntmap, LuaRegistry, "ntmap"); assert(LS.istable(tangibles)); assert(LS.istable(ntmap)); int nmodified = sb->read_int32(); for (int i = 0; i < nmodified; i++) { int64_t id = sb->read_int64(); LS.rawget(tab, tangibles, id); assert(LS.istable(tab)); DebugHeader(dbc) << "Tangible DB " << id << ":"; patch_table(LS, tangibles, ntmap, tab, sb, dbc); } } void World::diff_tangible_databases(const IdVector &basis, lua_State *master, StreamBuffer *sb) { lua_State *synch = state(); LuaVar stnmap, mtnmap, stangibles, mtangibles, stab, mtab; LuaExtStack SLS(synch, stnmap, stangibles, stab); LuaExtStack MLS(master, mtnmap, mtangibles, mtab); SLS.rawget(stnmap, LuaRegistry, "tnmap"); MLS.rawget(mtnmap, LuaRegistry, "tnmap"); SLS.rawget(stangibles, LuaRegistry, "tangibles"); MLS.rawget(mtangibles, LuaRegistry, "tangibles"); sb->write_int32(0); int write_count_after = sb->total_writes(); int nmodified = 0; int s_top = lua_gettop(synch); int m_top = lua_gettop(master); for (int64_t id : basis) { MLS.rawget(mtab, mtangibles, id); SLS.rawget(stab, stangibles, id); assert(MLS.istable(mtab)); assert(SLS.istable(stab)); int tw = sb->total_writes(); sb->write_int64(id); nmodified += 1; if (!diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, false, sb)) { sb->unwrite_to(tw); nmodified -= 1; } assert(lua_gettop(synch) == s_top); assert(lua_gettop(master) == m_top); } sb->overwrite_int32(write_count_after, nmodified); } //////////////////////////////////////////////////////////////////// // // Unit Testing Framework for Table Diff // // The function test_diffcompare creates a standalone lua environment, // evaluates two lua expressions to build a master table and a synch // table, runs diff_tables, and returns the result as a debug string. // //////////////////////////////////////////////////////////////////// #include "source.hpp" #include "traceback.hpp" #include "pprint.hpp" #include "wrap-sstream.hpp" class DiffTester { lua_State *master_L_ = nullptr; lua_State *synch_L_ = nullptr; lua_State *caller_; SourceDB master_sdb_; SourceDB synch_sdb_; static void transmit_value_debug_string(StreamBuffer *sb, eng::ostringstream &oss) { int kind = sb->read_uint8(); switch (kind) { case LUA_TBOOLEAN: { bool b = sb->read_bool(); oss << (b ? "true":"false"); return; } case LUA_TNUMBER: { oss << sb->read_double(); return; } case LUA_TSTRING: { oss << sb->read_string(); return; } case LUA_TLIGHTUSERDATA: { LuaToken token(sb->read_uint64()); oss << "[" << token.str() << "]"; return; } case LUA_TT_GENERAL: { oss << "tab" << sb->read_int32(); return; } case LUA_TT_CLASS: { oss << "class " << sb->read_string(); return; } case LUA_TT_TANGIBLE: { oss << "tan " << sb->read_int64(); return; } case LUA_TT_GLOBALENV: { oss << "globals"; return; } case LUA_TNIL: { oss << "nil"; return; } default: assert(false); // Should not get here. } } static eng::string diff_tables_debug_string(StreamBuffer *sb) { eng::vector sorted; eng::ostringstream oss; int ndiffs = sb->read_int32(); for (int i = 0; i < ndiffs; i++) { transmit_value_debug_string(sb, oss); oss << "="; transmit_value_debug_string(sb, oss); sorted.push_back(oss.str()); oss.str(""); } std::sort(sorted.begin(), sorted.end()); for (const eng::string &s : sorted) { oss << s << ";"; } return oss.str(); } public: DiffTester(lua_State *caller) : caller_(caller) { SourceDB::register_lua_builtins(); master_L_ = LuaCoreStack::newstate(eng::l_alloc); synch_L_ = LuaCoreStack::newstate(eng::l_alloc); master_sdb_.init(master_L_); synch_sdb_.init(synch_L_); } ~DiffTester() { if (master_L_) lua_close(master_L_); if (synch_L_) lua_close(synch_L_); } // Create stock test tables and their maps in a single lua environment. // Sets up globals tab1-tab9 and populates the tnmap and ntmap. void create_stock_tables(LuaCoreStack &LS, LuaSlot tnmap, LuaSlot ntmap) { lua_State *L = LS.state(); LuaVar globtab; LuaExtStack LS2(L, globtab); LS.set(tnmap, LuaNewTable); LS.set(ntmap, LuaNewTable); LS.getglobaltable(globtab); for (int i = 1; i <= 9; i++) { LuaVar tabi; LuaExtStack LS3(L, tabi); LS.set(tabi, LuaNewTable); LS.rawset(tabi, 1, util::ss("tab", i)); LS.rawset(globtab, util::ss("tab", i), tabi); LS.rawset(tnmap, tabi, i); LS.rawset(ntmap, i, tabi); } } // Pretty-print a lua value to a string. eng::string pprint(LuaCoreStack &LS, LuaSlot val) { std::ostringstream oss; PrettyPrint::Indented().print(LS, val, &oss); return eng::string(oss.str()); } // Evaluate a lua expression and store the result in the given slot. void eval(LuaCoreStack &LS, LuaSlot dest, const char *expr, const char *name) { lua_State *L = LS.state(); LuaVar closure; LuaExtStack LS2(L, closure); eng::string err = LS2.load(closure, util::ss("return ", expr), name); LuaAssert(caller_, err.empty()); lua_pushvalue(L, closure.index()); err = traceback_pcall(L, 0, 1); LuaAssert(caller_, err.empty()); lua_replace(L, dest.index()); } eng::string diffcompare(const char *master_expr, const char *synch_expr) { assert(lua_gettop(master_L_) == 0); assert(lua_gettop(synch_L_) == 0); LuaVar mtnmap, mntmap, mtab; LuaExtStack MLS(master_L_, mtnmap, mntmap, mtab); LuaVar stnmap, sntmap, stab; LuaExtStack SLS(synch_L_, stnmap, sntmap, stab); create_stock_tables(MLS, mtnmap, mntmap); create_stock_tables(SLS, stnmap, sntmap); eval(MLS, mtab, master_expr, "master"); eval(SLS, stab, synch_expr, "synch"); // Run diff_tables. StreamBuffer sb; diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, true, &sb); return diff_tables_debug_string(&sb); } bool diffapply(const char *master_expr, const char *synch_expr, bool verbose = false) { assert(lua_gettop(master_L_) == 0); assert(lua_gettop(synch_L_) == 0); LuaVar mtnmap, mntmap, mtab; LuaExtStack MLS(master_L_, mtnmap, mntmap, mtab); LuaVar stnmap, sntmap, stab, tangibles; LuaExtStack SLS(synch_L_, stnmap, sntmap, stab, tangibles); create_stock_tables(MLS, mtnmap, mntmap); create_stock_tables(SLS, stnmap, sntmap); eval(MLS, mtab, master_expr, "master"); eval(SLS, stab, synch_expr, "synch"); // Get the tangibles map from the synch environment. SLS.rawget(tangibles, LuaRegistry, "tangibles"); // Diff and patch. StreamBuffer sb; diff_tables(SLS, stnmap, stab, MLS, mtnmap, mtab, true, &sb); if (verbose) { StreamBuffer sb_copy(sb.view()); eng::string diff_str = diff_tables_debug_string(&sb_copy); printf("diffapply: master_expr = %s\n", master_expr); printf("diffapply: synch_expr = %s\n", synch_expr); printf("diffapply: diff = %s\n", diff_str.c_str()); } patch_table(SLS, tangibles, sntmap, stab, &sb, nullptr); // Check equality by pretty-printing both tables and comparing. eng::string master_str = pprint(MLS, mtab); eng::string synch_str = pprint(SLS, stab); if (verbose) { printf("diffapply: master pprint (%d chars) = %s\n", (int)master_str.size(), master_str.c_str()); printf("diffapply: synch pprint (%d chars) = %s\n", (int)synch_str.size(), synch_str.c_str()); } return master_str == synch_str; } }; LuaDefine(unittests_difftab, "", "unit tests for table diff") { DiffTester dt(L); // No differences in these simple-valued tables. LuaAssertStrEq(L, dt.diffcompare("{a=true}", "{a=true}"), ""); LuaAssertStrEq(L, dt.diffcompare("{a=5}", "{a=5}"), ""); LuaAssertStrEq(L, dt.diffcompare("{a='foo'}", "{a='foo'}"), ""); // Test transmission of missing simple values. LuaAssertStrEq(L, dt.diffcompare("{a=true}", "{}"), "a=true;"); LuaAssertStrEq(L, dt.diffcompare("{a=5}", "{}"), "a=5;"); LuaAssertStrEq(L, dt.diffcompare("{a='foo'}", "{}"), "a=foo;"); // Test the replacement of simple values. LuaAssertStrEq(L, dt.diffcompare("{a=true}", "{a=false}"), "a=true;"); LuaAssertStrEq(L, dt.diffcompare("{a=5}", "{a=4}"), "a=5;"); LuaAssertStrEq(L, dt.diffcompare("{a='foo'}", "{a='bar'}"), "a=foo;"); // Test the clearing of values. LuaAssertStrEq(L, dt.diffcompare("{}", "{a=true}"), "a=nil;"); LuaAssertStrEq(L, dt.diffcompare("{}", "{a=5}"), "a=nil;"); LuaAssertStrEq(L, dt.diffcompare("{}", "{a='foo'}"), "a=nil;"); // Try boolean keys. LuaAssertStrEq(L, dt.diffcompare("{[true]=3}", "{}"), "true=3;"); LuaAssertStrEq(L, dt.diffcompare("{}", "{[true]=3}"), "true=nil;"); // Try number keys. LuaAssertStrEq(L, dt.diffcompare("{[7]=3}", "{}"), "7=3;"); LuaAssertStrEq(L, dt.diffcompare("{}", "{[7]=3}"), "7=nil;"); // Try a table with multiple keys. LuaAssertStrEq(L, dt.diffcompare("{a=4, b=5, c=6}", "{b=5, c=7, d=8}"), "a=4;c=6;d=nil;"); // Nonsortable keys should be ignored (no diffs). LuaAssertStrEq(L, dt.diffcompare("{[{}]=3}", "{}"), ""); // Numbered tables: matching pairs produce no diff. LuaAssertStrEq(L, dt.diffcompare("{a=tab1}", "{a=tab1}"), ""); // Numbered tables: missing in synch. LuaAssertStrEq(L, dt.diffcompare("{a=tab1}", "{}"), "a=tab1;"); // Numbered tables: wrong table number. LuaAssertStrEq(L, dt.diffcompare("{a=tab1}", "{a=tab2}"), "a=tab1;"); // Numbered tables: replaced by simple value. LuaAssertStrEq(L, dt.diffcompare("{a=3}", "{a=tab1}"), "a=3;"); // Numbered tables: cleared. LuaAssertStrEq(L, dt.diffcompare("{}", "{a=tab1}"), "a=nil;"); // Unnumbered tables are forced to nil. LuaAssertStrEq(L, dt.diffcompare("{a={}}", "{}"), ""); LuaAssertStrEq(L, dt.diffcompare("{a={}}", "{a=3}"), "a=nil;"); // Class values. LuaAssertStrEq(L, dt.diffcompare("{a=deque}", "{}"), "a=class deque;"); // Global environment. LuaAssertStrEq(L, dt.diffcompare("{a=_G}", "{}"), "a=globals;"); // Metatable: set, match, clear. LuaAssertStrEq(L, dt.diffcompare("setmetatable({}, tab1)", "{}"), "nil=tab1;"); LuaAssertStrEq(L, dt.diffcompare("setmetatable({}, tab1)", "setmetatable({}, tab1)"), ""); LuaAssertStrEq(L, dt.diffcompare("{}", "setmetatable({}, tab1)"), "nil=nil;"); // Diff-apply: verify simple values. LuaAssert(L, dt.diffapply("{a=1}", "{}")); LuaAssert(L, dt.diffapply("{[true]='foo'}", "{}")); LuaAssert(L, dt.diffapply("{[3]=false}", "{}")); // Diff-apply: multiple simple values. LuaAssert(L, dt.diffapply("{a=1, b=2, c=3}", "{}")); // Diff-apply: remove or replace wrong values. LuaAssert(L, dt.diffapply("{a=1, b=2}", "{b=3, c=4}")); // Diff-apply: table containing a numbered table. LuaAssert(L, dt.diffapply("{a=tab1, b=tab2}", "{}")); // Diff-apply: table containing a class. LuaAssert(L, dt.diffapply("{a=deque, b=table}", "{}")); // Diff-apply: table containing the global environment. LuaAssert(L, dt.diffapply("{a=_G}", "{}")); // Diff-apply: unnumbered tables are forced to nil. LuaAssert(L, !dt.diffapply("{a={}}", "{a=3}")); // Diff-apply: set metatable. LuaAssert(L, dt.diffapply("setmetatable({}, tab1)", "{}")); // Diff-apply: clear metatable. LuaAssert(L, dt.diffapply("{}", "setmetatable({}, tab1)")); return 0; }