diff --git a/luprex/core/cpp/idalloc.cpp b/luprex/core/cpp/idalloc.cpp index 7fb4bd29..c4a908d4 100644 --- a/luprex/core/cpp/idalloc.cpp +++ b/luprex/core/cpp/idalloc.cpp @@ -19,6 +19,7 @@ IdGlobalPool::IdGlobalPool() { salvaged_.clear(); next_batch_ = 0; next_id_ = 0; + next_seqno_ = 0; } IdGlobalPool::~IdGlobalPool() { @@ -66,6 +67,7 @@ void IdGlobalPool::salvage(int64_t batch) { void IdGlobalPool::serialize(StreamBuffer *sb) const { sb->write_int64(next_batch_); sb->write_int64(next_id_); + sb->write_uint64(next_seqno_); sb->write_uint32(salvaged_.size()); for (int64_t batch : salvaged_) { sb->write_int64(batch); @@ -75,6 +77,7 @@ void IdGlobalPool::serialize(StreamBuffer *sb) const { void IdGlobalPool::deserialize(StreamBuffer *sb) { next_batch_ = sb->read_int64(); next_id_ = sb->read_int64(); + next_seqno_ = sb->read_uint64(); uint32_t salvaged_size = sb->read_uint32(); salvaged_.resize(salvaged_size); for (int i=0; i < int(salvaged_size); i++) { @@ -86,6 +89,7 @@ eng::string IdGlobalPool::debug_string() const { eng::ostringstream oss; oss << "next_batch:" << util::hex64() << next_batch_ << " "; oss << "next_id:" << util::hex64() << next_id_ << " "; + oss << "next_seqno: " << util::hex64() << next_seqno_ << " "; oss << "salvaged:"; for (const int64_t val : salvaged_) { oss << " " << util::hex64() << val; @@ -96,6 +100,7 @@ eng::string IdGlobalPool::debug_string() const { IdPlayerPool::IdPlayerPool(IdGlobalPool *g) { global_ = g; fifo_capacity_ = 0; + next_seqno_ = 0; } IdPlayerPool::~IdPlayerPool() { @@ -149,6 +154,7 @@ int64_t IdPlayerPool::get_one() { void IdPlayerPool::serialize(StreamBuffer *sb) const { sb->write_uint8(fifo_capacity_); sb->write_uint8(ranges_.size()); + sb->write_uint64(next_seqno_); for (int64_t batch : ranges_) { sb->write_int64(batch); } @@ -157,6 +163,7 @@ void IdPlayerPool::serialize(StreamBuffer *sb) const { void IdPlayerPool::deserialize(StreamBuffer *sb) { fifo_capacity_ = sb->read_uint8(); int ranges_size = sb->read_uint8(); + next_seqno_ = sb->read_uint64(); ranges_.resize(ranges_size); for (int i=0; i < ranges_size; i++) { ranges_[i] = sb->read_int64(); @@ -166,6 +173,7 @@ void IdPlayerPool::deserialize(StreamBuffer *sb) { bool IdPlayerPool::exactly_equal(const IdPlayerPool &other) const { if (fifo_capacity_ != other.fifo_capacity_) return false; if (ranges_.size() != other.ranges_.size()) return false; + if (next_seqno_ != other.next_seqno_) return false; for (int i = 0; i < int(ranges_.size()); i++) { if (ranges_[i] != other.ranges_[i]) { return false; @@ -190,10 +198,11 @@ void IdPlayerPool::diff(const IdPlayerPool &auth, StreamBuffer *sb) const { return; } - // Write the fifo capacity and nranges + // Write the fifo capacity, nranges, and next seqno assert(auth.fifo_capacity_ != 255); sb->write_uint8(auth.fifo_capacity_); sb->write_uint8(auth.ranges_.size()); + sb->write_uint64(auth.next_seqno_); // Build up an index of the known IDs. eng::map index; @@ -225,6 +234,7 @@ void IdPlayerPool::patch(StreamBuffer *sb, DebugCollector *dbc) { DebugLine(dbc) << "IdPlayerPool modified"; fifo_capacity_ = fifo_cap; int nranges = sb->read_uint8(); + next_seqno_ = sb->read_uint64(); eng::deque old = std::move(ranges_); ranges_.clear(); for (int i = 0; i < nranges; i++) { @@ -245,6 +255,7 @@ eng::string IdPlayerPool::debug_string() const { if (i > 0) oss << ","; oss << util::hex64() << ranges_[i]; } + oss << " seqno:" << util::hex64() << next_seqno_; return oss.str(); } diff --git a/luprex/core/cpp/idalloc.hpp b/luprex/core/cpp/idalloc.hpp index ff32a411..c59fa197 100644 --- a/luprex/core/cpp/idalloc.hpp +++ b/luprex/core/cpp/idalloc.hpp @@ -37,11 +37,21 @@ // // THE NUMERIC RANGES // -// * 0x0000+ : reserved for manually-created tangible IDs. +// The largest integer that can be stored losslessly in a lua_Number is: +// +// * 0x0020000000000000 +// +// In other words, any 53-bit number can be stored. We subdivide the range as +// follows: +// +// * 0x0000+ : manually created IDs. // * 0x0001+ : used by master model's IdGlobalPool to create batches. // * 0x0010+ : used by master model's IdGlobalPool to create individual IDs. // * 0x001E+ : used by sync model's IdGlobalPool to create individual IDs. // +// If you exhaust the Master Model's invididual pool at a rate of 10,000,000 IDs +// per second, the individual pool will last for 12 years. +// // BATCH REPRESENTATION // // A batch is represented as a 64-bit integer. The batch contains all IDs @@ -60,6 +70,22 @@ // // As a special case, the number 0 is used to indicate "invalid batch". // +// SEQUENCE NUMBERS +// +// If you allocate IDs from a player pool rapidly, you will exhaust the fifo in +// the player pool. As a result, ID allocation will fall back to the global +// pool, which will cause prediction failures. +// +// Depending on your requirements, you may be able to use sequence numbers +// instead of IDs. The player pool contains a monotonically increasing counter +// for generating sequence numbers. These can be allocated extremely fast +// and they'll never run out. These are highly predictable as long as the +// only person fetching sequence numbers is the player himself. The downside +// is that sequence numbers are not globally unique - they're only locally +// unique for a single player. That may be enough for your requirements. +// +// Currently, sequence numbers are used by the random number generator. +// /////////////////////////////////////////////////////////////////// #ifndef IDALLOC_HPP @@ -105,6 +131,10 @@ public: // zero, or the batch contains fewer than 128 IDs. void salvage(int64_t batch); + // Get a global sequence number. These are not the same as IDs. + // Sequence numbers are unlikely to be predicted correctly. + int64_t get_seqno() { return next_seqno_++; } + // Serialize to or deserialize from a streambuffer. void serialize(StreamBuffer *sb) const; void deserialize(StreamBuffer *sb); @@ -116,6 +146,7 @@ private: eng::vector salvaged_; int64_t next_batch_; int64_t next_id_; + int64_t next_seqno_; friend int unittests_idalloc(lua_State *L); }; @@ -143,6 +174,10 @@ public: // Get a single ID from the fifo. Also refills the fifo. int64_t get_one(); + // Get a player sequence number. This is not the same as an ID: player + // sequence numbers are unique per player, but not globally. + int64_t get_seqno() { return next_seqno_++; } + // Return the size of the queue. int size() { return ranges_.size(); } @@ -177,6 +212,7 @@ public: private: IdGlobalPool *global_; int fifo_capacity_; + int64_t next_seqno_; eng::deque ranges_; friend int lfn_unittests_idalloc(lua_State *L); }; diff --git a/luprex/core/cpp/luastack.hpp b/luprex/core/cpp/luastack.hpp index 051c27d0..6313079f 100644 --- a/luprex/core/cpp/luastack.hpp +++ b/luprex/core/cpp/luastack.hpp @@ -334,6 +334,10 @@ public: int result(); public: + // This is the largest integer that can be stored in a lua_Number. + // In other words, any 53-bit number can be stored. + static const int64_t MAXINT = 0x001FFFFFFFFFFFFF; + static lua_State *newstate (lua_Alloc allocf); lua_State *state() const { return L_; } diff --git a/luprex/core/cpp/source.cpp b/luprex/core/cpp/source.cpp index 81117469..1a7f32e2 100644 --- a/luprex/core/cpp/source.cpp +++ b/luprex/core/cpp/source.cpp @@ -351,6 +351,7 @@ eng::string SourceDB::rebuild() { LS.makeclass(mathclass, "math"); LS.rawset(mathclass, "pi", M_PI); LS.rawset(mathclass, "huge", HUGE_VAL); + LS.rawset(mathclass, "maxint", LuaStack::MAXINT); LS.result(); return errs; @@ -734,14 +735,15 @@ LuaDefineBuiltin(math_min, "x, x, x...", "return the smallest argument"); LuaDefineBuiltin(math_modf, "x", "returns the integral and fractional part of x"); LuaDefineBuiltin(math_pow, "x, y", "returns x ^ y, equivalent to the operator"); LuaDefineBuiltin(math_rad, "deg", "convert degrees to radians"); -LuaDefineBuiltin(math_random, "[m [, n]]", "return random [0.0-1.0), or [1-m], or [m-n]."); -LuaDefineBuiltin(math_randomseed, "x", "set x as the seed for random numbers"); LuaDefineBuiltin(math_sin, "x", "return the sine of x in radians"); LuaDefineBuiltin(math_sinh, "x", "return the hyperbolic sine of x in radians"); LuaDefineBuiltin(math_sqrt, "x", "return the square root of x"); LuaDefineBuiltin(math_tan, "x", "return the tangent of x in radians"); LuaDefineBuiltin(math_tanh, "x", "return the hyperbolic tangent of x in radians"); LuaSandboxBuiltin(math_log10, "", ""); +// math.random and math.randomseed are in world-accessor.cpp, because +// generating random numbers must manipulate global state which is +// stored in the world model. LuaDefineBuiltin(assert, "flag [,message]", "assert that flag is true, if not, raise error"); LuaDefineBuiltin(error, "message", "raise an error"); diff --git a/luprex/core/cpp/util.cpp b/luprex/core/cpp/util.cpp index 172e4bc9..fdff486a 100644 --- a/luprex/core/cpp/util.cpp +++ b/luprex/core/cpp/util.cpp @@ -131,6 +131,31 @@ eng::string hash_to_hex(const HashValue &hv) { oss << std::hex << std::setw(16) << std::setfill('0') << hv.second; return oss.str(); } +static inline uint64_t Rot64(uint64_t x, int k) +{ + return (x << k) | (x >> (64 - k)); +} + +uint64_t hash_ints(uint64_t a, uint64_t b, uint64_t c, uint64_t d) { + uint64_t h0 = c ^ 0xc548cebf3714dbb9; + uint64_t h1 = d ^ 0xd23a7edd44383f8d; + uint64_t h2 = a ^ 0x7356f92e4b154df7; + uint64_t h3 = b ^ 0x55ce09295766838d; + + h3 ^= h2; h2 = Rot64(h2,15); h3 += h2; + h0 ^= h3; h3 = Rot64(h3,52); h0 += h3; + h1 ^= h0; h0 = Rot64(h0,26); h1 += h0; + h2 ^= h1; h1 = Rot64(h1,51); h2 += h1; + h3 ^= h2; h2 = Rot64(h2,28); h3 += h2; + h0 ^= h3; h3 = Rot64(h3,9); h0 += h3; + h1 ^= h0; h0 = Rot64(h0,47); h1 += h0; + h2 ^= h1; h1 = Rot64(h1,54); h2 += h1; + h3 ^= h2; h2 = Rot64(h2,32); h3 += h2; + h0 ^= h3; h3 = Rot64(h3,25); h0 += h3; + h1 ^= h0; h0 = Rot64(h0,63); h1 += h0; + + return h1; +} StringVec split(const eng::string &s, char sep) { StringVec result; @@ -331,7 +356,6 @@ std::string_view sv_read_line(std::string_view &source) { } - double distance_squared(double x1, double y1, double x2, double y2) { double dx = x1 - x2; double dy = y1 - y2; diff --git a/luprex/core/cpp/util.hpp b/luprex/core/cpp/util.hpp index 6ea0d13e..46bf2961 100644 --- a/luprex/core/cpp/util.hpp +++ b/luprex/core/cpp/util.hpp @@ -57,15 +57,19 @@ eng::string id_vector_debug_string(const IdVector &idv); // Unions and sorts two ID vectors. IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2); -// Get a 64-bit hashvalue for a string. +// Get a 128-bit hashvalue for a string. HashValue hash_string(const eng::string &str); -// Get a 64-bit hashvalue for an ID vector. +// Get a 128-bit hashvalue for an ID vector. HashValue hash_id_vector(const IdVector &idv); -// Convert a hash to a hexadecimal string. +// Convert a 128-bit hash to a hexadecimal string. eng::string hash_to_hex(const HashValue &hash); +// Hash four integers together to 64 bits. +// This is a good hash, but not cryptographically good. +uint64_t hash_ints(uint64_t n1, uint64_t n2, uint64_t n3, uint64_t n4); + // Split a string into multiple strings StringVec split(const eng::string &s, char sep); diff --git a/luprex/core/cpp/world-accessor.cpp b/luprex/core/cpp/world-accessor.cpp index 878f1137..0e615b40 100644 --- a/luprex/core/cpp/world-accessor.cpp +++ b/luprex/core/cpp/world-accessor.cpp @@ -1,6 +1,8 @@ #include "world.hpp" #include "pprint.hpp" +#include +#include static void tangible_getall(LuaStack &LS0, LuaSlot list, const util::IdVector &idv) { LuaVar tangibles, tan; @@ -311,6 +313,161 @@ LuaDefine(tangible_nopredict, "", } } +LuaDefine(math_random, "(args...)", + "|Generate random numbers." + "|" + "|What it generates depends on the arguments:" + "|" + "| () - a float in range [0.0, 1.0)" + "| (high) - an int between 1 and high inclusive" + "| (low, high) - an int between low and high inclusive" + "|" + "|math.random tries to cooperate with predictive" + "|reexecution to be as predictable as possible." + "|To achieve predictability, we used an ad-hoc" + "|random number generator. It passes a variety of" + "|statistical tests, but it's not well-studied." + "|" + "|If you want actually want nonpredictability, or" + "|if you need the assurance of a well-studied random" + "|number generator, use math.mtrandom or" + "|math.cryptrandom instead.") { + // Parse the arguments. + // This is hairy because there's a lot of possibilities. + bool passed_in_randomstate = false; + int arg = 1; + if ((lua_gettop(L) >= arg) && (lua_istable(L, arg))) { + passed_in_randomstate = true; + arg += 1; + } + bool have_range = false; + int64_t low, high; + if ((lua_gettop(L) >= arg) && (lua_type(L, arg) == LUA_TNUMBER)) { + double lowf, highf; + if ((lua_gettop(L) >= arg+1) && (lua_type(L, arg+1) == LUA_TNUMBER)) { + lowf = std::floor(lua_tonumber(L, arg)); + highf = std::floor(lua_tonumber(L, arg + 1)); + arg += 2; + } else { + lowf = 1; + highf = std::floor(lua_tonumber(L, arg)); + arg += 1; + } + if ((lowf < -LuaStack::MAXINT) || (highf > LuaStack::MAXINT)) { + luaL_error(L, "math.random range exceeds MAXINT"); + return 0; + } + if (lowf > highf) { + luaL_error(L, "math.random range low > high"); + return 0; + } + low = int64_t(lowf); + high = int64_t(highf); + have_range = true; + } + if (lua_gettop(L) >= arg) { + luaL_error(L, "math.random accepts an optional randomstate and an optional range"); + return 0; + } + + // Generate the seed, count, and salt. + // The salt prevents accidental duplication between user-specified + // seeds and system-generated seeds. + uint64_t seed, count, salt; + if (passed_in_randomstate) { + lua_pushstring(L, "seed"); + lua_rawget(L, 1); + lua_pushstring(L, "count"); + lua_rawget(L, 1); + if ((lua_type(L, -1) != LUA_TNUMBER) || + (lua_type(L, -2) != LUA_TNUMBER)) { + luaL_error(L, "Not a valid randomstate table"); + return 0; + } + double dseed = lua_tonumber(L, -2); + double dcount = lua_tonumber(L, -1); + seed = uint64_t(dseed) & LuaStack::MAXINT; + count = uint64_t(dcount) & LuaStack::MAXINT; + if (dseed < 0) { + salt = 0x35c9a6082a097ade; + } else { + salt = 0x4785d086ead90c20; + } + lua_pop(L, 2); + lua_pushstring(L, "count"); + lua_pushnumber(L, double((count + 1) & LuaStack::MAXINT)); + lua_rawset(L, 1); + } else { + World *w = World::fetch_global_pointer(L); + if (w->lthread_use_ppool_) { + Tangible *actor = w->tangible_get(w->lthread_actor_id_); + seed = w->lthread_actor_id_; + count = actor->id_player_pool_.get_seqno(); + salt = 0x3ab0fb84aedc3764; + } else { + // TODO: maybe throw in a 'donotpredict' here. + seed = 123456; + count = w->id_global_pool_.get_seqno(); + salt = 0x6f493c90faf0139d; + } + } + + // Generate the hash and convert to a lua_Number. + uint64_t hash = util::hash_ints(seed, count, salt, 456); + if (!have_range) { + double result = (hash & LuaStack::MAXINT) * 0x1p-53; + lua_pushnumber(L, result); + } else { + uint64_t range = (high - low) + 1; + uint64_t offset = (hash & 0x7FFFFFFFFFFFFFFF) % range; + int64_t result = low + int64_t(offset); + lua_pushnumber(L, result); + } + return 1; +} + +LuaDefine(math_randomstate, "(seed)", + "|Create and return a randomstate table." + "|This is a lua table that stores the state for a random" + "|number generator. A randomstate table can be passed" + "|to math.random." + "|" + "|You can optionally omit the seed, in which case it will" + "|pick a seed randomly. Automatically-generated seeds are" + "|guaranteed never to be the same as user-specified seeds.") { + double seed; + if (lua_gettop(L) == 0) { + World *w = World::fetch_global_pointer(L); + int64_t iseed = (w->id_global_pool_.get_seqno() & LuaStack::MAXINT) + 1; + seed = -iseed; + } else if (lua_gettop(L) == 1) { + if (lua_type(L, 1) != LUA_TNUMBER) { + luaL_error(L, "math.randomstate takes an optional integer seed"); + return 0; + } + seed = lua_tonumber(L, 1); + if ((seed < 0.0) || (seed > LuaStack::MAXINT) || (std::floor(seed) != seed)) { + luaL_error(L, "math.randomstate seed must be an integer 0-MAXINT"); + return 0; + } + } else { + luaL_error(L, "math.randomstate takes an optional integer seed"); + return 0; + } + + lua_newtable(L); + lua_pushstring(L, "seed"); + lua_pushnumber(L, seed); + lua_rawset(L, -3); + lua_pushstring(L, "count"); + lua_pushnumber(L, 0); + lua_rawset(L, -3); + return 1; +} + +LuaSandboxBuiltin(math_randomseed, "", ""); + + LuaDefine(pprint, "obj1,obj2,...", "|Pretty-print object or objects.") { World *w = World::fetch_global_pointer(L); diff --git a/luprex/core/cpp/world.hpp b/luprex/core/cpp/world.hpp index 2154dbc8..d9c64c7a 100644 --- a/luprex/core/cpp/world.hpp +++ b/luprex/core/cpp/world.hpp @@ -501,6 +501,8 @@ private: friend int lfn_tangible_nopredict(lua_State *L); friend int lfn_tangible_near(lua_State *L); friend int lfn_tangible_scan(lua_State *L); + friend int lfn_math_random(lua_State *L); + friend int lfn_math_randomstate(lua_State *L); }; using UniqueWorld = std::unique_ptr; diff --git a/luprex/experiments/bigid.cpp b/luprex/experiments/bigid.cpp new file mode 100644 index 00000000..f6e2ecfe --- /dev/null +++ b/luprex/experiments/bigid.cpp @@ -0,0 +1,43 @@ +#include +#include + +bool storable(int64_t n) { + double d = n; + int64_t n1 = int64_t(d); + return n1 == n; +} + + +// Find the biggest number where the number is storable, +// and all numbers smaller are also storable. +int64_t find_biggest() { + int64_t v = 256; + int64_t best = 256; + while (true) { + for (int i = 0; i < 100; i++) { + if (!storable(v - i)) { + return best; + } + } + best = v; + v <<= 1; + } +} + + +int main(int argc, char **argv) { + int64_t best = find_biggest(); + + printf("%016lx ", best); + for (int i = -12; i <= 12; i++) { + if (i == 0) printf(" "); + if (storable(best + i)) { + printf("* "); + } else { + printf("- "); + } + if (i == 0) printf(" "); + } + printf("\n"); + return 0; +}