Check in code for new random number generator

luprex/core/cpp/idalloc.cpp

@@ -19,6 +19,7 @@ IdGlobalPool::IdGlobalPool() {
     salvaged_.clear();
     next_batch_ = 0;
     next_id_ = 0;
+    next_seqno_ = 0;
 }
 
 IdGlobalPool::~IdGlobalPool() {
@@ -66,6 +67,7 @@ void IdGlobalPool::salvage(int64_t batch) {
 void IdGlobalPool::serialize(StreamBuffer *sb) const {
     sb->write_int64(next_batch_);
     sb->write_int64(next_id_);
+    sb->write_uint64(next_seqno_);
     sb->write_uint32(salvaged_.size());
     for (int64_t batch : salvaged_) {
         sb->write_int64(batch);
@@ -75,6 +77,7 @@ void IdGlobalPool::serialize(StreamBuffer *sb) const {
 void IdGlobalPool::deserialize(StreamBuffer *sb) {
     next_batch_ = sb->read_int64();
     next_id_ = sb->read_int64();
+    next_seqno_ = sb->read_uint64();
     uint32_t salvaged_size = sb->read_uint32();
     salvaged_.resize(salvaged_size);
     for (int i=0; i < int(salvaged_size); i++) {
@@ -86,6 +89,7 @@ eng::string IdGlobalPool::debug_string() const {
     eng::ostringstream oss;
     oss << "next_batch:" << util::hex64() << next_batch_ << " ";
     oss << "next_id:" << util::hex64() << next_id_ << " ";
+    oss << "next_seqno: " << util::hex64() << next_seqno_ << " ";
     oss << "salvaged:";
     for (const int64_t val : salvaged_) {
         oss << " " << util::hex64() << val;
@@ -96,6 +100,7 @@ eng::string IdGlobalPool::debug_string() const {
 IdPlayerPool::IdPlayerPool(IdGlobalPool *g) {
     global_ = g;
     fifo_capacity_ = 0;
+    next_seqno_ = 0;
 }
 
 IdPlayerPool::~IdPlayerPool() {
@@ -149,6 +154,7 @@ int64_t IdPlayerPool::get_one() {
 void IdPlayerPool::serialize(StreamBuffer *sb) const {
     sb->write_uint8(fifo_capacity_);
     sb->write_uint8(ranges_.size());
+    sb->write_uint64(next_seqno_);
     for (int64_t batch : ranges_) {
         sb->write_int64(batch);
     }
@@ -157,6 +163,7 @@ void IdPlayerPool::serialize(StreamBuffer *sb) const {
 void IdPlayerPool::deserialize(StreamBuffer *sb) {
     fifo_capacity_ = sb->read_uint8();
     int ranges_size = sb->read_uint8();
+    next_seqno_ = sb->read_uint64();
     ranges_.resize(ranges_size);
     for (int i=0; i < ranges_size; i++) {
         ranges_[i] = sb->read_int64();
@@ -166,6 +173,7 @@ void IdPlayerPool::deserialize(StreamBuffer *sb) {
 bool IdPlayerPool::exactly_equal(const IdPlayerPool &other) const {
     if (fifo_capacity_ != other.fifo_capacity_) return false;
     if (ranges_.size() != other.ranges_.size()) return false;
+    if (next_seqno_ != other.next_seqno_) return false;
     for (int i = 0; i < int(ranges_.size()); i++) {
         if (ranges_[i] != other.ranges_[i]) {
             return false;
@@ -190,10 +198,11 @@ void IdPlayerPool::diff(const IdPlayerPool &auth, StreamBuffer *sb) const {
         return;
     }
 
-    // Write the fifo capacity and nranges
+    // Write the fifo capacity, nranges, and next seqno
     assert(auth.fifo_capacity_ != 255);
     sb->write_uint8(auth.fifo_capacity_);
     sb->write_uint8(auth.ranges_.size());
+    sb->write_uint64(auth.next_seqno_);
     
     // Build up an index of the known IDs.
     eng::map<int64_t, int> index;
@@ -225,6 +234,7 @@ void IdPlayerPool::patch(StreamBuffer *sb, DebugCollector *dbc) {
     DebugLine(dbc) << "IdPlayerPool modified";
     fifo_capacity_ = fifo_cap;
     int nranges = sb->read_uint8();
+    next_seqno_ = sb->read_uint64();
     eng::deque<int64_t> old = std::move(ranges_);
     ranges_.clear();
     for (int i = 0; i < nranges; i++) {
@@ -245,6 +255,7 @@ eng::string IdPlayerPool::debug_string() const {
         if (i > 0) oss << ",";
         oss << util::hex64() << ranges_[i];
     }
+    oss << " seqno:" << util::hex64() << next_seqno_;
     return oss.str();
 }
 

luprex/core/cpp/idalloc.hpp

@@ -37,11 +37,21 @@
 //
 // THE NUMERIC RANGES
 //
-//  * 0x0000+  : reserved for manually-created tangible IDs.
+// The largest integer that can be stored losslessly in a lua_Number is:
+//
+//  * 0x0020000000000000
+//
+// In other words, any 53-bit number can be stored.  We subdivide the range as
+// follows:
+//
+//  * 0x0000+  : manually created IDs.
 //  * 0x0001+  : used by master model's IdGlobalPool to create batches.
 //  * 0x0010+  : used by master model's IdGlobalPool to create individual IDs.
 //  * 0x001E+  : used by sync model's IdGlobalPool to create individual IDs.
 //
+// If you exhaust the Master Model's invididual pool at a rate of 10,000,000 IDs
+// per second, the individual pool will last for 12 years.
+//
 // BATCH REPRESENTATION
 //
 // A batch is represented as a 64-bit integer.  The batch contains all IDs
@@ -60,6 +70,22 @@
 //
 // As a special case, the number 0 is used to indicate "invalid batch".
 //
+// SEQUENCE NUMBERS
+//
+// If you allocate IDs from a player pool rapidly, you will exhaust the fifo in
+// the player pool.  As a result, ID allocation will fall back to the global
+// pool, which will cause prediction failures.
+//
+// Depending on your requirements, you may be able to use sequence numbers
+// instead of IDs.  The player pool contains a monotonically increasing counter
+// for generating sequence numbers.  These can be allocated extremely fast
+// and they'll never run out.  These are highly predictable as long as the
+// only person fetching sequence numbers is the player himself.  The downside
+// is that sequence numbers are not globally unique - they're only locally
+// unique for a single player.  That may be enough for your requirements.
+// 
+// Currently, sequence numbers are used by the random number generator.
+//
 ///////////////////////////////////////////////////////////////////
 
 #ifndef IDALLOC_HPP
@@ -105,6 +131,10 @@ public:
     // zero, or the batch contains fewer than 128 IDs.
     void salvage(int64_t batch);
 
+    // Get a global sequence number.  These are not the same as IDs.
+    // Sequence numbers are unlikely to be predicted correctly.
+    int64_t get_seqno() { return next_seqno_++; }
+
     // Serialize to or deserialize from a streambuffer.
     void serialize(StreamBuffer *sb) const;
     void deserialize(StreamBuffer *sb);
@@ -116,6 +146,7 @@ private:
     eng::vector<int64_t> salvaged_;
     int64_t next_batch_;
     int64_t next_id_;
+    int64_t next_seqno_;
     friend int unittests_idalloc(lua_State *L);
 };
 
@@ -143,6 +174,10 @@ public:
     // Get a single ID from the fifo.  Also refills the fifo.
     int64_t get_one();
 
+    // Get a player sequence number.  This is not the same as an ID: player
+    // sequence numbers are unique per player, but not globally.
+    int64_t get_seqno() { return next_seqno_++; }
+
     // Return the size of the queue.
     int size() { return ranges_.size(); }
 
@@ -177,6 +212,7 @@ public:
 private:
     IdGlobalPool *global_;
     int fifo_capacity_;
+    int64_t next_seqno_;
     eng::deque<int64_t> ranges_;
     friend int lfn_unittests_idalloc(lua_State *L);
 };

luprex/core/cpp/luastack.hpp

@@ -334,6 +334,10 @@ public:
     int result();
 
 public:
+    // This is the largest integer that can be stored in a lua_Number.
+    // In other words, any 53-bit number can be stored.
+    static const int64_t MAXINT = 0x001FFFFFFFFFFFFF;
+
     static lua_State *newstate (lua_Alloc allocf);
 
     lua_State *state() const { return L_; }

luprex/core/cpp/source.cpp

@@ -351,6 +351,7 @@ eng::string SourceDB::rebuild() {
     LS.makeclass(mathclass, "math");
     LS.rawset(mathclass, "pi", M_PI);
     LS.rawset(mathclass, "huge", HUGE_VAL);
+    LS.rawset(mathclass, "maxint", LuaStack::MAXINT);
 
     LS.result();
     return errs;
@@ -734,14 +735,15 @@ LuaDefineBuiltin(math_min, "x, x, x...", "return the smallest argument");
 LuaDefineBuiltin(math_modf, "x", "returns the integral and fractional part of x");
 LuaDefineBuiltin(math_pow, "x, y", "returns x ^ y, equivalent to the operator");
 LuaDefineBuiltin(math_rad, "deg", "convert degrees to radians");
-LuaDefineBuiltin(math_random, "[m [, n]]", "return random [0.0-1.0), or [1-m], or [m-n].");
-LuaDefineBuiltin(math_randomseed, "x", "set x as the seed for random numbers");
 LuaDefineBuiltin(math_sin, "x", "return the sine of x in radians");
 LuaDefineBuiltin(math_sinh, "x", "return the hyperbolic sine of x in radians");
 LuaDefineBuiltin(math_sqrt, "x", "return the square root of x");
 LuaDefineBuiltin(math_tan, "x", "return the tangent of x in radians");
 LuaDefineBuiltin(math_tanh, "x", "return the hyperbolic tangent of x in radians");
 LuaSandboxBuiltin(math_log10, "", "");
+// math.random and math.randomseed are in world-accessor.cpp, because
+// generating random numbers must manipulate global state which is
+// stored in the world model.
 
 LuaDefineBuiltin(assert, "flag [,message]", "assert that flag is true, if not, raise error");
 LuaDefineBuiltin(error, "message", "raise an error");

luprex/core/cpp/util.cpp

@@ -131,6 +131,31 @@ eng::string hash_to_hex(const HashValue &hv) {
     oss << std::hex << std::setw(16) << std::setfill('0') << hv.second;
     return oss.str();    
 }
+static inline uint64_t Rot64(uint64_t x, int k)
+{
+    return (x << k) | (x >> (64 - k));
+}
+
+uint64_t hash_ints(uint64_t a, uint64_t b, uint64_t c, uint64_t d) {
+    uint64_t h0 = c ^ 0xc548cebf3714dbb9;
+    uint64_t h1 = d ^ 0xd23a7edd44383f8d;
+    uint64_t h2 = a ^ 0x7356f92e4b154df7;
+    uint64_t h3 = b ^ 0x55ce09295766838d;
+    
+    h3 ^= h2;  h2 = Rot64(h2,15);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,52);  h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,26);  h1 += h0;
+    h2 ^= h1;  h1 = Rot64(h1,51);  h2 += h1;
+    h3 ^= h2;  h2 = Rot64(h2,28);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,9);   h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,47);  h1 += h0;
+    h2 ^= h1;  h1 = Rot64(h1,54);  h2 += h1;
+    h3 ^= h2;  h2 = Rot64(h2,32);  h3 += h2;
+    h0 ^= h3;  h3 = Rot64(h3,25);  h0 += h3;
+    h1 ^= h0;  h0 = Rot64(h0,63);  h1 += h0;
+
+    return h1;
+}
 
 StringVec split(const eng::string &s, char sep) {
     StringVec result;
@@ -331,7 +356,6 @@ std::string_view sv_read_line(std::string_view &source) {
 }
 
 
-
 double distance_squared(double x1, double y1, double x2, double y2) {
     double dx = x1 - x2;
     double dy = y1 - y2;

luprex/core/cpp/util.hpp

@@ -57,15 +57,19 @@ eng::string id_vector_debug_string(const IdVector &idv);
 // Unions and sorts two ID vectors.
 IdVector sort_union_id_vectors(const IdVector &v1, const IdVector &v2);
 
-// Get a 64-bit hashvalue for a string.
+// Get a 128-bit hashvalue for a string.
 HashValue hash_string(const eng::string &str);
 
-// Get a 64-bit hashvalue for an ID vector.
+// Get a 128-bit hashvalue for an ID vector.
 HashValue hash_id_vector(const IdVector &idv);
 
-// Convert a hash to a hexadecimal string.
+// Convert a 128-bit hash to a hexadecimal string.
 eng::string hash_to_hex(const HashValue &hash);
 
+// Hash four integers together to 64 bits.
+// This is a good hash, but not cryptographically good.
+uint64_t hash_ints(uint64_t n1, uint64_t n2, uint64_t n3, uint64_t n4);
+
 // Split a string into multiple strings
 StringVec split(const eng::string &s, char sep);
 

luprex/core/cpp/world-accessor.cpp

@@ -1,6 +1,8 @@
 
 #include "world.hpp"
 #include "pprint.hpp"
+#include <cmath>
+#include <iostream>
 
 static void tangible_getall(LuaStack &LS0, LuaSlot list, const util::IdVector &idv) {
     LuaVar tangibles, tan;
@@ -311,6 +313,161 @@ LuaDefine(tangible_nopredict, "",
     }
 }
 
+LuaDefine(math_random, "(args...)",
+    "|Generate random numbers."
+    "|"
+    "|What it generates depends on the arguments:"
+    "|"
+    "|   ()          - a float in range [0.0, 1.0)"
+    "|   (high)      - an int between 1 and high inclusive"
+    "|   (low, high) - an int between low and high inclusive"
+    "|"
+    "|math.random tries to cooperate with predictive"
+    "|reexecution to be as predictable as possible."
+    "|To achieve predictability, we used an ad-hoc"
+    "|random number generator.  It passes a variety of"
+    "|statistical tests, but it's not well-studied."
+    "|"
+    "|If you want actually want nonpredictability, or"
+    "|if you need the assurance of a well-studied random"
+    "|number generator, use math.mtrandom or"
+    "|math.cryptrandom instead.") {
+    // Parse the arguments.
+    // This is hairy because there's a lot of possibilities.
+    bool passed_in_randomstate = false;
+    int arg = 1;
+    if ((lua_gettop(L) >= arg) && (lua_istable(L, arg))) {
+        passed_in_randomstate = true;
+        arg += 1;
+    }
+    bool have_range = false;
+    int64_t low, high;
+    if ((lua_gettop(L) >= arg) && (lua_type(L, arg) == LUA_TNUMBER)) {
+        double lowf, highf;
+        if ((lua_gettop(L) >= arg+1) && (lua_type(L, arg+1) == LUA_TNUMBER)) {
+            lowf = std::floor(lua_tonumber(L, arg));
+            highf = std::floor(lua_tonumber(L, arg + 1));
+            arg += 2;
+        } else {
+            lowf = 1;
+            highf = std::floor(lua_tonumber(L, arg));
+            arg += 1;
+        }
+        if ((lowf < -LuaStack::MAXINT) || (highf > LuaStack::MAXINT)) {
+            luaL_error(L, "math.random range exceeds MAXINT");
+            return 0;
+        }
+        if (lowf > highf) {
+            luaL_error(L, "math.random range low > high");
+            return 0;
+        }
+        low = int64_t(lowf);
+        high = int64_t(highf);
+        have_range = true;
+    }
+    if (lua_gettop(L) >= arg) {
+        luaL_error(L, "math.random accepts an optional randomstate and an optional range");
+        return 0;
+    }
+
+    // Generate the seed, count, and salt.
+    // The salt prevents accidental duplication between user-specified
+    // seeds and system-generated seeds.
+    uint64_t seed, count, salt;
+    if (passed_in_randomstate) {
+        lua_pushstring(L, "seed");
+        lua_rawget(L, 1);
+        lua_pushstring(L, "count");
+        lua_rawget(L, 1);
+        if ((lua_type(L, -1) != LUA_TNUMBER) ||
+            (lua_type(L, -2) != LUA_TNUMBER)) {
+            luaL_error(L, "Not a valid randomstate table");
+            return 0;
+        }
+        double dseed = lua_tonumber(L, -2);
+        double dcount = lua_tonumber(L, -1);
+        seed = uint64_t(dseed) & LuaStack::MAXINT;
+        count = uint64_t(dcount) & LuaStack::MAXINT;
+        if (dseed < 0) {
+            salt = 0x35c9a6082a097ade;
+        } else {
+            salt = 0x4785d086ead90c20;
+        }
+        lua_pop(L, 2);
+        lua_pushstring(L, "count");
+        lua_pushnumber(L, double((count + 1) & LuaStack::MAXINT));
+        lua_rawset(L, 1);
+    } else {
+        World *w = World::fetch_global_pointer(L);
+        if (w->lthread_use_ppool_) {
+            Tangible *actor = w->tangible_get(w->lthread_actor_id_);
+            seed = w->lthread_actor_id_;
+            count = actor->id_player_pool_.get_seqno();
+            salt = 0x3ab0fb84aedc3764;
+        } else {
+            // TODO: maybe throw in a 'donotpredict' here.
+            seed = 123456;
+            count = w->id_global_pool_.get_seqno();
+            salt = 0x6f493c90faf0139d;
+        }
+    }
+
+    // Generate the hash and convert to a lua_Number.
+    uint64_t hash = util::hash_ints(seed, count, salt, 456);
+    if (!have_range) {
+        double result = (hash & LuaStack::MAXINT) * 0x1p-53;
+        lua_pushnumber(L, result);
+    } else {
+        uint64_t range = (high - low) + 1;
+        uint64_t offset = (hash & 0x7FFFFFFFFFFFFFFF) % range;
+        int64_t result = low + int64_t(offset);
+        lua_pushnumber(L, result);
+    }
+    return 1;
+}
+
+LuaDefine(math_randomstate, "(seed)",
+    "|Create and return a randomstate table."
+    "|This is a lua table that stores the state for a random"
+    "|number generator.  A randomstate table can be passed"
+    "|to math.random."
+    "|"
+    "|You can optionally omit the seed, in which case it will"
+    "|pick a seed randomly.  Automatically-generated seeds are"
+    "|guaranteed never to be the same as user-specified seeds.") {
+    double seed;
+    if (lua_gettop(L) == 0) {
+        World *w = World::fetch_global_pointer(L);
+        int64_t iseed = (w->id_global_pool_.get_seqno() & LuaStack::MAXINT) + 1;
+        seed = -iseed;
+    } else if (lua_gettop(L) == 1) {
+        if (lua_type(L, 1) != LUA_TNUMBER) {
+            luaL_error(L, "math.randomstate takes an optional integer seed");
+            return 0;
+        }
+        seed = lua_tonumber(L, 1);
+        if ((seed < 0.0) || (seed > LuaStack::MAXINT) || (std::floor(seed) != seed)) {
+            luaL_error(L, "math.randomstate seed must be an integer 0-MAXINT");
+            return 0;
+        }
+    } else {
+        luaL_error(L, "math.randomstate takes an optional integer seed");
+        return 0;
+    }
+
+    lua_newtable(L);
+    lua_pushstring(L, "seed");
+    lua_pushnumber(L, seed);
+    lua_rawset(L, -3);
+    lua_pushstring(L, "count");
+    lua_pushnumber(L, 0);
+    lua_rawset(L, -3);
+    return 1;
+}
+
+LuaSandboxBuiltin(math_randomseed, "", "");
+
+
 LuaDefine(pprint, "obj1,obj2,...",
     "|Pretty-print object or objects.") {
     World *w = World::fetch_global_pointer(L);

luprex/core/cpp/world.hpp

@@ -501,6 +501,8 @@ private:
     friend int lfn_tangible_nopredict(lua_State *L);
     friend int lfn_tangible_near(lua_State *L);
     friend int lfn_tangible_scan(lua_State *L);
+    friend int lfn_math_random(lua_State *L);
+    friend int lfn_math_randomstate(lua_State *L);
 };
 
 using UniqueWorld = std::unique_ptr<World>;

luprex/experiments/bigid.cpp

@@ -0,0 +1,43 @@
+#include <iostream>
+#include <cstdint>
+
+bool storable(int64_t n) {
+    double d = n;
+    int64_t n1 = int64_t(d);
+    return n1 == n;
+}
+
+
+// Find the biggest number where the number is storable,
+// and all numbers smaller are also storable.
+int64_t find_biggest() {
+    int64_t v = 256;
+    int64_t best = 256;
+    while (true) {
+        for (int i = 0; i < 100; i++) {
+            if (!storable(v - i)) {
+                return best;
+            }
+        }
+        best = v;
+        v <<= 1;
+    }
+}
+
+    
+int main(int argc, char **argv) {
+    int64_t best = find_biggest();
+
+    printf("%016lx   ", best);
+    for (int i = -12; i <= 12; i++) {
+        if (i == 0) printf("  ");
+        if (storable(best + i)) {
+            printf("* ");
+        } else {
+            printf("- ");
+        }
+        if (i == 0) printf("  ");
+    }
+    printf("\n");
+    return 0;
+}