Lots more work on eng::malloc

luprex/core/cpp/driver-linux.cpp

@@ -263,7 +263,6 @@ public:
 
 int main(int argc, char **argv)
 {
-    mallopt(M_MMAP_MAX, 0); // Keep malloc in the 'brk' area.
     disable_randomization(argc, argv);
     allocate_buffers();
     enable_tty_raw();

luprex/core/cpp/eng-malloc.cpp

@@ -0,0 +1,122 @@
+
+// We only use a custom allocator on linux (for now)
+// The allocator we chose is 'dlmalloc' (doug lea's malloc).
+// It's a good allocator for single-threaded programs.
+// It needs to be configured by setting a bunch of defines.
+//
+#ifdef __linux__
+
+// We need this to define dlmalloc, not malloc.
+#define USE_DL_PREFIX 1
+
+// We don't need mspaces.
+#define ONLY_MSPACES 0
+#define MSPACES 0
+
+// We don't need mallinfo
+#define NO_MALLINFO 1
+
+// We don't need dlmalloc_inspect_all
+#define MALLOC_INSPECT_ALL 0
+
+// Disable locking.  The entire engine is single-threaded.
+#define USE_LOCKS 0
+
+// This allocator can use sbrk to obtain RAM.  We're going to
+// emulate sbrk, in order to put it in a different memory region
+// than the system malloc, which also uses sbrk.
+#define HAVE_MORECORE 1
+#define MORECORE emulated_sbrk
+#define MORECORE_CANNOT_TRIM 1
+
+// We won't let the memory allocator use mmap.  This is because
+// opening the replay log may be implemented in stdio using mmap.
+// So using mmap might trigger different results under replay.
+#define HAVE_MMAP 0
+
+// Fix a warning in dlmalloc.
+#define MAX_RELEASE_CHECK_RATE INT_MAX
+
+// Don't generate random seeds, or time-based seeds.
+#define USE_DEV_RANDOM 0
+#define LACKS_TIME_H 1
+
+// Don't export any dlmalloc functions.
+#define DLMALLOC_EXPORT static inline
+
+#include <sys/mman.h>
+#include <cstdint>
+#include <cassert>
+#include <climits>
+
+static char     *emulated_sbrk_base;
+static intptr_t  emulated_sbrk_used;
+static intptr_t  emulated_sbrk_limit;
+
+// We assume that the increments are already aligned to the system
+// page size, because dlmalloc does that for us.
+//
+// Our emulated sbrk cannot trim the memory size.  It can only grow.
+//
+static void *emulated_sbrk(intptr_t increment) {
+    if (emulated_sbrk_base == 0) {
+        emulated_sbrk_limit = intptr_t(64) * 1024 * 1024 * 1024;
+        void *map = mmap(nullptr, emulated_sbrk_limit, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        assert(map != MAP_FAILED);
+        assert(map != nullptr);
+        emulated_sbrk_base = (char *)map;
+        emulated_sbrk_used = 0;
+    }
+    int64_t old_used = emulated_sbrk_used;
+    int64_t new_used = emulated_sbrk_used + increment;
+    if (new_used > emulated_sbrk_limit) {
+        return (void *)(-1);
+    }
+    assert(new_used >= old_used);
+    emulated_sbrk_used = new_used;
+    if (new_used > old_used) {
+        int status = mprotect(emulated_sbrk_base + old_used, new_used - old_used, PROT_READ | PROT_WRITE);
+        assert(status == 0);
+    }
+    return emulated_sbrk_base + old_used;
+}
+
+#include "dlmalloc/dlmalloc.c"
+
+namespace eng {
+    static uint32_t hash;
+    void *malloc(size_t size) {
+        void *result = dlmalloc(size);
+        hash += uint32_t(uintptr_t(result)) + 0x83748374;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        hash += uint32_t(size) + 0x85893489;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        return result;
+    }
+    void *realloc(void *p, size_t size) {
+        void *result = dlrealloc(p, size);
+        hash += uint32_t(uintptr_t(p)) + 0x74741912;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        hash += uint32_t(uintptr_t(result)) + 0x68843823;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        hash += uint32_t(size) + 0x81444120;
+        hash += hash << 10;
+        hash ^= hash >> 6;
+        return result;
+    }
+    void  free(void *p) {
+        hash += uint32_t(uintptr_t(p)) + 0x87823448;
+        dlfree(p);
+    }
+    int memhash() {
+        return hash;
+    }
+} // namespace eng
+
+#endif // ifdef __linux__
+
+

luprex/core/cpp/eng-malloc.hpp

@@ -0,0 +1,152 @@
+//
+// eng-malloc
+//
+// The engine has its own private eng::malloc which it uses to allocate
+// everything.  The engine's malloc heap only contains engine data structures.
+// This helps achieve determinism when playing a replay log.
+//
+// The engine's eng::malloc is a thin wrapper around Doug Lea's Malloc, a good
+// general-purpose single-threaded malloc.  It's probably not the fastest any
+// more (it was, once), but it's still quite good.  It's also fairly easy
+// to work with.
+//
+// In order to get all engine data structures into the eng::malloc heap, you
+// need to jump through quite a few hoops:
+//
+// * When using STL classes, you need to use the 'eng' variant of those classes:
+//
+//   - eng::string instead of std::string (include "wrap-string.hpp")
+//   - eng::vector instead of std::vector (include "wrap-vector.hpp")
+//   - eng::map    instead of std::map    (include "wrap-map.hpp")
+//   - etc.
+//
+//   These eng classes allocate memory using eng::malloc instead of malloc.
+//
+// * All your classes must derive from eng::opnew.  This adds a custom operator
+//   new and operator delete to your class, which allocate using eng::malloc.
+//
+// * Use eng::make_shared and eng::make_unique instead of std::make_shared and
+//   std::make_unique.
+//
+// * Simple classes like std::pair, std::string_view, std::less, std::hash, and
+//   so forth don't allocate memory.  Those classes are not wrapped.  There is
+//   no eng::pair, eng::less, etc.
+//
+// * Be aware that most C++ streams use the system malloc heap, and there's no
+//   way to change that.   Fortunately, eng::ostringstream uses the dlmalloc
+//   heap.
+//
+// * Failing to jump through all these hoops won't break your code in any
+//   obvious way - you'll just have some of your data structures in the malloc
+//   heap instead of the eng::malloc heap.  This can break determinism of
+//   replay.
+//
+#ifndef ENG_MALLOC_HPP
+#define ENG_MALLOC_HPP
+
+#include <cstddef>
+#include <memory>
+
+namespace eng {
+#ifdef __linux__
+void* malloc(size_t x);
+void  free(void *p);
+void* realloc(void*, size_t);
+int   memhash();
+#else
+inline void *malloc(size_t x) { return ::malloc(x); }
+inline void  free(void *p) { return ::free(x); }
+inline void *realloc(void *p, size_t x) { return ::realloc(p, x); }
+inline int   memhash() { return 0; }
+#endif
+} // namespace eng
+
+// An allocator for lua states that uses eng::malloc and eng::free
+namespace eng {
+    inline void *l_alloc(void *ud, void *ptr, size_t osize, size_t nsize) {
+        if (nsize == 0) {
+            ::eng::free(ptr);
+            return NULL;
+        } else {
+            return ::eng::realloc(ptr, nsize);
+        }
+    }
+} // namespace eng
+
+// eng_allocator is similar to std::allocator, but allocates
+// objects using eng::malloc and eng::free.
+template <class T>
+class eng_allocator
+{
+public:
+    using value_type    = T;
+    eng_allocator() noexcept {}
+    template <class U> eng_allocator(eng_allocator<U> const&) noexcept {}
+
+    value_type* allocate(std::size_t n)
+    {
+        return static_cast<value_type*>(eng::malloc(n*sizeof(value_type)));
+    }
+
+    void deallocate(value_type* p, std::size_t) noexcept
+    {
+        eng::free(p);
+    }
+};
+
+// Another name for eng_allocator is eng::allocator.
+namespace eng {
+    template<class T>
+    using allocator = ::eng_allocator<T>;
+} // namespace eng
+
+// Mandated equality and inequality operators for eng_allocator.
+template <class T, class U>
+bool operator==(const eng_allocator<T> &, const eng_allocator<U> &) noexcept
+{
+    return true;
+}
+template <class T, class U>
+bool operator!=(const eng_allocator<T> &, const eng_allocator<U> &) noexcept
+{
+    return false;
+}
+
+// eng::opnew.  A class containing operator new and operator delete,
+// meant to be used as a base class for inheritance.
+namespace eng {
+    class opnew {
+    public:
+        void *operator new(size_t size)
+        {
+            return ::eng::malloc(size);
+        }
+
+        void operator delete(void *p, size_t size)
+        {
+            return ::eng::free(p);
+        }
+    };
+} // namespace eng
+
+// eng::make_shared allocates shared objects using eng::malloc.
+namespace eng {
+    template<class T, class... Args>
+    inline ::std::shared_ptr<T> make_shared(Args&&... args) {
+        static eng::allocator<T> alloc;
+        return std::allocate_shared<T>(alloc, args...);
+    }
+}
+
+// eng::make_unique doesn't do anything different than std::make_unique:
+// they both use operator new and delete.  You must
+// derive from eng::opnew to change operator new and delete.
+namespace eng {
+    template<class T, class... Args>
+    inline ::std::unique_ptr<T> make_unique(Args&&... args) {
+        return std::make_unique<T>(args...);
+    }
+} // namespace eng
+
+#endif // ENG_MALLOC_HPP
+

luprex/core/cpp/eng-tests.cpp

@@ -95,7 +95,7 @@ public:
     virtual void event_update() {
         double clock = get_clock();
         if (clock > last_clock_ + 0.5) {
-            int ms = dlmalloc_hash();
+            int ms = eng::memhash();
             stdostream() << std::fixed << std::setprecision(2) << clock << "   " << std::hex << ms << "  ";
             count_++;
             last_clock_ = clock;

luprex/core/cpp/lpxserver.cpp

@@ -9,7 +9,7 @@
 
 #include <memory>
 
-class Client : public eng::heap {
+class Client : public eng::opnew {
 public:
     int64_t actor_id_;
     SharedChannel channel_;

luprex/core/cpp/luaconsole.cpp

@@ -1,4 +1,4 @@
-#include "two-mallocs.hpp"
+#include "eng-malloc.hpp"
 #include "luastack.hpp"
 #include "wrap-string.hpp"
 #include "wrap-vector.hpp"
@@ -10,7 +10,7 @@
 #include <iostream>
 
 LuaConsole::LuaConsole() {
-    lua_state_ = LuaStack::newstate(lalloc_dlmalloc);
+    lua_state_ = LuaStack::newstate(eng::l_alloc);
     clear_raw_input();
 }
 

luprex/core/cpp/luasnap.cpp

@@ -10,7 +10,7 @@
 
 
 LuaSnap::LuaSnap() {
-    state_ = LuaStack::newstate(lalloc_dlmalloc);
+    state_ = LuaStack::newstate(eng::l_alloc);
     LuaStack LS(state_);
 
     // Create the persist table and the unpersist table.

luprex/core/cpp/luastack.cpp

@@ -34,11 +34,21 @@ static int panicf(lua_State *L) {
     exit(1);
 }
 
+// An allocator for lua states that uses system malloc and system free.
+static void *l_alloc(void *ud, void *ptr, size_t osize, size_t nsize) {
+    if (nsize == 0) {
+        free(ptr);
+        return NULL;
+    } else {
+        return realloc(ptr, nsize);
+    }
+}
 
 lua_State *LuaStack::newstate (lua_Alloc allocf) {
-  lua_State *L = lua_newstate(allocf, NULL);
-  if (L) lua_atpanic(L, &panicf);
-  return L;
+    if (allocf == nullptr) allocf = l_alloc;
+    lua_State *L = lua_newstate(allocf, NULL);
+    if (L) lua_atpanic(L, &panicf);
+    return L;
 }
 
 bool LuaStack::ckboolean(LuaSlot s) const {

luprex/core/cpp/printbuffer.cpp

@@ -5,7 +5,7 @@
 #include <algorithm>
 #include <cstdio>
 
-struct PrintBufferCore  : public eng::heap {
+struct PrintBufferCore  : public eng::opnew {
     // The most recent lines printed.
     eng::deque<eng::string> lines_;
 

luprex/core/cpp/printbuffer.hpp

@@ -89,7 +89,7 @@
 
 struct PrintBufferCore;
 
-class PrintBuffer : public eng::heap {
+class PrintBuffer : public eng::opnew {
 private:
     PrintBufferCore *core_;
 
@@ -138,7 +138,7 @@ public:
 };
 
 
-class PrintChanneler : public eng::heap {
+class PrintChanneler : public eng::opnew {
 private:
     int64_t line_;
 public:

luprex/core/cpp/source.cpp

@@ -498,7 +498,7 @@ void SourceDB::deserialize_source(util::LuaSourceVec *sv, StreamBuffer *sb) {
 
 // This function should not touch the dlmalloc heap.
 void SourceDB::register_lua_builtins() {
-    lua_State *L = LuaStack::newstate(lalloc_malloc);
+    lua_State *L = LuaStack::newstate(nullptr);
     luaL_openlibs(L);
     LuaVar globals,classtab,func;
     LuaStack LS(L, globals, classtab, func);

luprex/core/cpp/streambuffer.cpp

@@ -1,6 +1,6 @@
 #include "wrap-string.hpp"
 
-#include "two-mallocs.hpp"
+#include "eng-malloc.hpp"
 #include "streambuffer.hpp"
 #include "spookyv2.hpp"
 
@@ -25,7 +25,7 @@ StreamBuffer::StreamBuffer() {
 
 StreamBuffer::StreamBuffer(int64_t size, bool fixed) {
     assert(size >= 0);
-    init(fixed, true, (char*)dlmalloc(size), size);
+    init(fixed, true, (char*)eng::malloc(size), size);
 }
 
 StreamBuffer::StreamBuffer(const char *s, int64_t size) {
@@ -40,7 +40,7 @@ StreamBuffer::StreamBuffer(const eng::string &src) {
 }
 
 StreamBuffer::~StreamBuffer() {
-    if (owned_ && (buf_lo_ != 0)) dlfree(buf_lo_);
+    if (owned_ && (buf_lo_ != 0)) eng::free(buf_lo_);
 }
 
 int64_t StreamBuffer::total_reads() const {
@@ -87,9 +87,9 @@ void StreamBuffer::make_space_slow(int64_t bytes) {
     } else if (existing_size >= desired_size) {
         if (data_size > 0) memcpy(buf_lo_, read_cursor_, data_size);
     } else {
-        char *nbuf = (char *)dlmalloc(desired_size);
+        char *nbuf = (char *)eng::malloc(desired_size);
         if (data_size > 0) memcpy(nbuf, read_cursor_, data_size);
-        if (buf_lo_ != nullptr) dlfree(buf_lo_);
+        if (buf_lo_ != nullptr) eng::free(buf_lo_);
         buf_lo_ = nbuf;
         buf_hi_ = nbuf + desired_size;
     }
@@ -116,7 +116,7 @@ char *StreamBuffer::get_overwrite(int64_t size, int64_t write_count_after) {
 void StreamBuffer::clear() {
     assert(owned_);
     if (!fixed_size_) {
-        if (buf_lo_ != nullptr) dlfree(buf_lo_);
+        if (buf_lo_ != nullptr) eng::free(buf_lo_);
         buf_lo_ = 0;
         buf_hi_ = 0;
     }

luprex/core/cpp/two-mallocs.cpp

@@ -1,64 +0,0 @@
-
-// We only use a custom allocator on linux (for now)
-// The allocator we chose is 'dlmalloc' (doug lea's malloc).
-// It's a good allocator for single-threaded programs.
-// It needs to be configured by setting a bunch of defines.
-//
-#ifdef __linux__
-
-// We need this to define dlmalloc, not malloc.
-#define USE_DL_PREFIX 1
-
-// We don't need mspaces.
-#define ONLY_MSPACES 0
-#define MSPACES 0
-
-// We don't need mallinfo
-#define NO_MALLINFO 1
-
-// We don't need dlmalloc_inspect_all
-#define MALLOC_INSPECT_ALL 0
-
-// We don't need dlmalloc_stats.
-#define NO_MALLOC_STATS 1
-
-// Disable locking.  The entire engine is single-threaded.
-#define USE_LOCKS 0
-
-// For now, we'll let the allocator use mmap to get memory.
-// It's not clear if this is going to be deterministic.
-#define HAVE_MMAP 1
-
-// One way to force determinism would be to emulate sbrk
-// using mmap and mremap, always putting the heap at a fixed
-// address.  For now, we're not doing that, we're just
-// using mmap and relying on that being deterministic (we hope).
-#define HAVE_MORECORE 0
-#define MORECORE emulate_sbrk
-
-// The properties of mmap under linux
-#define MMAP_CLEARS 1
-#define HAVE_MREMAP 1
-
-// Don't generate random seeds, or time-based seeds.
-#define USE_DEV_RANDOM 0
-#define LACKS_TIME_H 1
-
-#include "dlmalloc/dlmalloc.c"
-
-#endif
-
-int dlmalloc_hash() {
-    void *blocks[15];
-    int hash = 0;
-    for (int i = 0; i < 15; i++) {
-        void *blk = dlmalloc(1 << i);
-        blocks[i] = blk;
-        hash = (hash * 17) + (int)(intptr_t)(blk);
-    }
-    for (int i = 0; i < 15; i++) {
-        dlfree(blocks[i]);
-    }
-    return (hash & 0x7FFFFFFF) | (0x40000000);
-}
-

luprex/core/cpp/two-mallocs.hpp

@@ -1,157 +0,0 @@
-//
-// Two Mallocs:
-//
-// The purpose of this file is to put all engine data structures into 
-// a private heap that isn't used by any other library.  This helps
-// achieve determinism when playing a replay log.
-//
-// For the engine's heap, we chose Doug Lea's Malloc (dlmalloc).  It's
-// a good general-purpose single-threaded malloc.  It's probably not
-// the fastest any more (it was, once), but it's still quite good.  It
-// may be possible to replace dlmalloc with another malloc, but
-// dlmalloc is pretty easy to work with.
-//
-// In order to get all engine data structures into the dlmalloc heap,
-// you need to jump through quite a few hoops:
-//
-// * When using STL classes, you need to use the 'eng' variant of those
-//   classes:
-//
-//    use 'eng::string' instead of std::string (include "wrap-string.hpp")
-//    use 'eng::vector' instead of std::vector (include "wrap-vector.hpp")
-//    use 'eng::map'    instead of std::map    (include "wrap-map.hpp")
-//    etc.
-//
-//   These eng classes allocate memory from dlmalloc instead of malloc.
-//
-// * All your classes must derive from eng::heap.  This adds a custom
-//   operator new and operator delete to your class.
-//
-// * Use eng::make_shared and eng::make_unique instead of std::make_shared
-//   and std::make_unique.
-//
-// * Simple classes like std::pair, std::string_view, std::less,
-//   std::hash, and so forth don't allocate memory.  Those classes
-//   are not wrapped.  There is no eng::pair, eng:less, etc.
-//
-// * Be aware that most C++ streams use the malloc heap, and there's no
-//   way to change that.   Fortunately, eng::ostringstream uses the
-//   dlmalloc heap.
-//
-// * Failing to jump through all these hoops won't break your code in
-//   any obvious way - you'll just have some of your data structures in
-//   the malloc heap instead of the dlmalloc heap.  This can break
-//   determinism of replay.
-//
-#ifndef TWO_MALLOCS_HPP
-#define TWO_MALLOCS_HPP
-
-#include <cstddef>
-#include <memory>
-
-// dlmalloc is only used on linux.
-extern "C" {
-#ifdef __linux__
-void* dlmalloc(size_t x);
-void  dlfree(void *p);
-void* dlrealloc(void*, size_t);
-#else
-void* dlmalloc(size_t x) { return malloc(x); }
-void  dlfree(void *p) { free(p); }
-void* dlrealloc(void *p, size_t x) { return realloc(p,x); }
-#endif
-}
-
-// Return the current state of the dlmalloc allocator as a 30-bit hash.
-extern int dlmalloc_hash();
-
-// An allocator for lua states that uses dlmalloc and dlfree.
-inline void *lalloc_dlmalloc (void *ud, void *ptr, size_t osize, size_t nsize) {
-    if (nsize == 0) {
-        dlfree(ptr);
-        return NULL;
-    } else {
-        return dlrealloc(ptr, nsize);
-    }
-}
-
-// An allocator for lua states that uses malloc and free.
-inline void *lalloc_malloc (void *ud, void *ptr, size_t osize, size_t nsize) {
-    if (nsize == 0) {
-        free(ptr);
-        return NULL;
-    } else {
-        return realloc(ptr, nsize);
-    }
-}
-
-// eng:allocator: a class meant to be used as an STL Allocator.
-// Causes objects to be allocated using dlmalloc and dlfree.
-template <class T>
-class dlmalloc_allocator
-{
-public:
-    using value_type    = T;
-    dlmalloc_allocator() noexcept {}
-    template <class U> dlmalloc_allocator(dlmalloc_allocator<U> const&) noexcept {}
-
-    value_type* allocate(std::size_t n)
-    {
-        return static_cast<value_type*>(dlmalloc(n*sizeof(value_type)));
-    }
-
-    void deallocate(value_type* p, std::size_t) noexcept
-    {
-        dlfree(p);
-    }
-};
-
-template <class T, class U>
-bool operator==(const dlmalloc_allocator<T> &, const dlmalloc_allocator<U> &) noexcept
-{
-    return true;
-}
-template <class T, class U>
-bool operator!=(const dlmalloc_allocator<T> &, const dlmalloc_allocator<U> &) noexcept
-{
-    return false;
-}
-
-// Another name for dlmalloc_allocator
-namespace eng {
-    template <class T>
-    using allocator = dlmalloc_allocator<T>;
-}
-
-// eng::heap a class meant to be used as a base class.
-// Causes 'new' and 'delete' for this class to use dlmalloc and dlfree.
-namespace eng {
-    class heap {
-    public:
-        void *operator new(size_t size)
-        {
-            return dlmalloc(size);
-        }
-
-        void operator delete(void *p, size_t size)
-        {
-            return dlfree(p);
-        }
-    };
-} // namespace
-
-// eng::make_shared allocates shared objects in the dlmalloc heap.
-namespace eng {
-    template<class T, class... Args>
-    inline std::shared_ptr<T> make_shared(Args&&... args) {
-        static eng::allocator<T> alloc;
-        return std::allocate_shared<T>(alloc, args...);
-    }
-    template<class T, class... Args>
-    inline std::unique_ptr<T> make_unique(Args&&... args) {
-        return std::make_unique<T>(args...);
-    }
-} // namespace eng
-
-#endif // TWO_MALLOCS_HPP
-