talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Undo all the work on determinism in the driver. New plan soon.

Browse Source
This commit is contained in:
jyelon
2022-02-21 19:28:54 -05:00
parent ba1e923b5a
commit 19b6951e0f
14 changed files with 105 additions and 1146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
luprex/core/Makefile
View File

@@ -64,7 +64,6 @@ LUA_OBJ_FILES=\
CORE_OBJ_FILES=\
obj/invocation.o\
obj/umm-malloc.o\
obj/spookyv2.o\
obj/debugcollector.o\
obj/drivenengine.o\
@@ -95,7 +94,6 @@ CORE_OBJ_FILES=\
obj/lpxclient.o\
obj/drivertests.o\
obj/printbuffer.o\
obj/main.o \
obj/driver-util.o\
obj/$(DRIVER).o\

7
luprex/core/TODO Normal file
View File

@@ -0,0 +1,7 @@
Improve table.findremove to work on tables, not just vectors.
Finish documenting all builtins.
Get rid of source_install_builtins after documenting all builtins.
- but don't forget that source_install_builtins sets the string metatable.

43
luprex/core/cpp/driver-common.cpp
View File

@@ -1,15 +1,3 @@
#define OPENSSL_HEAP_SIZE (4*1024*1024)
#define CHBUF_SIZE (256*1024)
#define POLLVEC_SIZE (DrivenEngine::MAX_CHAN+1)
int mallocstate(int n) {
int64_t result = 0;
for (int i = 0; i < n; i++) {
int64_t n = int64_t(malloc(1));
result = (result * 17) + n;
}
return result & 0x7fffffff;
}
static MonoClock monoclock;
@@ -19,6 +7,18 @@ namespace util {
}
}
static void initialize_engine() {
SourceDB::register_lua_builtins();
DrivenEngine::register_maker("textgame", make_TextGame);
DrivenEngine::register_maker("lpxclient", make_LpxClient);
DrivenEngine::register_maker("lpxserver", make_LpxServer);
DrivenEngine::register_maker("driverstubtest", make_DriverStubTest);
DrivenEngine::register_maker("driverwebservertest", make_DriverWebServerTest);
DrivenEngine::register_maker("driverdnsfailtest", make_DriverDNSFailTest);
DrivenEngine::register_maker("driverprintclocktest", make_DriverPrintClockTest);
DrivenEngine::register_maker("unittest", make_RunUnitTests);
}
static void if_error_print_and_exit(const UmmString &str) {
if (!str.empty()) {
std::cerr << std::endl << "error: " << str << std::endl;
@@ -70,8 +70,6 @@ static int ssl_ctx_use_privatekey_str(SSL_CTX *ctx, const char *str) {
return status;
}
static std::unique_ptr<char[]> chbuf;
static std::unique_ptr<struct pollfd[]> pollvec;
class Driver {
public:
@@ -503,23 +501,6 @@ public:
void driver_drive(int argc, char *argv[]) {
// The only place in the driver where we're allowed to use malloc
// is here, before even looking at the arguments. That way, the
// impact on the malloc heap is always exactly the same, which
// doesn't break the determinism of the execution during replay.
umm_init_heap(malloc(OPENSSL_HEAP_SIZE), OPENSSL_HEAP_SIZE);
CRYPTO_set_mem_functions(umm_malloc_ssl, umm_realloc_ssl, umm_free_ssl);
chbuf.reset(new char[CHBUF_SIZE]);
pollvec.reset(new struct pollfd[POLLVEC_SIZE]);
ERR_load_crypto_strings();
SSL_load_error_strings();
std::cerr << "#2 " << std::hex << mallocstate(1) << std::endl;
Driver driver;
if (argc < 2) {
DrivenEngine::print_usage(std::cerr, argv[0]);

32
luprex/core/cpp/driver-linux.cpp
View File

@@ -1,10 +1,14 @@
#include "driver.hpp"
#include "umm-malloc.hpp"
#include "driver-util.hpp"
#include "util.hpp"
#include "drivenengine.hpp"
#include "dummycert.hpp"
#include "util.hpp"
#include "textgame.hpp"
#include "lpxclient.hpp"
#include "lpxserver.hpp"
#include "drivertests.hpp"
#include "source.hpp"
#include <map>
#include <vector>
#include <iostream>
@@ -30,11 +34,18 @@
#include <openssl/err.h>
#include <openssl/bio.h>
#include <openssl/pem.h>
#include <openssl/conf.h>
#define OPENSSL_HEAP_SIZE (4*1024*1024)
#define CHBUF_SIZE (256*1024)
#define POLLVEC_SIZE (DrivenEngine::MAX_CHAN+1)
using SOCKET=int;
const int INVALID_SOCKET = -1;
struct termios orig_termios;
static std::unique_ptr<char[]> chbuf;
static std::unique_ptr<struct pollfd[]> pollvec;
static UmmString strerror_str(int err) {
char errbuf[256];
@@ -255,10 +266,6 @@ static void disable_randomization(int argc, char *argv[]) {
}
}
void driver_sysinit(int argc, char *argv[]) {
disable_randomization(argc, argv);
enable_tty_raw();
}
class MonoClock {
private:
@@ -279,3 +286,16 @@ public:
};
#include "driver-common.cpp"
int main(int argc, char **argv)
{
disable_randomization(argc, argv);
enable_tty_raw();
chbuf.reset(new char[CHBUF_SIZE]);
pollvec.reset(new struct pollfd[POLLVEC_SIZE]);
OPENSSL_init_ssl(0, NULL);
initialize_engine();
driver_drive(argc, argv);
}

28
luprex/core/cpp/driver-mingw.cpp
View File

@@ -2,11 +2,15 @@
#define _WIN32_WINNT 0x0600
#include "driver.hpp"
#include "umm-malloc.hpp"
#include "driver-util.hpp"
#include "util.hpp"
#include "drivenengine.hpp"
#include "dummycert.hpp"
#include "util.hpp"
#include "textgame.hpp"
#include "lpxclient.hpp"
#include "lpxserver.hpp"
#include "drivertests.hpp"
#include "source.hpp"
#include <map>
#include <iostream>
#include <cstdio>
@@ -25,6 +29,12 @@
#include <openssl/bio.h>
#include <openssl/pem.h>
#define CHBUF_SIZE (256*1024)
#define POLLVEC_SIZE (DrivenEngine::MAX_CHAN+1)
static std::unique_ptr<char[]> chbuf;
static std::unique_ptr<struct pollfd[]> pollvec;
static void set_nonblocking(SOCKET sock) {
u_long mode = 1; // 1 to enable non-blocking socket
int status = ioctlsocket(sock, FIONBIO, &mode);
@@ -235,7 +245,6 @@ static int console_read(char *bytes, int nbytes) {
}
void driver_sysinit(int argc, char *argv[]) {
init_winsock();
}
class MonoClock {
@@ -257,4 +266,15 @@ public:
}
};
#include "driver-common.cpp"
#include "driver-common.cpp"
int main(int argc, char **argv)
{
init_winsock();
chbuf.reset(new char[CHBUF_SIZE]);
pollvec.reset(new struct pollfd[POLLVEC_SIZE]);
OPENSSL_init_ssl(0, NULL);
initialize_engine();
driver_drive(argc, argv);
}

10
luprex/core/cpp/driver-util.hpp
View File

@@ -2,7 +2,15 @@
#ifndef DRIVER_UTIL_HPP
#define DRIVER_UTIL_HPP
#include "umm-malloc.hpp"
#include <string>
#include <vector>
#include <map>
using UmmString = std::string;
template <typename T>
using UmmVector = std::vector<T>;
template <typename K, typename V, class C = std::less<K>>
using UmmMap = std::map<K, V, C>;
using UmmStringVec = UmmVector<UmmString>;

21
luprex/core/cpp/drivertests.cpp
View File

@@ -19,6 +19,13 @@ static void dump_lines(StreamBuffer *in, StreamBuffer *out, int chid) {
}
}
// This test is the minimal possible DrivenEngine.
class DriverStubTest : public DrivenEngine {
virtual void event_init(int argc, char *argv[]) {
stop_driver();
}
};
// This test connects to a public webserver and prints
// the output from the server.
class DriverWebServerTest : public DrivenEngine {
@@ -72,15 +79,6 @@ public:
}
};
static int64_t mallocstate() {
int64_t result = 0;
for (int i = 0; i < 10; i++) {
int64_t n = int64_t(malloc(1));
result = (result * 17) + n;
}
return result;
}
// This test just prints the time.
class DriverPrintClockTest : public DrivenEngine {
public:
@@ -94,7 +92,7 @@ public:
virtual void event_update() {
double clock = get_clock();
if (clock > last_clock_ + 0.5) {
int64_t ms = mallocstate();
int ms = util::hash_of_mallocs();
stdostream() << std::fixed << std::setprecision(2) << clock << " " << std::hex << ms << " ";
count_++;
last_clock_ = clock;
@@ -120,6 +118,9 @@ private:
}
};
UniqueDrivenEngine make_DriverStubTest() {
return UniqueDrivenEngine(new DriverStubTest);
}
UniqueDrivenEngine make_DriverWebServerTest() {
return UniqueDrivenEngine(new DriverWebServerTest);

1
luprex/core/cpp/drivertests.hpp
View File

@@ -3,6 +3,7 @@
#include "drivenengine.hpp"
UniqueDrivenEngine make_DriverStubTest();
UniqueDrivenEngine make_DriverListenTest();
UniqueDrivenEngine make_DriverWebServerTest();
UniqueDrivenEngine make_DriverDNSFailTest();

26
luprex/core/cpp/main.cpp
View File

@@ -1,26 +0,0 @@
#include "textgame.hpp"
#include "lpxclient.hpp"
#include "lpxserver.hpp"
#include "drivertests.hpp"
#include "driver.hpp"
#include "source.hpp"
#include <iostream>
#include <time.h>
int main(int argc, char **argv)
{
driver_sysinit(argc, argv);
SourceDB::register_lua_builtins();
DrivenEngine::register_maker("textgame", make_TextGame);
DrivenEngine::register_maker("lpxclient", make_LpxClient);
DrivenEngine::register_maker("lpxserver", make_LpxServer);
DrivenEngine::register_maker("driverwebservertest", make_DriverWebServerTest);
DrivenEngine::register_maker("driverdnsfailtest", make_DriverDNSFailTest);
DrivenEngine::register_maker("driverprintclocktest", make_DriverPrintClockTest);
DrivenEngine::register_maker("unittest", make_RunUnitTests);
driver_drive(argc, argv);
}

2
luprex/core/cpp/source.cpp
View File

@@ -401,7 +401,7 @@ static std::string source_load_lfunctions(lua_State *L) {
std::string SourceDB::rebuild() {
lua_State *L = lua_state_;
source_clear_globals(L);
source_install_builtins(L);
// source_install_builtins(L);
source_load_cfunctions(L);
std::string errs = source_load_lfunctions(L);
return errs;

929
luprex/core/cpp/umm-malloc.cpp
View File

@@ -1,929 +0,0 @@
/* ----------------------------------------------------------------------------
* umm_malloc.c - a memory allocator for embedded systems (microcontrollers)
*
* The MIT License (MIT)
*
* Copyright (c) 2015 Ralph Hempel
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use, copy,
* modify, merge, publish, distribute, sublicense, and/or sell copies
* of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* ----------------------------------------------------------------------------
*
* Note: this version is significantly modified from the version
* distributed by Ralph Hempel. In particular, block numbers are
* 32 bits in this version.
* ----------------------------------------------------------------------------
*/
#include <stdio.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <stdbool.h>
#include <math.h>
#include "umm-malloc.hpp"
/*
* --------------------------------------------------------------------------
* UMM_BEST_FIT (default)
*
* Set this if you want to use a best-fit algorithm for allocating new blocks.
* On by default, turned off by UMM_FIRST_FIT
*
* UMM_FIRST_FIT
*
* Set this if you want to use a first-fit algorithm for allocating new blocks.
* Faster than UMM_BEST_FIT but can result in higher fragmentation.
*
* ----------------------------------------------------------------------------
*/
#define UMM_BEST_FIT
#undef UMM_FIRST_FIT
/*
* --------------------------------------------------------------------------
*
* Three macros to make it easier to protect the memory allocator in a
* multitasking system. You should set these macros up to use whatever your
* system uses for this purpose. You can disable interrupts entirely, or just
* disable task switching - it's up to you
*
* NOTE WELL that these macros MUST be allowed to nest, because umm_free() is
* called from within umm_malloc()
*
* --------------------------------------------------------------------------
*/
#define UMM_CRITICAL_DECL(tag)
#define UMM_CRITICAL_ENTRY(tag)
#define UMM_CRITICAL_EXIT(tag)
/*
* --------------------------------------------------------------------------
*
* Debug Logging.
*
* --------------------------------------------------------------------------
*/
// #define DBGLOG(format, ...) { fprintf(stderr, format,##__VA_ARGS__); fflush(stderr); }
#define DBGLOG(format, ...)
/* ------------------------------------------------------------------------- */
typedef struct umm_ptr_t {
uint32_t next;
uint32_t prev;
} umm_ptr;
typedef struct umm_block_t {
union {
umm_ptr used;
} header;
union {
umm_ptr free;
uint8_t data[8];
} body;
} umm_block;
#define UMM_FREELIST_MASK ((uint32_t)(0x80000000))
#define UMM_BLOCKNO_MASK ((uint32_t)(0x7FFFFFFF))
/* ------------------------------------------------------------------------- */
struct umm_heap_config {
umm_block *pheap;
size_t heap_size;
uint32_t numblocks;
};
static struct umm_heap_config umm_heap_current;
#define UMM_HEAP (umm_heap_current.pheap)
#define UMM_HEAPSIZE (umm_heap_current.heap_size)
#define UMM_NUMBLOCKS (umm_heap_current.numblocks)
#define UMM_BLOCKSIZE (sizeof(umm_block))
#define UMM_BLOCK_LAST (UMM_NUMBLOCKS - 1)
/* -------------------------------------------------------------------------
* These macros evaluate to the address of the block and data respectively
*/
#define UMM_BLOCK(b) (UMM_HEAP[b])
#define UMM_DATA(b) (UMM_BLOCK(b).body.data)
/* -------------------------------------------------------------------------
* These macros evaluate to the index of the block - NOT the address!!!
*/
#define UMM_NBLOCK(b) (UMM_BLOCK(b).header.used.next)
#define UMM_PBLOCK(b) (UMM_BLOCK(b).header.used.prev)
#define UMM_NFREE(b) (UMM_BLOCK(b).body.free.next)
#define UMM_PFREE(b) (UMM_BLOCK(b).body.free.prev)
/* ----------------------------------------------------------------------------
* One of the coolest things about this little library is that it's VERY
* easy to get debug information about the memory heap by simply iterating
* through all of the memory blocks.
*
* As you go through all the blocks, you can check to see if it's a free
* block by looking at the high order bit of the next block index. You can
* also see how big the block is by subtracting the next block index from
* the current block number.
*
* The umm_info function does all of that and makes the results available
* in the ummHeapInfo structure.
* ----------------------------------------------------------------------------
*/
typedef struct UMM_HEAP_INFO_t {
unsigned int totalEntries;
unsigned int usedEntries;
unsigned int freeEntries;
unsigned int totalBlocks;
unsigned int usedBlocks;
unsigned int freeBlocks;
unsigned int freeBlocksSquared;
unsigned int maxFreeContiguousBlocks;
int usage_metric;
int fragmentation_metric;
}
UMM_HEAP_INFO;
static UMM_HEAP_INFO ummHeapInfo;
static void compute_usage_metric(void)
{
if (0 == ummHeapInfo.freeBlocks) {
ummHeapInfo.usage_metric = -1; // No free blocks!
} else {
ummHeapInfo.usage_metric = (int)((ummHeapInfo.usedBlocks * 100) / (ummHeapInfo.freeBlocks));
}
}
static void compute_fragmentation_metric(void)
{
if (0 == ummHeapInfo.freeBlocks) {
ummHeapInfo.fragmentation_metric = 0; // No free blocks ... so no fragmentation either!
} else {
ummHeapInfo.fragmentation_metric = 100 - (((uint32_t)(sqrtf(ummHeapInfo.freeBlocksSquared)) * 100) / (ummHeapInfo.freeBlocks));
}
}
#define DBGLOG_32_BIT_PTR(x) ((uint32_t)(((uintptr_t)(x)) & 0xffffffff))
void umm_info() {
uint32_t blockNo = 0;
UMM_CRITICAL_DECL(id_info);
/* Protect the critical section... */
UMM_CRITICAL_ENTRY(id_info);
/*
* Clear out all of the entries in the ummHeapInfo structure before doing
* any calculations..
*/
memset(&ummHeapInfo, 0, sizeof(ummHeapInfo));
DBGLOG("\n");
DBGLOG("+----------+-------+--------+--------+-------+--------+--------+\n");
DBGLOG("|0x%08x|B %5i|NB %5i|PB %5i|Z %5i|NF %5i|PF %5i|\n",
DBGLOG_32_BIT_PTR(&UMM_BLOCK(blockNo)),
blockNo,
UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK,
UMM_PBLOCK(blockNo),
(UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK) - blockNo,
UMM_NFREE(blockNo),
UMM_PFREE(blockNo));
/*
* Now loop through the block lists, and keep track of the number and size
* of used and free blocks. The terminating condition is an nb pointer with
* a value of zero...
*/
blockNo = UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK;
while (UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK) {
size_t curBlocks = (UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK) - blockNo;
++ummHeapInfo.totalEntries;
ummHeapInfo.totalBlocks += curBlocks;
/* Is this a free block? */
if (UMM_NBLOCK(blockNo) & UMM_FREELIST_MASK) {
++ummHeapInfo.freeEntries;
ummHeapInfo.freeBlocks += curBlocks;
ummHeapInfo.freeBlocksSquared += (curBlocks * curBlocks);
if (ummHeapInfo.maxFreeContiguousBlocks < curBlocks) {
ummHeapInfo.maxFreeContiguousBlocks = curBlocks;
}
DBGLOG("|0x%08x|B %5i|NB %5i|PB %5i|Z %5u|NF %5i|PF %5i|\n",
DBGLOG_32_BIT_PTR(&UMM_BLOCK(blockNo)),
blockNo,
UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK,
UMM_PBLOCK(blockNo),
(uint32_t)curBlocks,
UMM_NFREE(blockNo),
UMM_PFREE(blockNo));
} else {
++ummHeapInfo.usedEntries;
ummHeapInfo.usedBlocks += curBlocks;
DBGLOG("|0x%08x|B %5i|NB %5i|PB %5i|Z %5u| |\n",
DBGLOG_32_BIT_PTR(&UMM_BLOCK(blockNo)),
blockNo,
UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK,
UMM_PBLOCK(blockNo),
(uint32_t)curBlocks);
}
blockNo = UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK;
}
/*
* The very last block is used as a placeholder to indicate that
* there are no more blocks in the heap, so it cannot be used
* for anything - at the same time, the size of this block must
* ALWAYS be exactly 1 !
*/
DBGLOG("|0x%08x|B %5i|NB %5i|PB %5i|Z %5i|NF %5i|PF %5i|\n",
DBGLOG_32_BIT_PTR(&UMM_BLOCK(blockNo)),
blockNo,
UMM_NBLOCK(blockNo) & UMM_BLOCKNO_MASK,
UMM_PBLOCK(blockNo),
UMM_NUMBLOCKS - blockNo,
UMM_NFREE(blockNo),
UMM_PFREE(blockNo));
DBGLOG("+----------+-------+--------+--------+-------+--------+--------+\n");
DBGLOG("Total Entries %5i Used Entries %5i Free Entries %5i\n",
ummHeapInfo.totalEntries,
ummHeapInfo.usedEntries,
ummHeapInfo.freeEntries);
DBGLOG("Total Blocks %5i Used Blocks %5i Free Blocks %5i\n",
ummHeapInfo.totalBlocks,
ummHeapInfo.usedBlocks,
ummHeapInfo.freeBlocks);
DBGLOG("+--------------------------------------------------------------+\n");
compute_usage_metric();
DBGLOG("Usage Metric: %5i\n", ummHeapInfo.usage_metric);
compute_fragmentation_metric();
DBGLOG("Fragmentation Metric: %5i\n", ummHeapInfo.fragmentation_metric);
DBGLOG("+--------------------------------------------------------------+\n");
/* Release the critical section... */
UMM_CRITICAL_EXIT(id_info);
}
/* ------------------------------------------------------------------------ */
static uint32_t umm_blocks(size_t size) {
/*
* The calculation of the block size is not too difficult, but there are
* a few little things that we need to be mindful of.
*
* When a block removed from the free list, the space used by the free
* pointers is available for data. That's what the first calculation
* of size is doing.
*
* We don't check for the special case of (size == 0) here as this needs
* special handling in the caller depending on context. For example when we
* realloc() a block to size 0 it should simply be freed.
*
* We do NOT need to check for allocating more blocks than the heap can
* possibly hold - the allocator figures this out for us.
*
* There are only two cases left to consider:
*
* 1. (size <= body) Obviously this is just one block
* 2. (blocks > (2^15)) This should return ((2^15)) to force a
* failure when the allocator runs
*
* If the requested size is greater that 32677-2 blocks (max block index
* minus the overhead of the top and bottom bookkeeping blocks) then we
* will return an incorrectly truncated value when the result is cast to
* a uint32_t.
*/
if (size <= (sizeof(((umm_block *)0)->body))) {
return 1;
}
/*
* If it's for more than that, then we need to figure out the number of
* additional whole blocks the size of an umm_block are required, so
* reduce the size request by the number of bytes in the body of the
* first block.
*/
size -= (sizeof(((umm_block *)0)->body));
/* NOTE WELL that we take advantage of the fact that INT16_MAX is the
* number of blocks that we can index in 15 bits :-)
*
* The below expression looks wierd, but it's right. Assuming body
* size of 4 bytes and a block size of 8 bytes:
*
* BYTES (BYTES-BODY) (BYTES-BODY-1)/BLOCKSIZE BLOCKS
* 1 n/a n/a 1
* 5 1 0 2
* 12 8 0 2
* 13 9 1 3
*/
size_t blocks = (2 + ((size-1) / (UMM_BLOCKSIZE)));
if (blocks > (INT16_MAX)) {
blocks = INT16_MAX;
}
return (uint32_t)blocks;
}
/* ------------------------------------------------------------------------ */
/*
* Split the block `c` into two blocks: `c` and `c + blocks`.
*
* - `new_freemask` should be `0` if `c + blocks` used, or `UMM_FREELIST_MASK`
* otherwise.
*
* Note that free pointers are NOT modified by this function.
*/
static void umm_split_block(uint32_t c,
uint32_t blocks,
uint32_t new_freemask) {
UMM_NBLOCK(c + blocks) = (UMM_NBLOCK(c) & UMM_BLOCKNO_MASK) | new_freemask;
UMM_PBLOCK(c + blocks) = c;
UMM_PBLOCK(UMM_NBLOCK(c) & UMM_BLOCKNO_MASK) = (c + blocks);
UMM_NBLOCK(c) = (c + blocks);
}
/* ------------------------------------------------------------------------ */
static void umm_disconnect_from_free_list(uint32_t c) {
/* Disconnect this block from the FREE list */
UMM_NFREE(UMM_PFREE(c)) = UMM_NFREE(c);
UMM_PFREE(UMM_NFREE(c)) = UMM_PFREE(c);
/* And clear the free block indicator */
UMM_NBLOCK(c) &= (~UMM_FREELIST_MASK);
}
/* ------------------------------------------------------------------------
* The umm_assimilate_up() function does not assume that UMM_NBLOCK(c)
* has the UMM_FREELIST_MASK bit set. It only assimilates up if the
* next block is free.
*/
static void umm_assimilate_up(uint32_t c) {
if (UMM_NBLOCK(UMM_NBLOCK(c)) & UMM_FREELIST_MASK) {
/*
* The next block is a free block, so assimilate up and remove it from
* the free list
*/
DBGLOG("Assimilate up to next block, which is FREE\n");
/* Disconnect the next block from the FREE list */
umm_disconnect_from_free_list(UMM_NBLOCK(c));
/* Assimilate the next block with this one */
UMM_PBLOCK(UMM_NBLOCK(UMM_NBLOCK(c)) & UMM_BLOCKNO_MASK) = c;
UMM_NBLOCK(c) = UMM_NBLOCK(UMM_NBLOCK(c)) & UMM_BLOCKNO_MASK;
}
}
/* ------------------------------------------------------------------------
* The umm_assimilate_down() function assumes that UMM_NBLOCK(c) does NOT
* have the UMM_FREELIST_MASK bit set. In other words, try to assimilate
* up before assimilating down.
*/
static uint32_t umm_assimilate_down(uint32_t c, uint32_t freemask) {
// We are going to assimilate down to the previous block because
// it was free, so remove it from the fragmentation metric
UMM_NBLOCK(UMM_PBLOCK(c)) = UMM_NBLOCK(c) | freemask;
UMM_PBLOCK(UMM_NBLOCK(c)) = UMM_PBLOCK(c);
return UMM_PBLOCK(c);
}
/* ------------------------------------------------------------------------- */
void umm_init_heap(void *ptr, size_t size)
{
/* init heap pointer and size, and memset it to 0 */
UMM_HEAP = (umm_block *)ptr;
UMM_HEAPSIZE = size;
UMM_NUMBLOCKS = (UMM_HEAPSIZE / UMM_BLOCKSIZE);
memset(UMM_HEAP, 0x00, UMM_HEAPSIZE);
/* Set up umm_block[0], which just points to umm_block[1] */
UMM_NBLOCK(0) = 1;
UMM_NFREE(0) = 1;
UMM_PFREE(0) = 1;
/*
* Now, we need to set the whole heap space as a huge free block. We should
* not touch umm_block[0], since it's special: umm_block[0] is the head of
* the free block list. It's a part of the heap invariant.
*
* See the detailed explanation at the beginning of the file.
*
* umm_block[1] has pointers:
*
* - next `umm_block`: the last one umm_block[n]
* - prev `umm_block`: umm_block[0]
*
* Plus, it's a free `umm_block`, so we need to apply `UMM_FREELIST_MASK`
*
* And it's the last free block, so the next free block is 0 which marks
* the end of the list. The previous block and free block pointer are 0
* too, there is no need to initialize these values due to the init code
* that memsets the entire umm_ space to 0.
*/
UMM_NBLOCK(1) = UMM_BLOCK_LAST | UMM_FREELIST_MASK;
/*
* Last umm_block[n] has the next block index at 0, meaning it's
* the end of the list, and the previous block is umm_block[1].
*
* The last block is a special block and can never be part of the
* free list, so its pointers are left at 0 too.
*/
UMM_PBLOCK(UMM_BLOCK_LAST) = 1;
// DBGLOG(true, "nblock(0) %04x pblock(0) %04x nfree(0) %04x pfree(0) %04x\n", UMM_NBLOCK(0) & UMM_BLOCKNO_MASK, UMM_PBLOCK(0), UMM_NFREE(0), UMM_PFREE(0));
// DBGLOG(true, "nblock(1) %04x pblock(1) %04x nfree(1) %04x pfree(1) %04x\n", UMM_NBLOCK(1) & UMM_BLOCKNO_MASK, UMM_PBLOCK(1), UMM_NFREE(1), UMM_PFREE(1));
}
/* ------------------------------------------------------------------------
* Must be called only from within critical sections guarded by
* UMM_CRITICAL_ENTRY(id) and UMM_CRITICAL_EXIT(id).
*/
static void umm_free_core(void *ptr) {
uint32_t c;
/*
* FIXME: At some point it might be a good idea to add a check to make sure
* that the pointer we're being asked to free up is actually within
* the umm_heap!
*
*/
/* Figure out which block we're in. Note the use of truncated division... */
c = (((uint8_t *)ptr) - (uint8_t *)(&(UMM_HEAP[0]))) / UMM_BLOCKSIZE;
DBGLOG("Freeing block %6i\n", c);
/* Now let's assimilate this block with the next one if possible. */
umm_assimilate_up(c);
/* Then assimilate with the previous block if possible */
if (UMM_NBLOCK(UMM_PBLOCK(c)) & UMM_FREELIST_MASK) {
DBGLOG("Assimilate down to previous block, which is FREE\n");
c = umm_assimilate_down(c, UMM_FREELIST_MASK);
} else {
/*
* The previous block is not a free block, so add this one to the head
* of the free list
*/
DBGLOG("Just add to head of free list\n");
UMM_PFREE(UMM_NFREE(0)) = c;
UMM_NFREE(c) = UMM_NFREE(0);
UMM_PFREE(c) = 0;
UMM_NFREE(0) = c;
UMM_NBLOCK(c) |= UMM_FREELIST_MASK;
}
}
/* ------------------------------------------------------------------------ */
void umm_free(void *ptr) {
UMM_CRITICAL_DECL(id_free);
/* If we're being asked to free a NULL pointer, well that's just silly! */
if ((void *)0 == ptr) {
DBGLOG("free a null pointer -> do nothing\n");
return;
}
/* Free the memory withing a protected critical section */
UMM_CRITICAL_ENTRY(id_free);
umm_free_core(ptr);
UMM_CRITICAL_EXIT(id_free);
}
/* ------------------------------------------------------------------------
* Must be called only from within critical sections guarded by
* UMM_CRITICAL_ENTRY(id) and UMM_CRITICAL_EXIT(id).
*/
static void *umm_malloc_core(size_t size) {
uint32_t blocks;
uint32_t blockSize = 0;
uint32_t bestSize;
uint32_t bestBlock;
uint32_t cf;
blocks = umm_blocks(size);
/*
* Now we can scan through the free list until we find a space that's big
* enough to hold the number of blocks we need.
*
* This part may be customized to be a best-fit, worst-fit, or first-fit
* algorithm
*/
cf = UMM_NFREE(0);
bestBlock = UMM_NFREE(0);
bestSize = 0x7FFFFFFF;
while (cf) {
blockSize = (UMM_NBLOCK(cf) & UMM_BLOCKNO_MASK) - cf;
DBGLOG("Looking at block %6i size %6i\n", cf, blockSize);
#if defined UMM_BEST_FIT
if ((blockSize >= blocks) && (blockSize < bestSize)) {
bestBlock = cf;
bestSize = blockSize;
}
#elif defined UMM_FIRST_FIT
/* This is the first block that fits! */
if ((blockSize >= blocks)) {
break;
}
#else
#error "No UMM_*_FIT is defined - check umm_malloc_cfg.h"
#endif
cf = UMM_NFREE(cf);
}
if (0x7FFFFFFF != bestSize) {
cf = bestBlock;
blockSize = bestSize;
}
if (UMM_NBLOCK(cf) & UMM_BLOCKNO_MASK && blockSize >= blocks) {
/*
* This is an existing block in the memory heap, we just need to split off
* what we need, unlink it from the free list and mark it as in use, and
* link the rest of the block back into the freelist as if it was a new
* block on the free list...
*/
if (blockSize == blocks) {
/* It's an exact fit and we don't neet to split off a block. */
DBGLOG("Allocating %6i blocks starting at %6i - exact\n", blocks, cf);
/* Disconnect this block from the FREE list */
umm_disconnect_from_free_list(cf);
} else {
/* It's not an exact fit and we need to split off a block. */
DBGLOG("Allocating %6i blocks starting at %6i - existing\n", blocks, cf);
/*
* split current free block `cf` into two blocks. The first one will be
* returned to user, so it's not free, and the second one will be free.
*/
umm_split_block(cf, blocks, UMM_FREELIST_MASK /*new block is free*/);
/*
* `umm_split_block()` does not update the free pointers (it affects
* only free flags), but effectively we've just moved beginning of the
* free block from `cf` to `cf + blocks`. So we have to adjust pointers
* to and from adjacent free blocks.
*/
/* previous free block */
UMM_NFREE(UMM_PFREE(cf)) = cf + blocks;
UMM_PFREE(cf + blocks) = UMM_PFREE(cf);
/* next free block */
UMM_PFREE(UMM_NFREE(cf)) = cf + blocks;
UMM_NFREE(cf + blocks) = UMM_NFREE(cf);
}
} else {
/* Out of memory */
DBGLOG("Can't allocate %5i blocks\n", blocks);
return (void *)NULL;
}
return (void *)&UMM_DATA(cf);
}
/* ------------------------------------------------------------------------ */
void *umm_malloc(size_t size) {
UMM_CRITICAL_DECL(id_malloc);
void *ptr = NULL;
/*
* the very first thing we do is figure out if we're being asked to allocate
* a size of 0 - and if we are we'll simply return a null pointer. if not
* then reduce the size by 1 byte so that the subsequent calculations on
* the number of blocks to allocate are easier...
*/
if (0 == size) {
DBGLOG("malloc a block of 0 bytes -> do nothing\n");
return ptr;
}
/* Allocate the memory withing a protected critical section */
UMM_CRITICAL_ENTRY(id_malloc);
ptr = umm_malloc_core(size);
UMM_CRITICAL_EXIT(id_malloc);
return ptr;
}
/* ------------------------------------------------------------------------ */
void *umm_realloc(void *ptr, size_t size) {
UMM_CRITICAL_DECL(id_realloc);
uint32_t blocks;
uint32_t blockSize;
uint32_t prevBlockSize = 0;
uint32_t nextBlockSize = 0;
uint32_t c;
size_t curSize;
/*
* This code looks after the case of a NULL value for ptr. The ANSI C
* standard says that if ptr is NULL and size is non-zero, then we've
* got to work the same a malloc(). If size is also 0, then our version
* of malloc() returns a NULL pointer, which is OK as far as the ANSI C
* standard is concerned.
*/
if (((void *)NULL == ptr)) {
DBGLOG("realloc the NULL pointer - call malloc()\n");
return umm_malloc(size);
}
/*
* Now we're sure that we have a non_NULL ptr, but we're not sure what
* we should do with it. If the size is 0, then the ANSI C standard says that
* we should operate the same as free.
*/
if (0 == size) {
DBGLOG("realloc to 0 size, just free the block\n");
umm_free(ptr);
return (void *)NULL;
}
/*
* Otherwise we need to actually do a reallocation. A naiive approach
* would be to malloc() a new block of the correct size, copy the old data
* to the new block, and then free the old block.
*
* While this will work, we end up doing a lot of possibly unnecessary
* copying. So first, let's figure out how many blocks we'll need.
*/
blocks = umm_blocks(size);
/* Figure out which block we're in. Note the use of truncated division... */
c = (((uint8_t *)ptr) - (uint8_t *)(&(UMM_HEAP[0]))) / UMM_BLOCKSIZE;
/* Figure out how big this block is ... the free bit is not set :-) */
blockSize = (UMM_NBLOCK(c) - c);
/* Figure out how many bytes are in this block */
curSize = (blockSize * UMM_BLOCKSIZE) - (sizeof(((umm_block *)0)->header));
/* Protect the critical section... */
UMM_CRITICAL_ENTRY(id_realloc);
/* Now figure out if the previous and/or next blocks are free as well as
* their sizes - this will help us to minimize special code later when we
* decide if it's possible to use the adjacent blocks.
*
* We set prevBlockSize and nextBlockSize to non-zero values ONLY if they
* are free!
*/
if ((UMM_NBLOCK(UMM_NBLOCK(c)) & UMM_FREELIST_MASK)) {
nextBlockSize = (UMM_NBLOCK(UMM_NBLOCK(c)) & UMM_BLOCKNO_MASK) - UMM_NBLOCK(c);
}
if ((UMM_NBLOCK(UMM_PBLOCK(c)) & UMM_FREELIST_MASK)) {
prevBlockSize = (c - UMM_PBLOCK(c));
}
DBGLOG("realloc blocks %i blockSize %i nextBlockSize %i prevBlockSize %i\n", blocks, blockSize, nextBlockSize, prevBlockSize);
/*
* Ok, now that we're here we know how many blocks we want and the current
* blockSize. The prevBlockSize and nextBlockSize are set and we can figure
* out the best strategy for the new allocation as follows:
*
* 1. If the new block is the same size or smaller than the current block do
* nothing.
* 2. If the next block is free and adding it to the current block gives us
* EXACTLY enough memory, assimilate the next block. This avoids unwanted
* fragmentation of free memory.
*
* The following cases may be better handled with memory copies to reduce
* fragmentation
*
* 3. If the previous block is NOT free and the next block is free and
* adding it to the current block gives us enough memory, assimilate
* the next block. This may introduce a bit of fragmentation.
* 4. If the prev block is free and adding it to the current block gives us
* enough memory, remove the previous block from the free list, assimilate
* it, copy to the new block.
* 5. If the prev and next blocks are free and adding them to the current
* block gives us enough memory, assimilate the next block, remove the
* previous block from the free list, assimilate it, copy to the new block.
* 6. Otherwise try to allocate an entirely new block of memory. If the
* allocation works free the old block and return the new pointer. If
* the allocation fails, return NULL and leave the old block intact.
*
* TODO: Add some conditional code to optimise for less fragmentation
* by simply allocating new memory if we need to copy anyways.
*
* All that's left to do is decide if the fit was exact or not. If the fit
* was not exact, then split the memory block so that we use only the requested
* number of blocks and add what's left to the free list.
*/
// Case 1 - block is same size or smaller
if (blockSize >= blocks) {
DBGLOG("realloc the same or smaller size block - %i, do nothing\n", blocks);
/* This space intentionally left blank */
// Case 2 - block + next block fits EXACTLY
} else if ((blockSize + nextBlockSize) == blocks) {
DBGLOG("exact realloc using next block - %i\n", blocks);
umm_assimilate_up(c);
blockSize += nextBlockSize;
// Case 3 - prev block NOT free and block + next block fits
} else if ((0 == prevBlockSize) && (blockSize + nextBlockSize) >= blocks) {
DBGLOG("realloc using next block - %i\n", blocks);
umm_assimilate_up(c);
blockSize += nextBlockSize;
// Case 4 - prev block + block fits
} else if ((prevBlockSize + blockSize) >= blocks) {
DBGLOG("realloc using prev block - %i\n", blocks);
umm_disconnect_from_free_list(UMM_PBLOCK(c));
c = umm_assimilate_down(c, 0);
memmove((void *)&UMM_DATA(c), ptr, curSize);
ptr = (void *)&UMM_DATA(c);
blockSize += prevBlockSize;
// Case 5 - prev block + block + next block fits
} else if ((prevBlockSize + blockSize + nextBlockSize) >= blocks) {
DBGLOG("realloc using prev and next block - %i\n", blocks);
umm_assimilate_up(c);
umm_disconnect_from_free_list(UMM_PBLOCK(c));
c = umm_assimilate_down(c, 0);
memmove((void *)&UMM_DATA(c), ptr, curSize);
ptr = (void *)&UMM_DATA(c);
blockSize += (prevBlockSize + nextBlockSize);
// Case 6 - default is we need to realloc a new block
} else {
DBGLOG("realloc a completely new block %i\n", blocks);
void *oldptr = ptr;
if ((ptr = umm_malloc_core(size))) {
DBGLOG("realloc %i to a bigger block %i, copy, and free the old\n", blockSize, blocks);
memcpy(ptr, oldptr, curSize);
umm_free_core(oldptr);
} else {
DBGLOG("realloc %i to a bigger block %i failed - return NULL and leave the old block!\n", blockSize, blocks);
/* This space intentionally left blnk */
}
blockSize = blocks;
}
/* Now all we need to do is figure out if the block fit exactly or if we
* need to split and free ...
*/
if (blockSize > blocks) {
DBGLOG("split and free %i blocks from %i\n", blocks, blockSize);
umm_split_block(c, blocks, 0);
umm_free_core((void *)&UMM_DATA(c + blocks));
}
/* Release the critical section... */
UMM_CRITICAL_EXIT(id_realloc);
return ptr;
}
// The openssl library expects a version of malloc that takes
// extra parameters 'file' and 'line'. We provide these for openssl.
// The extra parameters are ignored.
void *umm_malloc_ssl(size_t size, const char *file, int line) {
return umm_malloc(size);
}
void *umm_realloc_ssl(void *ptr, size_t size, const char *file, int line) {
return umm_realloc(ptr, size);
}
void umm_free_ssl(void *ptr, const char *file, int line) {
umm_free(ptr);
}

108
luprex/core/cpp/umm-malloc.hpp
View File

@@ -1,108 +0,0 @@
////////////////////////////////////////////////////////////////////
//
// To achieve determinism of replay logs, the driver is not allowed
// to touch the malloc heap. We've given the driver its own
// separate heap. The driver's heap is accessed using umm_malloc,
// umm_free, and umm_realloc.
//
// To initialize the umm heap, you allocate a single large
// block of RAM from the OS, perhaps 4M. You pass that block
// to umm_heap_init. The umm_malloc routine will allocate out of
// that block. The umm heap cannot grow, if the block fills up, umm_malloc
// will fail. It is fine to get the initial heap block from the
// regular system malloc! That doesn't compromise determinism,
// since you're always allocating the same 4M block.
//
// The UMM malloc library is distributed under the MIT license
// by Ralph Hempel. This is a slightly-modified version, you can
// find the original online. Mr. Hempel considers this to be a malloc
// for microcontrollers, but it works just fine on workstations.
// Of all the mallocs to choose from, I picked this one because it
// satisfies three main criteria: it works out of a fixed block
// of RAM, it's not terribly inefficient, and it's not too complicated
// to modify.
//
// Warning! Don't forget that using the C++ STL tends to use malloc
// all over the place. For example, if you create a std::string,
// you're using malloc! Don't use normal STL classes in the driver!
// Fortunately, most STL classes allow you to specify a custom
// allocator class. The following classes use the UMM allocator:
//
// UmmString - same as std::string.
// UmmVector<T> - same as std::vector<T>.
// UmmMap<K,V> - same as std::map<K,V>
// UmmSet<T> - same as std::set<T>
//
// Sadly, routines that accept std::string, std::vector, std::map,
// or std::set do not accept their Umm equivalents. It is possible
// to write a routine that accepts both std::string and UmmString
// by accepting a std::string_view parameter. Other than that,
// these types are separate.
//
////////////////////////////////////////////////////////////////////
#ifndef UMM_MALLOC_HPP
#define UMM_MALLOC_HPP
#include <stdint.h>
#include <stddef.h>
#include <string>
#include <vector>
#include <map>
#include <set>
#include <utility>
void umm_init_heap(void *ptr, size_t size);
void *umm_malloc(size_t size);
void *umm_realloc(void *ptr, size_t size);
void umm_free(void *ptr);
void umm_info();
void *umm_malloc_ssl(size_t size, const char *file, int line);
void *umm_realloc_ssl(void *ptr, size_t size, const char *file, int line);
void umm_free_ssl(void *ptr, const char *file, int line);
template <class T>
class UmmAllocator
{
public:
using value_type = T;
UmmAllocator() noexcept {}
template <class U> UmmAllocator(UmmAllocator<U> const&) noexcept {}
value_type* allocate(std::size_t n)
{
return static_cast<value_type*>(umm_malloc(n*sizeof(value_type)));
}
void deallocate(value_type* p, std::size_t) noexcept
{
umm_free(p);
}
};
template <class T, class U>
bool operator==(UmmAllocator<T> const&, UmmAllocator<U> const&) noexcept
{
return true;
}
template <class T, class U>
bool operator!=(UmmAllocator<T> const&, UmmAllocator<U> const&) noexcept
{
return false;
}
using UmmString = std::basic_string<char, std::char_traits<char>, UmmAllocator<char>>;
template <class T>
using UmmVector = std::vector<T, UmmAllocator<T>>;
template <class K, class T, class C=std::less<K>>
using UmmMap = std::map<K, T, C, UmmAllocator<std::pair<const K, T>>>;
template <class K, class C=std::less<K>>
using UmmSet = std::set<K, C, UmmAllocator<K>>;
#endif // UMM_MALLOC_HPP

36
luprex/core/cpp/util.cpp
View File

@@ -353,34 +353,18 @@ bool is_lua_comment(const std::string &s) {
return s.substr(start, 2) == "--";
}
static std::string get_file_contents(const std::string &fn) {
std::ifstream fs(fn);
std::stringstream buffer;
buffer << fs.rdbuf();
return buffer.str();
}
static StringVec read_control_lst(const std::string &path) {
StringVec lines = split(get_file_contents(path), '\n');
util::StringVec result;
for (int i = 0; i < int(lines.size()); i++) {
std::string trimmed = trim(lines[i]);
if ((trimmed.size() > 0) && (trimmed[0] != '#')) {
result.push_back(trimmed);
}
int hash_of_mallocs() {
void *blocks[15];
int hash = 0;
for (int i = 0; i < 15; i++) {
void *blk = malloc(1 << i);
blocks[i] = blk;
hash = (hash * 17) + (int)(ptrdiff_t)(blk);
}
return result;
}
LuaSourcePtr read_lua_source(const std::string &dir) {
StringVec files = read_control_lst(dir + "/control.lst");
assert (!files.empty());
LuaSourcePtr result(new LuaSourceVec);
for (const std::string &file : files) {
std::string data = get_file_contents(dir + "/" + file);
result->emplace_back(file, data);
for (int i = 0; i < 15; i++) {
free(blocks[i]);
}
return result;
return (hash & 0x7FFFFFFF) | (0x40000000);
}
std::string XYZ::debug_string() const {

6
luprex/core/cpp/util.hpp
View File

@@ -125,8 +125,10 @@ LuaSourcePtr make_lua_source(const std::string &code);
// Return true if the line of code is a lua comment.
bool is_lua_comment(const std::string &line);
// This has to go away.
LuaSourcePtr read_lua_source(const std::string &dir);
// Malloc some blocks of RAM, and calculate a hash from
// their addresses. This can be used to determine if the malloc
// heap is in a deterministic state.
int hash_of_mallocs();
// Remove nullptrs from a vector of unique pointers.
template<class T>