From 596e39add898298a6d31cf5538e807aabc64209a Mon Sep 17 00:00:00 2001 From: jyelon Date: Fri, 24 Feb 2023 17:47:13 -0500 Subject: [PATCH 1/7] Overhauled SSL to use explicit BIO buffers --- luprex/TODO | 2 +- luprex/cpp/core/drivenengine.cpp | 5 +- luprex/cpp/core/drivenengine.hpp | 2 +- luprex/cpp/drv/driver-common.cpp | 398 ++++++++++++++++++++----------- luprex/cpp/drv/driver-linux.cpp | 28 ++- luprex/cpp/drv/driver-mingw.cpp | 23 +- 6 files changed, 293 insertions(+), 165 deletions(-) diff --git a/luprex/TODO b/luprex/TODO index d8f0f169..d9181688 100644 --- a/luprex/TODO +++ b/luprex/TODO @@ -8,4 +8,4 @@ Do something about std::cerr && std::cout once and for all. Fix math.random (?) - +Do a better job handling 'close' in the driver (need some equivalent of SSL_shutdown) \ No newline at end of file diff --git a/luprex/cpp/core/drivenengine.cpp b/luprex/cpp/core/drivenengine.cpp index 623061b6..10d57634 100644 --- a/luprex/cpp/core/drivenengine.cpp +++ b/luprex/cpp/core/drivenengine.cpp @@ -505,12 +505,13 @@ void DrivenEngine::drv_sent_outgoing(uint32_t chid, uint32_t nbytes) { } void DrivenEngine::drv_recv_incoming(uint32_t chid, uint32_t nbytes, const char *bytes) { + std::string_view sbytes(bytes, nbytes); if (nbytes > 0) { Channel *ch = get_chid(chid); if (ch->sb_drvout_ != ch->sb_out_) { - ch->feed_readline(bytes); + ch->feed_readline(sbytes); } else { - ch->sb_in_->write_bytes(bytes); + ch->sb_in_->write_bytes(sbytes); } } } diff --git a/luprex/cpp/core/drivenengine.hpp b/luprex/cpp/core/drivenengine.hpp index a24ec9c6..42f2cc8b 100644 --- a/luprex/cpp/core/drivenengine.hpp +++ b/luprex/cpp/core/drivenengine.hpp @@ -77,7 +77,7 @@ public: // const eng::string &target() const { return target_; } - // True if the remote closed the connection, or a failure occurred. + // True if the remote has closed the connection. // bool closed() const { return closed_; } diff --git a/luprex/cpp/drv/driver-common.cpp b/luprex/cpp/drv/driver-common.cpp index 669ff5cb..2228ca9f 100644 --- a/luprex/cpp/drv/driver-common.cpp +++ b/luprex/cpp/drv/driver-common.cpp @@ -1,4 +1,5 @@ #define POLLVEC_SIZE (DRV_MAX_CHAN + 1) +#define MAX_BIO_BUFFER (128 * 1024) static void if_error_print_and_exit(const std::string_view str) { @@ -21,15 +22,26 @@ class Driver { int chid; SOCKET socket; SSL *ssl; + BIO *recv_bio; + BIO *send_bio; + + // If recent_error is set, that means that a recent IO operation generated + // an error. As a special case, EOF on read is considered an error, we use + // the string "EOF" for this case. + std::string recent_error; + + // OpenSSL has a rule: if you try to SSL_write and it returns + // SSL_ERROR_WANT_READ, then you have to retry the write with the same + // number of bytes. In this event, we record how many bytes we + // attempted to write, which will enable us to retry. + int retry_write_nbytes; + + // True if the channel needs to be advanced. + bool need_advance; ChanState state; uint32_t nbytes; const char *bytes; - bool ready_now; - bool ready_on_pollin; - bool ready_on_pollout; - bool ready_on_outgoing; - uint32_t last_write_nbytes; bool marked_for_deletion() const { return state == CHAN_INACTIVE; } }; @@ -45,6 +57,82 @@ class Driver { sslutil::UniqueCTX ssl_client_secure_ctx_; sslutil::UniqueCTX ssl_client_insecure_ctx_; + // Return the amount of 'space left' in a BIO. This is a fiction, + // because MEM BIOs technically have unlimited capacity. We're + // artificially limiting them to a certain size because there's no + // reason to buffer huge amounts of data. + // + int bio_space(BIO *bio) { + int space = (MAX_BIO_BUFFER) - BIO_pending(bio); + if (space < 0) space = 0; + return space; + } + + // This is a terribly inefficient way to discard data that has + // already been processed. There has to be something better. + // + void bio_discard(BIO *b, int nbytes) { + while (nbytes > 0) { + int nread = nbytes; + if (nread > DRV_SHORTSTRING_SIZE) nread = DRV_SHORTSTRING_SIZE; + int ndropped = BIO_read(b, chbuf_.get(), nread); + assert(ndropped == nread); + nbytes -= ndropped; + } + } + + void make_channel(SOCKET sock, int chid, SSL_CTX *ctx, ChanState state) { + ChanInfo newchan; + newchan.chid = chid; + newchan.socket = sock; + newchan.recv_bio = BIO_new(BIO_s_mem()); + newchan.send_bio = BIO_new(BIO_s_mem()); + newchan.recent_error.clear(); + newchan.retry_write_nbytes = 0; + newchan.need_advance = true; + + if (state == CHAN_PLAINTEXT) { + newchan.ssl = nullptr; + } else { + newchan.ssl = SSL_new(ctx); + SSL_set_bio(newchan.ssl, newchan.recv_bio, newchan.send_bio); + } + + newchan.state = state; + newchan.nbytes = 0; + newchan.bytes = 0; + chans_.push_back(newchan); + } + + void close_channel(ChanInfo &chan, std::string_view err) { + // std::cerr << "Closing channel " << chan.chid << " with " << err << std::endl; + assert(chan.state != CHAN_INACTIVE); + + // Close and release the SSL channel. + // This frees the BIO objects as well. + if (chan.ssl != nullptr) { + SSL_free(chan.ssl); + chan.ssl = nullptr; + } + chan.recv_bio = nullptr; + chan.send_bio = nullptr; + chan.recent_error.clear(); + chan.retry_write_nbytes = 0; + chan.need_advance = false; + + // Close and release the socket. + assert(chan.socket != INVALID_SOCKET); + assert(socket_close(chan.socket) == 0); + chan.socket = INVALID_SOCKET; + + // Close everything else. + engw.play_notify_close(&engw, chan.chid, err.size(), err.data()); + chan.state = CHAN_INACTIVE; + chan.chid = -1; + chan.nbytes = 0; + chan.bytes = 0; + } + void handle_listen_ports() { uint32_t nports; const uint32_t *ports; engw.get_listen_ports(&engw, &nports, &ports); @@ -69,30 +157,6 @@ class Driver { } } - void close_channel(ChanInfo &chan, std::string_view err) { - // std::cerr << "Closing channel " << chan.chid << std::endl; - assert(chan.state != CHAN_INACTIVE); - // Close and release the SSL channel. - if (chan.ssl != nullptr) { - SSL_free(chan.ssl); - chan.ssl = nullptr; - } - // Close and release the socket. - assert(chan.socket != INVALID_SOCKET); - assert(socket_close(chan.socket) == 0); - chan.socket = INVALID_SOCKET; - // Close everything else. - engw.play_notify_close(&engw, chan.chid, err.size(), err.data()); - chan.state = CHAN_INACTIVE; - chan.chid = -1; - chan.nbytes = 0; - chan.bytes = 0; - chan.ready_now = false; - chan.ready_on_pollin = false; - chan.ready_on_pollout = false; - chan.ready_on_outgoing = false; - chan.last_write_nbytes = 0; - } void handle_console_output() { while (true) { @@ -117,25 +181,6 @@ class Driver { } } - void make_channel(SOCKET sock, int chid, SSL_CTX *ctx, ChanState state) { - ChanInfo newchan; - newchan.chid = chid; - newchan.socket = sock; - newchan.ssl = SSL_new(ctx); - newchan.state = state; - newchan.nbytes = 0; - newchan.bytes = 0; - newchan.ready_now = false; - newchan.ready_on_pollin = false; - newchan.ready_on_pollout = true; - newchan.ready_on_outgoing = false; - newchan.last_write_nbytes = 0; - SSL_set_fd(newchan.ssl, newchan.socket); - // SSL_set_msg_callback(newchan.ssl, SSL_trace); - // SSL_set_msg_callback_arg(newchan.ssl, BIO_new_fp(stderr,0)); - chans_.push_back(newchan); - } - void handle_new_outgoing_sockets() { uint32_t nchids; const uint32_t *chids; engw.get_new_outgoing(&engw, &nchids, &chids); @@ -166,7 +211,6 @@ class Driver { engw.play_notify_close(&engw, chid, err.size(), err.c_str()); continue; } - // std::cerr << "Opening channel " << chid << std::endl; make_channel(sock, chid, ctx, CHAN_SSL_CONNECTING); } engw.play_clear_new_outgoing(&engw); @@ -178,123 +222,188 @@ class Driver { if_error_print_and_exit(err); if (socket != INVALID_SOCKET) { uint32_t chid = engw.play_notify_accept(&engw, port); - // std::cerr << "Accepted channel " << chid << std::endl; make_channel(socket, chid, ssl_server_ctx_.get(), CHAN_SSL_ACCEPTING); } } - void advance_plaintext(ChanInfo &chan) { - std::string err; + // Copy data from the socket into the recv bio. + // + // If it detects an error or EOF, sets the recent_errno flag. + // + void transfer_socket_to_recv_bio(ChanInfo &chan) { + if ((chan.state == CHAN_INACTIVE) || (!chan.recent_error.empty())) { + return; + } - // Try to write plaintext to the channel. - uint32_t ndata; const char *data; - engw.get_outgoing(&engw, chan.chid, &ndata, &data); - if (ndata > 0) { - int sbytes = ndata; - if (sbytes > DRV_SHORTSTRING_SIZE) sbytes = DRV_SHORTSTRING_SIZE; - int wbytes = socket_send(chan.socket, data, sbytes, err); - if (wbytes < 0) { - close_channel(chan, err.c_str()); + std::string err; + int nread = socket_recv(chan.socket, chbuf_.get(), DRV_SHORTSTRING_SIZE, err); + // std::cerr << "chan " << chan.chid << " recv " << nread << " err=" << err << std::endl; + if (nread < 0) { + chan.recent_error = err; + } else { + if (nread == 0) { + chan.recent_error = "EOF"; } else { - engw.play_sent_outgoing(&engw, chan.chid, wbytes); + int nstored = BIO_write(chan.recv_bio, chbuf_.get(), nread); + assert(nstored == nread); + chan.need_advance = true; + // std::cerr << "chan " << chan.chid << " stored " << nread << " bytes" << std::endl; } } - - // Try to read plaintext from the channel. - // Someday, find a way to avoid this copy. - int nrecv = socket_recv(chan.socket, chbuf_.get(), DRV_SHORTSTRING_SIZE, err); - if (nrecv < 0) { - close_channel(chan, err.c_str()); - } else { - engw.play_recv_incoming(&engw, chan.chid, nrecv, chbuf_.get()); - } - - // Update the ready-flags for next time. - chan.ready_on_outgoing = true; - chan.ready_on_pollin = true; } - void process_ssl_error(ChanInfo &chan, int retval) { - int error = SSL_get_error(chan.ssl, retval); - // std::cerr << "SSL error code = " << error << " "; - if (error == SSL_ERROR_WANT_READ) { - chan.ready_on_pollin = true; - } else if (error == SSL_ERROR_WANT_WRITE) { - chan.ready_on_pollout = true; + // Copy data from the send BIO into the socket. + // + // If it detects an error, sets the recent_errno flag. + // + void transfer_send_bio_to_socket(ChanInfo &chan) { + if ((chan.state == CHAN_INACTIVE) || (!chan.recent_error.empty())) { + return; + } + + char *data; + int ndata = BIO_get_mem_data(chan.send_bio, &data); + if (ndata > DRV_SHORTSTRING_SIZE) ndata = DRV_SHORTSTRING_SIZE; + std::string err; + int nwrote = socket_send(chan.socket, data, ndata, err); + // std::cerr << "chan " << chan.chid << " send " << nwrote << " err=" << err << std::endl; + if (nwrote < 0) { + chan.recent_error = err; } else { - std::string error = sslutil::error_string(); - if (error == "") error = "unknown error"; - close_channel(chan, error); + assert(nwrote != 0); + bio_discard(chan.send_bio, nwrote); + chan.need_advance = true; + } + } + + // Close the channel if there's a serious OpenSSL error. + // + // The 'retval' is the return value of the SSL function that returned an + // error. + // + // All errors are considered serious except for SSL_ERROR_WANT_READ, which + // is not serious because it is transient. However, if you get an + // SSL_ERROR_WANT_READ when there's tons of data available in the read + // buffer, that's inexplicable and therefore serious. + // + void if_error_is_serious_close_channel(ChanInfo &chan, int retval) { + int error = SSL_get_error(chan.ssl, retval); + //std::cerr << "chan " << chan.chid << " ssl error = " << error << std::endl; + + // Should never have write errors, because we're + // using a memory BIO with unlimited capacity. + assert(error != SSL_ERROR_WANT_WRITE); + + // If we get a read error, make sure it's plausible: + // if the recv bio is full, that makes no sense. + if (error == SSL_ERROR_WANT_READ) { + if (bio_space(chan.recv_bio) == 0) { + close_channel(chan, "ssl waiting for data, but there's tons of data"); + } + return; + } + + // Any other error is an actual error. Close + // the channel. + std::string errstr = sslutil::error_string(); + if (errstr == "") errstr = "unknown error"; + close_channel(chan, errstr); + } + + void advance_plaintext(ChanInfo &chan) { + uint32_t ndata; const char *data; + + // Transfer all data from the recv BIO into the channel. + ndata = BIO_get_mem_data(chan.recv_bio, &data); + if (ndata > 0) { + engw.play_recv_incoming(&engw, chan.chid, ndata, data); + bio_discard(chan.recv_bio, ndata); + } + + // Transfer all data from the channel to the send BIO. + engw.get_outgoing(&engw, chan.chid, &ndata, &data); + if (ndata > 0) { + int nwrote = BIO_write(chan.send_bio, data, ndata); + assert(nwrote == int(ndata)); + engw.play_sent_outgoing(&engw, chan.chid, ndata); } } void advance_ssl_connecting(ChanInfo &chan) { - // std::cerr << "In advance_ssl_connecting" << std::endl; int retval = SSL_connect(chan.ssl); + //std::cerr << "chan " << chan.chid << " ssl_connect returns " << retval << std::endl; if (retval == 1) { - // Connection successful. chan.state = CHAN_SSL_READWRITE; - chan.ready_now = true; + chan.need_advance = true; } else { - // std::cerr << "ssl_connect_error"; - process_ssl_error(chan, retval); + if_error_is_serious_close_channel(chan, retval); } } void advance_ssl_accepting(ChanInfo &chan) { - // std::cerr << "In advance_ssl_accepting" << std::endl; int retval = SSL_accept(chan.ssl); + //std::cerr << "chan " << chan.chid << " ssl_accept returns " << retval << std::endl; if (retval == 1) { - // Connection successful. chan.state = CHAN_SSL_READWRITE; - chan.ready_now = true; + chan.need_advance = true; } else { - process_ssl_error(chan, retval); + if_error_is_serious_close_channel(chan, retval); } } void advance_ssl_readwrite(ChanInfo &chan) { - // std::cerr << "In advance_ssl_readwrite" << std::endl; - // Try to read data. - int read_result = SSL_read(chan.ssl, chbuf_.get(), DRV_SHORTSTRING_SIZE); - if (read_result > 0) { - engw.play_recv_incoming(&engw, chan.chid, read_result, chbuf_.get()); - chan.ready_now = true; - } else { - process_ssl_error(chan, read_result); - if (chan.state == CHAN_INACTIVE) return; + // Read as much as we can, which of course will be limited + // by the fact that the recv_bio contains finite data. + while (true) { + int read_result = SSL_read(chan.ssl, chbuf_.get(), DRV_SHORTSTRING_SIZE); + if (read_result > 0) { + engw.play_recv_incoming(&engw, chan.chid, read_result, chbuf_.get()); + } else { + if_error_is_serious_close_channel(chan, read_result); + break; + } + } + + // The read process could have generated an error which could + // have closed the channel. If so, don't try writing. + if (chan.state == CHAN_INACTIVE) { + return; } // Try to write data. - uint32_t wbytes; - if (chan.last_write_nbytes > 0) { - wbytes = chan.last_write_nbytes; - assert(wbytes < chan.nbytes); - } else { - wbytes = chan.nbytes; - if (wbytes > 65536) wbytes = 65536; - } - if (wbytes > 0) { + while (chan.nbytes) { + uint32_t wbytes; + if (chan.retry_write_nbytes > 0) { + wbytes = chan.retry_write_nbytes; + assert(wbytes < chan.nbytes); + } else { + wbytes = chan.nbytes; + if (wbytes > DRV_SHORTSTRING_SIZE) wbytes = DRV_SHORTSTRING_SIZE; + } + if (wbytes == 0) break; int write_result = SSL_write(chan.ssl, chan.bytes, wbytes); if (write_result > 0) { engw.play_sent_outgoing(&engw, chan.chid, write_result); - chan.last_write_nbytes = 0; - chan.ready_on_outgoing = true; + chan.retry_write_nbytes = 0; + chan.nbytes -= write_result; + chan.bytes += write_result; } else { - chan.last_write_nbytes = wbytes; - process_ssl_error(chan, write_result); - if (chan.state == CHAN_INACTIVE) return; + if_error_is_serious_close_channel(chan, write_result); + chan.retry_write_nbytes = wbytes; + break; } - } else { - chan.ready_on_outgoing = true; } - // std::cerr << "rpi=" << chan.ready_on_pollin << ".rpo=" << - // chan.ready_on_pollout << ".rn=" << chan.ready_now << ".rog=" << - // chan.ready_on_outgoing << " "; } void advance_channel(ChanInfo &chan) { sslutil::clear_all_errors(); + + // We set the need_advance flag to false here, but + // the rest of the advance routine is allowed to set + // it back to true in the event that the advance routine + // only processes some of the available data. + chan.need_advance = false; + switch (chan.state) { case CHAN_PLAINTEXT: advance_plaintext(chan); @@ -349,13 +458,17 @@ class Driver { pfd.fd = chan.socket; pfd.events = 0; pfd.revents = 0; - if (chan.ready_now) mstimeout = 0; - if (chan.ready_on_pollin) pfd.events |= POLLIN; - if (chan.ready_on_pollout) pfd.events |= POLLOUT; - if (chan.ready_on_outgoing && (chan.nbytes > 0)) + // If there's room in the receive buffer, set POLLIN + if (bio_space(chan.recv_bio) > 0) { + pfd.events |= POLLIN; + } + // If there's data in the outgoing buffer, set POLLOUT + if (BIO_pending(chan.send_bio) > 0) { pfd.events |= POLLOUT; - // std::cerr << "evt=" << pfd.events << ".nb=" << chan.nbytes << - // std::endl; + } + if (chan.need_advance) { + mstimeout = 0; + } } // Do the poll. @@ -370,23 +483,26 @@ class Driver { accept_connection(p.first, p.second); } } - // Advance channels where possible. for (ChanInfo &chan : chans_) { struct pollfd &pfd = pollvec_[index++]; - bool pollin = ((pfd.revents & POLLIN) != 0); - bool pollout = ((pfd.revents & POLLOUT) != 0); - bool pollerr = ((pfd.revents & (POLLERR | POLLHUP)) != 0); - if (chan.ready_now || pollerr || - (chan.ready_on_pollin && pollin) || - (chan.ready_on_pollout && pollout) || - (chan.ready_on_outgoing && (chan.nbytes > 0) && pollout)) { - chan.ready_now = false; - chan.ready_on_pollin = false; - chan.ready_on_pollout = false; - chan.ready_on_outgoing = false; + if ((pfd.revents & POLLIN) != 0) { + transfer_socket_to_recv_bio(chan); + } + if ((pfd.revents & POLLOUT) != 0) { + transfer_send_bio_to_socket(chan); + } + if (chan.need_advance || (!chan.recent_error.empty())) { advance_channel(chan); } + if (!chan.recent_error.empty()) { + if (chan.recent_error == "EOF") { + close_channel(chan, ""); + } else { + close_channel(chan, chan.recent_error); + } + chan.recent_error.clear(); + } chan.nbytes = 0; chan.bytes = 0; } @@ -486,10 +602,10 @@ class Driver { } // Cleanup - engw.release(&engw); for (ChanInfo &chan : chans_) { close_channel(chan, ""); } + engw.release(&engw); return 0; } }; diff --git a/luprex/cpp/drv/driver-linux.cpp b/luprex/cpp/drv/driver-linux.cpp index 55663724..589e3862 100644 --- a/luprex/cpp/drv/driver-linux.cpp +++ b/luprex/cpp/drv/driver-linux.cpp @@ -149,40 +149,42 @@ static SOCKET accept_on_socket(SOCKET listen_socket, std::string &err) { } } -// the return values for socket_send and socket_recv are: +// the return values for socket_send: // -// positive: sent or received bytes successfully -// zero: would block -// negative: channel closed, possibly cleanly or possibly with error +// positive: sent bytes successfully +// negative: error. +// If the error message is empty, then it's "would block" +// Any other error generates an error message. // static int socket_send(SOCKET socket, const char *bytes, int nbytes, std::string &err) { - err.clear(); int wbytes = send(socket, bytes, nbytes, 0); if (wbytes < 0) { if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { - return 0; + err.clear(); } else { err = drvutil::strerror_str(errno); - return -1; } + return -1; } else { + err.clear(); return wbytes; } } static int socket_recv(SOCKET socket, char *bytes, int nbytes, std::string &err) { - err.clear(); int nrecv = recv(socket, bytes, nbytes, 0); if (nrecv < 0) { - if ((errno == EWOULDBLOCK) || (errno == EAGAIN)) { - err = drvutil::strerror_str(errno); - return -1; + if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { + err.clear(); } else { - return 0; + err = drvutil::strerror_str(errno); } - } else if (nrecv == 0) { return -1; + } else if (nrecv == 0) { + err.clear(); + return 0; } else { + err.clear(); return nrecv; } } diff --git a/luprex/cpp/drv/driver-mingw.cpp b/luprex/cpp/drv/driver-mingw.cpp index a1e4318e..706a404c 100644 --- a/luprex/cpp/drv/driver-mingw.cpp +++ b/luprex/cpp/drv/driver-mingw.cpp @@ -152,37 +152,46 @@ static SOCKET accept_on_socket(SOCKET listen_socket, std::string &err) { } } +// the return values for socket_send: +// +// positive: sent bytes successfully +// negative: error. +// If the error message is empty, then it's "would block" +// Any other error generates an error message. +// + static int socket_send(SOCKET socket, const char *bytes, int nbytes, std::string &err) { - err.clear(); int wbytes = send(socket, bytes, nbytes, 0); if (wbytes == SOCKET_ERROR) { int errcode = WSAGetLastError(); if (errcode == WSAEWOULDBLOCK) { - return 0; + err.clear(); } else { err = "send failure"; - return -1; } + return -1; } else { assert(wbytes > 0); + err.clear(); return wbytes; } } static int socket_recv(SOCKET socket, char *bytes, int nbytes, std::string &err) { - err.clear(); int nrecv = recv(socket, bytes, nbytes, 0); if (nrecv < 0) { int errcode = WSAGetLastError(); if (errcode == WSAEWOULDBLOCK) { - return 0; + err = ""; } else { err = "recv failure"; - return -1; } - } else if (nrecv == 0) { return -1; + } else if (nrecv == 0) { + err.clear(); + return 0; } else { + err.clear(); return nrecv; } } From aa77480fb57d74b0009f181b40d861314df5b4d0 Mon Sep 17 00:00:00 2001 From: jyelon Date: Mon, 27 Feb 2023 17:21:00 -0500 Subject: [PATCH 2/7] Making progress on util::dprintf --- luprex/cpp/core/drivenengine.cpp | 1 + luprex/cpp/core/drivenengine.hpp | 1 - luprex/cpp/core/enginewrapper.hpp | 17 +++++++++ luprex/cpp/core/util.cpp | 57 +++++++++++++++++++++++++++++++ luprex/cpp/core/util.hpp | 16 +++++++++ luprex/cpp/drv/driver-common.cpp | 6 ++++ 6 files changed, 97 insertions(+), 1 deletion(-) diff --git a/luprex/cpp/core/drivenengine.cpp b/luprex/cpp/core/drivenengine.cpp index 10d57634..8a2ab6e5 100644 --- a/luprex/cpp/core/drivenengine.cpp +++ b/luprex/cpp/core/drivenengine.cpp @@ -1016,6 +1016,7 @@ static void init_engine_wrapper_helper(EngineWrapper *w) { w->replay_initialize = replaycore_initialize; w->replay_step = replaycore_step; + w->hook_dprintf = util::hook_dprintf; w->release = release; }; diff --git a/luprex/cpp/core/drivenengine.hpp b/luprex/cpp/core/drivenengine.hpp index 42f2cc8b..1c8e1f32 100644 --- a/luprex/cpp/core/drivenengine.hpp +++ b/luprex/cpp/core/drivenengine.hpp @@ -318,7 +318,6 @@ private: friend class Channel; }; - ////////////////////////////////////////////////////////////////////////////////// diff --git a/luprex/cpp/core/enginewrapper.hpp b/luprex/cpp/core/enginewrapper.hpp index b7d9d4a4..783d24c9 100644 --- a/luprex/cpp/core/enginewrapper.hpp +++ b/luprex/cpp/core/enginewrapper.hpp @@ -227,6 +227,23 @@ struct EngineWrapper { ////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////// + // Hook dprintf + // + // The engine provides a function 'util::dprintf' to print debugging + // messages. It does not use stderr or std::cerr, because in windows, those + // go to the bit-bucket. + // + // This routine hooks dprintf to change where the output goes. Ideally, + // the intent is to send the output somewhere easily accessible, like the + // visual studio debug output log, or the unreal editor output log, or + // something like that. Or, better yet, to all those places at once. + // + // The hook function will get passed one line of output at a time. The line + // will only contain printable characters. The line will not contain a + // newline - the newline is implied. + // + void (*hook_dprintf)(void (*func)(const char *oneline)); + // Restore the wrapper to its initial blank state. // // Note that the wrapper must have already been initialized using diff --git a/luprex/cpp/core/util.cpp b/luprex/cpp/core/util.cpp index 5b942bd1..887282ca 100644 --- a/luprex/cpp/core/util.cpp +++ b/luprex/cpp/core/util.cpp @@ -720,6 +720,63 @@ eng::string XYZ::debug_string() const { return oss.str(); } + +void (*dprintf_hook)(const char *oneline); + +void hook_dprintf(void (*func)(const char *oneline)) { + dprintf_hook = func; +} + +static void chop_up_dprintf(char *buffer, int size) { + // Drop the final newline, if any. We're going + // to automatically end with a newline and we don't + // want to double up. + if ((size > 0) && (buffer[size-1] == '\n')) { + size -= 1; + } + + // Chop it up into lines and call the hook one line + // at a time. Replace control characters as we go. + const char *base = buffer; + for (int i = 0; i <= size; i++) { + if (buffer[i] < ' ') { + if ((buffer[i] == '\n') || (buffer[i] == 0)) { + buffer[i] = 0; + if (dprintf_hook == nullptr) { + fprintf(stderr, "%s\n", base); + } else { + dprintf_hook(base); + } + base = buffer + i + 1; + } else { + buffer[i] = ' '; + } + } + } + + // If we're sending to stderr, flush. If not, then + // the hook routine is responsible for flushing its own + // output. + if (dprintf_hook == nullptr) { + fflush(stderr); + } +} + +void dprintf(const char *format, ...) { + char buffer[256]; + va_list args; + va_start (args, format); + int n = vsnprintf(buffer, 256, format, args); + if (n <= 255) { + chop_up_dprintf(buffer, n); + } else { + char *lbuffer = (char *)malloc(n + 1); + vsnprintf(lbuffer, n+1, format, args); + chop_up_dprintf(lbuffer, n); + free(lbuffer); + } +} + } // namespace util diff --git a/luprex/cpp/core/util.hpp b/luprex/cpp/core/util.hpp index ab0c6d74..a8fb5a7c 100644 --- a/luprex/cpp/core/util.hpp +++ b/luprex/cpp/core/util.hpp @@ -357,6 +357,22 @@ inline eng::string ss(const ARGS & ... args) { return oss.str(); } + + +// dprintf +// +// Send a debugging message to somewhere that it can be seen. This routine +// initially just sends output to stderr. But it can be hooked to send output +// somewhere else, like to a debug output window. +// +// The hook function must be a function that accepts a single line of text. The +// hook function will always be passed one line, consisting of printable +// characters only. There will be no control characters. The newline is +// implied. +// +void dprintf(const char *format, ...); +void hook_dprintf(void (*func)(const char *oneline)); + // A better API than std::setfill, std::hex, std::setw, std::setprecision // // Usage examples: diff --git a/luprex/cpp/drv/driver-common.cpp b/luprex/cpp/drv/driver-common.cpp index 2228ca9f..934d094f 100644 --- a/luprex/cpp/drv/driver-common.cpp +++ b/luprex/cpp/drv/driver-common.cpp @@ -9,6 +9,11 @@ static void if_error_print_and_exit(const std::string_view str) { } } +static void dprintf_callback(const char *oneline) { + fprintf(stderr, "DPRINTF: %s\n", oneline); + fflush(stderr); +} + class Driver { public: enum ChanState { @@ -536,6 +541,7 @@ class Driver { // Load the DLL and gain access to its functions. call_init_engine_wrapper(&engw); engw.replay_cb_sent_outgoing = replay_cb_sent_outgoing; + engw.hook_dprintf(dprintf_callback); // If argv contains "replay ", do a replay, // and then skip everything else. From 9ce34d950b6d191d88b4eb6f41452bbad6250abc Mon Sep 17 00:00:00 2001 From: jyelon Date: Wed, 1 Mar 2023 16:07:13 -0500 Subject: [PATCH 3/7] Refactor code to make it easier to do 'nopredict' inside of any function without having a dependency on world model. --- luprex/cpp/core/animqueue.cpp | 8 ++++---- luprex/cpp/core/animqueue.hpp | 2 +- luprex/cpp/core/eng-tests.cpp | 2 +- luprex/cpp/core/globaldb.cpp | 10 +++++----- luprex/cpp/core/lpxclient.cpp | 2 +- luprex/cpp/core/lpxserver.cpp | 4 ++-- luprex/cpp/core/luasnap.cpp | 3 ++- luprex/cpp/core/luastack.cpp | 18 ++++++++++++++++++ luprex/cpp/core/luastack.hpp | 18 ++++++++++++++++++ luprex/cpp/core/textgame.cpp | 2 +- luprex/cpp/core/util.hpp | 7 ------- luprex/cpp/core/world-core.cpp | 24 +++++++++++++++--------- luprex/cpp/core/world-pairtab.cpp | 2 -- luprex/cpp/core/world-testing.cpp | 22 +++++++++++----------- luprex/cpp/core/world.hpp | 11 +++++++---- 15 files changed, 86 insertions(+), 49 deletions(-) diff --git a/luprex/cpp/core/animqueue.cpp b/luprex/cpp/core/animqueue.cpp index 76c773b0..4a19fe5a 100644 --- a/luprex/cpp/core/animqueue.cpp +++ b/luprex/cpp/core/animqueue.cpp @@ -323,8 +323,8 @@ bool AnimStep::from_string(const eng::string &config) { return true; } -AnimQueue::AnimQueue(util::WorldType wt) { - version_autoinc_ = (wt == util::WORLD_TYPE_MASTER); +AnimQueue::AnimQueue(WorldType wt) { + version_autoinc_ = (wt == WORLD_TYPE_MASTER); size_limit_ = 10; // Default size limit. steps_.emplace_back(); steps_.front().keep_state_only(); @@ -576,8 +576,8 @@ static bool diff_works(const AnimQueue &master, AnimQueue &sync) { LuaDefine(unittests_animqueue, "", "some unit tests") { // Useful objects. AnimStep stp; - AnimQueue aq(util::WORLD_TYPE_MASTER); - AnimQueue aqds(util::WORLD_TYPE_S_SYNC); + AnimQueue aq(WORLD_TYPE_MASTER); + AnimQueue aqds(WORLD_TYPE_PREDICTIVE); StreamBuffer sb; // Debug string of a newly initialized queue diff --git a/luprex/cpp/core/animqueue.hpp b/luprex/cpp/core/animqueue.hpp index f3aaf94f..e9d16433 100644 --- a/luprex/cpp/core/animqueue.hpp +++ b/luprex/cpp/core/animqueue.hpp @@ -169,7 +169,7 @@ private: class AnimQueue : public eng::nevernew { public: // World type determines whether versions increment or autozero - AnimQueue(util::WorldType wt); + AnimQueue(WorldType wt); // Simple getters. const AnimStep &nth(int n) const { return steps_[n]; } diff --git a/luprex/cpp/core/eng-tests.cpp b/luprex/cpp/core/eng-tests.cpp index f21186a7..e4f17ba5 100644 --- a/luprex/cpp/core/eng-tests.cpp +++ b/luprex/cpp/core/eng-tests.cpp @@ -114,7 +114,7 @@ private: void event_init(int argc, char *argv[]) { - world_.reset(new World(util::WORLD_TYPE_STANDALONE)); + world_.reset(new World(WORLD_TYPE_MASTER)); world_->update_source(get_lua_source()); world_->run_unittests(); stop_driver(); diff --git a/luprex/cpp/core/globaldb.cpp b/luprex/cpp/core/globaldb.cpp index d90367f1..e2de6a1a 100644 --- a/luprex/cpp/core/globaldb.cpp +++ b/luprex/cpp/core/globaldb.cpp @@ -7,6 +7,8 @@ LuaDefine(global_once, "name", "for a given string, returns true exactly once") LuaVar oncedb, val; LuaStack LS(L, name, flag, oncedb, val); + LS.guard_nopredict("global.once"); + // Get a pointer to the oncedb. LS.rawget(oncedb, LuaRegistry, "oncedb"); if (!LS.istable(oncedb)) { @@ -25,11 +27,14 @@ LuaDefine(global_once, "name", "for a given string, returns true exactly once") return LS.result(); } + LuaDefine(global_clearonce, "name", "reset the specified once-flag") { LuaArg name; LuaVar oncedb; LuaStack LS(L, name, oncedb); + LS.guard_nopredict("global.clearonce"); + // Get a pointer to the oncedb. LS.rawget(oncedb, LuaRegistry, "oncedb"); if (!LS.istable(oncedb)) { @@ -50,11 +55,6 @@ LuaDefine(global_table, "globalname", "get a table where global data can be stor // Get a pointer to the globaldb. LS.rawget(globaldb, LuaRegistry, "globaldb"); - // nopredict - if (lua_isyieldable(L) && (!LS.istable(globaldb))) { - return lua_yield(L, 0); - } - // Get the globaltab from the globaldb, sanity check it. LS.rawget(globaltab, globaldb, globalname); if (LS.istable(globaltab)) { diff --git a/luprex/cpp/core/lpxclient.cpp b/luprex/cpp/core/lpxclient.cpp index 811d97eb..02d1d345 100644 --- a/luprex/cpp/core/lpxclient.cpp +++ b/luprex/cpp/core/lpxclient.cpp @@ -24,7 +24,7 @@ public: public: void set_initial_state() { // Create the world model. - world_.reset(new World(util::WORLD_TYPE_C_SYNC)); + world_.reset(new World(WORLD_TYPE_PREDICTIVE)); // This is a temporary actor that will be used only until the server sends // us the first difference transmission. We do this only to establish diff --git a/luprex/cpp/core/lpxserver.cpp b/luprex/cpp/core/lpxserver.cpp index 27dffcc4..9bdec15b 100644 --- a/luprex/cpp/core/lpxserver.cpp +++ b/luprex/cpp/core/lpxserver.cpp @@ -33,7 +33,7 @@ public: public: virtual void event_init(int argc, char *argv[]) { // Create the master world model. - master_.reset(new World(util::WORLD_TYPE_MASTER)); + master_.reset(new World(WORLD_TYPE_MASTER)); // Update the source code of the master model. master_->update_source(get_lua_source()); @@ -189,7 +189,7 @@ public: Client *client = new Client; client->actor_id_ = master_->create_login_actor(); client->channel_ = std::move(chan); - client->sync_.reset(new World(util::WORLD_TYPE_S_SYNC)); + client->sync_.reset(new World(WORLD_TYPE_PREDICTIVE)); client->sync_->create_login_actor(); clients_.emplace_back(client); stdostream() << "New client: actor id=" << client->actor_id_ << std::endl; diff --git a/luprex/cpp/core/luasnap.cpp b/luprex/cpp/core/luasnap.cpp index beb39d59..ae307e71 100644 --- a/luprex/cpp/core/luasnap.cpp +++ b/luprex/cpp/core/luasnap.cpp @@ -85,7 +85,8 @@ void LuaSnap::deserialize(StreamBuffer *sb) { void *ud = sb->lua_reader_ud(len); // Call eris with the permanents table and passing the snapshot as a lua_Reader. - lua_getfield(state_, LUA_REGISTRYINDEX, "unpersist"); + lua_pushstring(state_, "unpersist"); + lua_rawget(state_, LUA_REGISTRYINDEX); eris_undump(state_, sb->lua_reader, ud); assert(lua_gettop(state_) == 2); diff --git a/luprex/cpp/core/luastack.cpp b/luprex/cpp/core/luastack.cpp index 919e9301..1f30cf7b 100644 --- a/luprex/cpp/core/luastack.cpp +++ b/luprex/cpp/core/luastack.cpp @@ -466,6 +466,24 @@ void LuaStack::setvisited(LuaSlot tab, bool visited) const { lua_modflagbits(L_, tab.index(), 0x0010, visited ? 0x0010 : 0); } +WorldType LuaStack::world_type() const { + lua_pushstring(L_, "worldtype"); + lua_rawget(L_, LUA_REGISTRYINDEX); + lua_Integer n = lua_tointeger(L_, -1); + lua_pop(L_, 1); + assert(n != 0); + return (WorldType)n; +} + +void LuaStack::guard_nopredict(const char *fn) { + if (lua_isyieldable(L_)) { + if (!is_authoritative()) { + lua_yield(L_, 0); + luaL_error(L_, "unexplained nopredict failure in %s", fn); + } + } +} + LuaKeywordParser::LuaKeywordParser(lua_State *L, int slot) { L_ = L; slot_ = slot; diff --git a/luprex/cpp/core/luastack.hpp b/luprex/cpp/core/luastack.hpp index 6600e14e..2460514f 100644 --- a/luprex/cpp/core/luastack.hpp +++ b/luprex/cpp/core/luastack.hpp @@ -226,6 +226,13 @@ int LuaTypeTagValue(lua_State *L) { return 0; } #define LUA_TT_GLOBALDB 22 #define LUA_TT_CLASS 23 +// World types enum. + +enum WorldType { + WORLD_TYPE_MASTER = 1, + WORLD_TYPE_PREDICTIVE = 2, +}; + // We use lightuserdata to store 'tokens': short // strings of 8 characters or less. These tokens // are useful as unique markers. The 8 characters @@ -517,6 +524,17 @@ public: bool getvisited(LuaSlot tab) const; void setvisited(LuaSlot tab, bool visited) const; + // Return the world type (from the registry). + WorldType world_type() const; + + // World types that are authoritative. + static bool is_authoritative(WorldType t) { return (t == WORLD_TYPE_MASTER); } + bool is_authoritative() { return is_authoritative(world_type()); } + + // Stop execution of this thread if in a nonauth model, + // and if the thread is not a probe. + void guard_nopredict(const char *fn); + // Return true if the int64 value can be stored as a lua number. static bool int64_storable(int64_t v) { return (v <= MAXINT) && (v >= -MAXINT); } }; diff --git a/luprex/cpp/core/textgame.cpp b/luprex/cpp/core/textgame.cpp index cfac55c2..7fe96480 100644 --- a/luprex/cpp/core/textgame.cpp +++ b/luprex/cpp/core/textgame.cpp @@ -98,7 +98,7 @@ private: void event_init(int argc, char *argv[]) { - world_.reset(new World(util::WORLD_TYPE_STANDALONE)); + world_.reset(new World(WORLD_TYPE_MASTER)); world_->update_source(get_lua_source()); world_->run_unittests(); actor_id_ = world_->create_login_actor(); diff --git a/luprex/cpp/core/util.hpp b/luprex/cpp/core/util.hpp index a8fb5a7c..55b57f9f 100644 --- a/luprex/cpp/core/util.hpp +++ b/luprex/cpp/core/util.hpp @@ -198,13 +198,6 @@ bool valid_number(string_view v, bool plus, bool minus, bool dec, bool exp); namespace util { -enum WorldType { - WORLD_TYPE_STANDALONE, - WORLD_TYPE_C_SYNC, - WORLD_TYPE_S_SYNC, - WORLD_TYPE_MASTER, -}; - enum MessageType { MSG_NULL, MSG_DIFF, diff --git a/luprex/cpp/core/world-core.cpp b/luprex/cpp/core/world-core.cpp index 64122f91..87e5e676 100644 --- a/luprex/cpp/core/world-core.cpp +++ b/luprex/cpp/core/world-core.cpp @@ -29,12 +29,12 @@ World *World::fetch_global_pointer(lua_State *L) { World::~World() { } -World::World(util::WorldType wt) { +World::World(WorldType wt) { // Master world model by default. world_type_ = wt; // Initialize the ID allocator in master mode. - if (wt == util::WORLD_TYPE_MASTER || wt == util::WORLD_TYPE_STANDALONE) { + if (is_authoritative()) { id_global_pool_.init_master(); } else { id_global_pool_.init_synch(); @@ -63,11 +63,12 @@ World::World(util::WorldType wt) { // Create the tangibles table in the registry. LS.rawset(LuaRegistry, "tangibles", LuaNewTable); + // Store the world type in the registry. + LS.rawset(LuaRegistry, "worldtype", wt); + // Create the globaldb and oncedb in the registry. - if ((wt == util::WORLD_TYPE_MASTER) || (wt == util::WORLD_TYPE_STANDALONE)) { - LS.rawset(LuaRegistry, "globaldb", LuaNewTable); - LS.rawset(LuaRegistry, "oncedb", LuaNewTable); - } + LS.rawset(LuaRegistry, "globaldb", LuaNewTable); + LS.rawset(LuaRegistry, "oncedb", LuaNewTable); // Initialize the SourceDB. At this stage, the sourcedb is // empty, so it's just populating the lua builtins. @@ -788,12 +789,12 @@ void World::invoke_lua_source(int64_t actor_id, int64_t place_id, const eng::str void World::guard_blockable(lua_State *L, const char *fn) { if (lthread_thread_id_ == 0) { - // in a probe, http.get throws an error. + // in a probe, blocking functions like http.get throw an error. luaL_error(L, "cannot %s in a probe", fn); assert(false); } if (!is_authoritative()) { - // in a nonauth model, http.get is converted to nopredict. + // in a nonauth model, blocking functions like http.get are converted to nopredict. lua_yield(L, 0); luaL_error(L, "unexplained nopredict failure in %s", fn); assert(false); @@ -801,6 +802,8 @@ void World::guard_blockable(lua_State *L, const char *fn) { } void World::guard_nopredict(lua_State *L, const char *fn) { + // Caution: this code must be equivalent to the + // code in LuaStack::guard_nopredict. if (lthread_thread_id_ == 0) { return; } @@ -883,7 +886,10 @@ void World::run_scheduled_threads() { LS.rawset(thinfo, "isnew", false); LS.rawset(thinfo, "useppool", false); } else { - // In a nonauth model, a yield is converted to a 'nopredict'. + // When a nonauthoritative model yields, for any reason, + // the thread is discarded. This is also used as a way to implement + // nopredict: the thread that wants to 'nopredict' just yields, + // knowing that this will cause it to be killed. LS.rawset(threads, sched.thread_id(), LuaNil); } } else { diff --git a/luprex/cpp/core/world-pairtab.cpp b/luprex/cpp/core/world-pairtab.cpp index f6aab82a..fc1b829b 100644 --- a/luprex/cpp/core/world-pairtab.cpp +++ b/luprex/cpp/core/world-pairtab.cpp @@ -136,8 +136,6 @@ void World::pair_lua_tables(const IdVector &basis, lua_State *master) { // If the master table is already paired, skip. MLS.rawget(midx, mtnmap, mtab); if (MLS.isnumber(midx)) continue; - // If the synch table is not a table, skip. - if (!SLS.istable(stab)) continue; // If the synch table doesn't have a number, skip. SLS.rawget(sidx, stnmap, stab); if (!SLS.isnumber(sidx)) continue; diff --git a/luprex/cpp/core/world-testing.cpp b/luprex/cpp/core/world-testing.cpp index 0d777d08..889fb748 100644 --- a/luprex/cpp/core/world-testing.cpp +++ b/luprex/cpp/core/world-testing.cpp @@ -241,9 +241,9 @@ static bool worlds_identical(const UniqueWorld &w1, const UniqueWorld &w2) { } LuaDefine(unittests_world1animdiff, "", "some unit tests") { - UniqueWorld m(new World(util::WORLD_TYPE_MASTER)); - UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC)); - UniqueWorld cs(new World(util::WORLD_TYPE_C_SYNC)); + UniqueWorld m(new World(WORLD_TYPE_MASTER)); + UniqueWorld ss(new World(WORLD_TYPE_PREDICTIVE)); + UniqueWorld cs(new World(WORLD_TYPE_PREDICTIVE)); StreamBuffer sb; util::IdVector ids = util::id_vector_create(123, 345); @@ -311,8 +311,8 @@ LuaDefine(unittests_world1animdiff, "", "some unit tests") { } LuaDefine(unittests_world2pairtab, "", "some unit tests") { - UniqueWorld m(new World(util::WORLD_TYPE_MASTER)); - UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC)); + UniqueWorld m(new World(WORLD_TYPE_MASTER)); + UniqueWorld ss(new World(WORLD_TYPE_PREDICTIVE)); StreamBuffer sb; int ncreate; @@ -359,9 +359,9 @@ LuaDefine(unittests_world2pairtab, "", "some unit tests") { } LuaDefine(unittests_world3diffluatab, "", "some unit tests") { - UniqueWorld m(new World(util::WORLD_TYPE_MASTER)); - UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC)); - UniqueWorld cs(new World(util::WORLD_TYPE_C_SYNC)); + UniqueWorld m(new World(WORLD_TYPE_MASTER)); + UniqueWorld ss(new World(WORLD_TYPE_PREDICTIVE)); + UniqueWorld cs(new World(WORLD_TYPE_PREDICTIVE)); StreamBuffer sb; // Initialize all three models so that a tangible exists. @@ -413,9 +413,9 @@ LuaDefine(unittests_world3diffluatab, "", "some unit tests") { } LuaDefine(unittests_world4difftanclass, "", "some unit tests") { - UniqueWorld m(new World(util::WORLD_TYPE_MASTER)); - UniqueWorld ss(new World(util::WORLD_TYPE_S_SYNC)); - UniqueWorld cs(new World(util::WORLD_TYPE_C_SYNC)); + UniqueWorld m(new World(WORLD_TYPE_MASTER)); + UniqueWorld ss(new World(WORLD_TYPE_PREDICTIVE)); + UniqueWorld cs(new World(WORLD_TYPE_PREDICTIVE)); StreamBuffer sb; // Initialize all three models so that a tangible exists. diff --git a/luprex/cpp/core/world.hpp b/luprex/cpp/core/world.hpp index b506d91d..6c416105 100644 --- a/luprex/cpp/core/world.hpp +++ b/luprex/cpp/core/world.hpp @@ -104,7 +104,7 @@ public: // The constructor also calls 'lua_open' to create a new // lua interpreter for this world model. // - World(util::WorldType wt); + World(WorldType wt); // Destructor. // @@ -236,8 +236,8 @@ public: // Check if the world is authoritative. // - bool is_authoritative() const { return (world_type_ == util::WORLD_TYPE_MASTER) || (world_type_ == util::WORLD_TYPE_STANDALONE); } - + bool is_authoritative() const { return LuaStack::is_authoritative(world_type_); } + // Get a table showing all outstanding HTTP requests. // const HttpClientRequestMap &http_requests() const { return http_requests_; } @@ -249,6 +249,9 @@ public: // Snapshot and rollback. // + // These are used by the client to convert the synchronous model + // to an asynchronous model and back. + // void snapshot(); void rollback(); bool snapshot_empty() { return snapshot_.empty(); } @@ -494,7 +497,7 @@ public: private: // Type of model - util::WorldType world_type_; + WorldType world_type_; // A lua intepreter with snapshot function. // From db234c29348c7e9241fddd42856ab8b21269a1cc Mon Sep 17 00:00:00 2001 From: jyelon Date: Wed, 1 Mar 2023 17:35:06 -0500 Subject: [PATCH 4/7] Stub out the new global.set and global.get operators --- luprex/cpp/core/globaldb.cpp | 120 +++++++++++++++++---------------- luprex/cpp/core/table.cpp | 56 ++++++++++++++- luprex/cpp/core/world-core.cpp | 3 +- luprex/lua/control.lst | 1 - luprex/lua/ut-globaldb.lua | 13 ---- luprex/lua/ut-tablecmp.lua | 10 --- 6 files changed, 115 insertions(+), 88 deletions(-) delete mode 100644 luprex/lua/ut-globaldb.lua diff --git a/luprex/cpp/core/globaldb.cpp b/luprex/cpp/core/globaldb.cpp index e2de6a1a..9a1778ab 100644 --- a/luprex/cpp/core/globaldb.cpp +++ b/luprex/cpp/core/globaldb.cpp @@ -1,72 +1,74 @@ #include "luastack.hpp" #include "globaldb.hpp" -LuaDefine(global_once, "name", "for a given string, returns true exactly once") { + +LuaDefine(global_set, "varname, value", + "|Store data in the global data table." + "|" + "|You can store global data using global.set, then you can" + "|retrieve it using global.get. You can also retrieve data using" + "|gv.varname, which is just shorthand for global.get. You may not" + "|store data using gv.varname=value, this yields the error 'Use " + "|global.set to store data in the global data table.'" + "|" + "|Values stored using global.set are difference transmitted to all" + "|connected clients immediately. When a new client connects," + "|he will receive all the global data." + "|" + "|The global data table is not the same thing as the lua " + "|environment table. Trying to store data in the lua environment" + "|table will seem to work, at first, but the data will not get" + "|difference transmitted, and will eventually be cleared and lost." + "|Therefore, it is essential that global data be stored in the" + "|global data table (using global.set) instead of in the lua" + "|environment table." + "|" + "|There are certain restrictions on the values that you store." + "|The value can contain strings, numbers, simple tables, and" + "|tangibles. Nothing else is allowed. Simple tables are tables" + "|that don't have metatables, and that aren't special tables such" + "|as classes, the lua environment table, the registry, or the like." + "|" + "|When you store the value, a recursive copy is made and stored." + "|When you call global.get, you obtain the copy. Any attempt to" + "|mutate the copy will fail with this lua error message: 'Tables" + "|returned by global.get are immutable.' This rule prevents'" + "|aliasing between global data and other data structures." + "|") { + return 0; +} + +LuaDefine(global_get, "varname", + "|Get data stored using global.set" + "|" + "|See doc(global.set) for information on how to store global data." + "|") { + return 0; +} + +LuaDefine(global_once, "name", + "|For a given string, returns true exactly once" + "|" + "|The semantics and difference transmission behavior of global.once" + "|are identical to the semantics of global.set, since global.once" + "|uses global.set under the covers." + "|") { LuaArg name; LuaRet flag; LuaVar oncedb, val; LuaStack LS(L, name, flag, oncedb, val); - - LS.guard_nopredict("global.once"); - - // Get a pointer to the oncedb. - LS.rawget(oncedb, LuaRegistry, "oncedb"); - if (!LS.istable(oncedb)) { - LS.set(flag, false); - return LS.result(); - } - - LS.checkstring(name, "name"); - LS.rawget(val, oncedb, name); - if (!LS.isnil(val)) { - LS.set(flag, false); - return LS.result(); - } - LS.rawset(oncedb, name, true); - LS.set(flag, true); - return LS.result(); + return 0; } - -LuaDefine(global_clearonce, "name", "reset the specified once-flag") { +LuaDefine(global_clearonce, "name", + "|Reset the specified once-flag" + "|" + "|The semantics and difference transmission behavior of global.clearonce" + "|are identical to the semantics of global.set, since global.once" + "|uses global.set under the covers." + "|") { LuaArg name; LuaVar oncedb; LuaStack LS(L, name, oncedb); - - LS.guard_nopredict("global.clearonce"); - - // Get a pointer to the oncedb. - LS.rawget(oncedb, LuaRegistry, "oncedb"); - if (!LS.istable(oncedb)) { - return LS.result(); - } - LS.checkstring(name, "name"); - LS.rawset(oncedb, name, LuaNil); - return LS.result(); -} - -LuaDefine(global_table, "globalname", "get a table where global data can be stored") { - LuaArg globalname; - LuaRet globaltab; - LuaVar globaldb; - LuaStack LS(L, globalname, globaltab, globaldb); - LS.checkstring(globalname, "globalname"); - - // Get a pointer to the globaldb. - LS.rawget(globaldb, LuaRegistry, "globaldb"); - - // Get the globaltab from the globaldb, sanity check it. - LS.rawget(globaltab, globaldb, globalname); - if (LS.istable(globaltab)) { - return LS.result(); - } else if (!LS.isnil(globaltab)) { - luaL_error(L, "%s is not a global", LS.ckstring(globalname).c_str()); - } - - // Create a new globaltab and store it in the globaldb. - LS.newtable(globaltab); - LS.rawset(globaldb, globalname, globaltab); - LS.rawset(globaltab, "__global", globalname); - LS.settabletype(globaltab, LUA_TT_GLOBALDB); - return LS.result(); + return 0; } diff --git a/luprex/cpp/core/table.cpp b/luprex/cpp/core/table.cpp index cc722c35..e3788b70 100644 --- a/luprex/cpp/core/table.cpp +++ b/luprex/cpp/core/table.cpp @@ -589,7 +589,17 @@ LuaDefine(table_nextsortedpair, "sortedpairs,dummy", "next function used by sort } } -LuaDefine(table_sortedpairs, "table", "iterate over table, sorting all keys") { +LuaDefine(table_sortedpairs, "table", + "|Iterate over table, sorting all keys" + "|" + "|Some keys can't be sorted. For example, you can use a closure" + "|as a table key. If you try to iterate over a table containing" + "|a non-sortable key, the error 'Cannot sort the table keys' will" + "|be generated." + "|" + "|See doc(genlt) for information about the sort order." + "|" + "|") { LuaArg tab; LuaRet closure, rtab, key; LuaStack LS(L, tab, closure, rtab, key); @@ -602,7 +612,17 @@ LuaDefine(table_sortedpairs, "table", "iterate over table, sorting all keys") { return LS.result(); } -LuaDefine(table_semisortedpairs, "table", "iterate over table, sorting those keys that can be sorted") { +LuaDefine(table_semisortedpairs, "table", + "|Iterate over table, sorting all the keys that can be sorted." + "|" + "|Some keys can't be sorted. For example, you can use a closure" + "|as a table key. If you try to iterate over a table containing" + "|a non-sortable key, the non-sortable elements will appear at" + "|the end of the iteration, after all the sortable elements. The" + "|non-sortable elements will be in an arbitrary order." + "|" + "|See doc(genlt) for information about the sort order." + "|") { LuaArg tab; LuaRet closure, rtab, key; LuaStack LS(L, tab, closure, rtab, key); @@ -612,7 +632,37 @@ LuaDefine(table_semisortedpairs, "table", "iterate over table, sorting those key return LS.result(); } -LuaDefine(genlt, "obj1,obj2", "return true if obj1 is less than obj2 in general ordering") { +#define LUA_TNIL 0 +#define LUA_TBOOLEAN 1 +#define LUA_TLIGHTUSERDATA 2 +#define LUA_TNUMBER 3 +#define LUA_TSTRING 4 +#define LUA_TTABLE 5 +#define LUA_TFUNCTION 6 +#define LUA_TUSERDATA 7 +#define LUA_TTHREAD 8 + +#define LUA_NUMTAGS 9 + +LuaDefine(genlt, "obj1,obj2", + "|Generalized less-than function" + "|" + "|This comparison function can compare any two objects. The" + "|return value is as follows:" + "|" + "|* Numbers are compared in the obvious numeric manner." + "|* Strings are compared alphabetically." + "|* Booleans are compared with false being less than true." + "|* Tables are all considered equal to other tables." + "|* Functions are all considered equal to other functions." + "|* Coroutines are all considered equal to other coroutines." + "|" + "|* Numbers are less than strings." + "|* Strings are less than booleans." + "|* Booleans are less than functions." + "|* Functions are less than coroutines." + "|* Coroutines are less than tables." + "|") { LuaArg o1,o2; LuaRet lt; LuaStack LS(L, o1, o2, lt); diff --git a/luprex/cpp/core/world-core.cpp b/luprex/cpp/core/world-core.cpp index 87e5e676..c7ec45b7 100644 --- a/luprex/cpp/core/world-core.cpp +++ b/luprex/cpp/core/world-core.cpp @@ -66,9 +66,8 @@ World::World(WorldType wt) { // Store the world type in the registry. LS.rawset(LuaRegistry, "worldtype", wt); - // Create the globaldb and oncedb in the registry. + // Create the globaldb in the registry. LS.rawset(LuaRegistry, "globaldb", LuaNewTable); - LS.rawset(LuaRegistry, "oncedb", LuaNewTable); // Initialize the SourceDB. At this stage, the sourcedb is // empty, so it's just populating the lua builtins. diff --git a/luprex/lua/control.lst b/luprex/lua/control.lst index f41943a2..4a6a9ba5 100644 --- a/luprex/lua/control.lst +++ b/luprex/lua/control.lst @@ -4,7 +4,6 @@ # ut-table.lua -ut-globaldb.lua ut-tablecmp.lua basics.lua uglyglobals.lua diff --git a/luprex/lua/ut-globaldb.lua b/luprex/lua/ut-globaldb.lua deleted file mode 100644 index 431ce663..00000000 --- a/luprex/lua/ut-globaldb.lua +++ /dev/null @@ -1,13 +0,0 @@ -makeclass("unittests") - -function unittests.globaldb() - local g1a = global.table("unittest-g1") - local g2a = global.table("unittest-g2") - local g1b = global.table("unittest-g1") - local g2b = global.table("unittest-g2") - assert(g1a == g1b) - assert(g2a == g2b) - assert(g1a.__global == "unittest-g1") - assert(g2a.__global == "unittest-g2") -end - diff --git a/luprex/lua/ut-tablecmp.lua b/luprex/lua/ut-tablecmp.lua index 53d5b101..5699b104 100644 --- a/luprex/lua/ut-tablecmp.lua +++ b/luprex/lua/ut-tablecmp.lua @@ -51,11 +51,6 @@ function unittests.diffcompare() -- Try a table containing a pointer to the global environment. assert(tdc({}, {a=_G}, {}, {}) == "a=globals;") - -- GlobalDB tables should be forced to NIL. - assert(tdc({}, {a=global.table("foo")}, {}, {a=global.table("foo")}) == "a=nil;"); - assert(tdc({}, {}, {}, {a=global.table("foo")}) == "a=nil;"); - assert(tdc({}, {a=global.table("foo")}, {}, {}) == ""); - -- Set up some numbered tables for tests involving such. local mtab10 = {} local stab10 = {} @@ -116,11 +111,6 @@ function unittests.diffapply() -- verify a table containing the global environment. assert(tda(tnmap, {a=_G}, {})) - -- GlobalDB tables should be forced to NIL. - rtab={a=3} - assert(not tda({}, {a=global.table("foo")}, rtab)) - assert(rtab.a == nil) - -- Unnumbered tables should be forced to NIL. rtab={a=3} assert(not tda({}, {a={}}, rtab)) From 86a27ef2d44fa2019b04c92c48795e9de7503061 Mon Sep 17 00:00:00 2001 From: jyelon Date: Sun, 5 Mar 2023 01:51:25 -0500 Subject: [PATCH 5/7] Fix util::ostringstream, fix pretty-printing, stub out new globals --- luprex/cpp/core/globaldb.cpp | 5 +- luprex/cpp/core/luastack.hpp | 7 +- luprex/cpp/core/pprint.cpp | 583 ++++++++++++++++------------- luprex/cpp/core/pprint.hpp | 10 +- luprex/cpp/core/table.hpp | 1 + luprex/cpp/core/util.hpp | 27 +- luprex/cpp/core/world-accessor.cpp | 54 ++- luprex/cpp/core/world-core.cpp | 4 +- luprex/cpp/core/world-testing.cpp | 4 +- luprex/cpp/drv/driver-common.cpp | 6 +- luprex/cpp/drv/drvutil.hpp | 20 +- luprex/cpp/wrap/wrap-sstream.hpp | 3 + luprex/lua/ut-globaldb.lua | 13 + 13 files changed, 455 insertions(+), 282 deletions(-) create mode 100644 luprex/lua/ut-globaldb.lua diff --git a/luprex/cpp/core/globaldb.cpp b/luprex/cpp/core/globaldb.cpp index 9a1778ab..2e372a00 100644 --- a/luprex/cpp/core/globaldb.cpp +++ b/luprex/cpp/core/globaldb.cpp @@ -5,13 +5,16 @@ LuaDefine(global_set, "varname, value", "|Store data in the global data table." "|" + "|The variable name must be a string which is a valid" + "|lua identifier." + "|" "|You can store global data using global.set, then you can" "|retrieve it using global.get. You can also retrieve data using" "|gv.varname, which is just shorthand for global.get. You may not" "|store data using gv.varname=value, this yields the error 'Use " "|global.set to store data in the global data table.'" "|" - "|Values stored using global.set are difference transmitted to all" + "|Values stored using global.set are transmitted to all" "|connected clients immediately. When a new client connects," "|he will receive all the global data." "|" diff --git a/luprex/cpp/core/luastack.hpp b/luprex/cpp/core/luastack.hpp index 2460514f..01fae478 100644 --- a/luprex/cpp/core/luastack.hpp +++ b/luprex/cpp/core/luastack.hpp @@ -474,8 +474,9 @@ public: return lua_rawequal(L_, v1, v2); } - bool rawequal(LuaSlot v1, const char *name) const { - push_any_value(name); + template + bool rawequal(LuaSlot v1, VT value) const { + push_any_value(value); bool result = lua_rawequal(L_, v1, -1); lua_pop(L_, 1); return result; @@ -617,9 +618,11 @@ public: }; #define LuaTokenConstant(name, tvalue, docs) \ + LuaToken ltoken_##name(tvalue); \ LuaConstantReg reg_##name(#name, docs, LuaToken(tvalue), 0); #define LuaNumberConstant(name, nvalue, docs) \ + lua_Number lnumber_##name(nvalue); \ LuaConstantReg reg_##name(#name, docs, LuaToken(), nvalue); #define LuaDefine(name, args, docs) \ diff --git a/luprex/cpp/core/pprint.cpp b/luprex/cpp/core/pprint.cpp index 61a2c918..80f74be3 100644 --- a/luprex/cpp/core/pprint.cpp +++ b/luprex/cpp/core/pprint.cpp @@ -7,255 +7,352 @@ #include #include +class PrintMachine { +public: + LuaVar tabchpos_; + LuaStack LS_; + int next_id_; + bool indent_; + std::ostream *output_; + eng::map chpos_to_tabnum_; -void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os) { - int tt = LS.type(val); - switch (tt) { - case LUA_TNIL: - (*os) << "nil"; - return; - case LUA_TSTRING: - if (quote) { - util::quote_string(LS.ckstring(val), os); - } else { - // TODO: this could be more efficient. - (*os) << LS.ckstring(val); - } - return; - case LUA_TNUMBER: { - double value = LS.cknumber(val); - if (std::isnan(value)) { - (*os) << "nan"; - } else { - int64_t ivalue = int64_t(value); - if (double(ivalue) == value) { - (*os) << ivalue; + void atomic_print(int xtype, LuaSlot val, bool quote) { + switch (xtype) { + case LUA_TNIL: { + (*output_) << "nil"; + return; + } + case LUA_TSTRING: { + if (quote) { + util::quote_string(LS_.ckstring(val), output_); } else { - (*os) << value; - } + // TODO: this could be more efficient. + (*output_) << LS_.ckstring(val); + } + return; } - return; - } - case LUA_TBOOLEAN: - (*os) << (LS.ckboolean(val) ? "true" : "false"); - return; - case LUA_TFUNCTION: { - (*os) << ""; - return; - } - case LUA_TLIGHTUSERDATA: { - LuaToken token = LS.cktoken(val); - (*os) << "[" << token.str() << "]"; - return; - } - default: - (*os) << "<" << lua_typename(LS.state(), tt) << ">"; - return; - } -} - - -// Find tables recursively. -// -// Builds a table (tabcount) whose keys are tables. If a table -// is visited exactly once, then tabcount[t]=0. If a table is -// visited more than once, then tabcount[t]=-1. -// -static void findtables(LuaStack &LS0, LuaSlot root, LuaSlot tabcount) { - lua_State *L = LS0.state(); - LuaVar key, val, tab, count; - LuaStack LS(L, key, val, tab, count); - - LS.newtable(tabcount); - int top = lua_gettop(L); - if (LS.istable(root)) { - lua_pushvalue(L, root.index()); - } - while (lua_gettop(L) > top) { - lua_checkstack(L, 20); - lua_replace(L, tab.index()); - LS.rawget(count, tabcount, tab); - if (LS.isnil(count)) { - LS.rawset(tabcount, tab, 0); - LS.set(key, LuaNil); - while (LS.next(tab, key, val)) { - lua_checkstack(L, 20); - if (LS.istable(key)) { - lua_pushvalue(L, key.index()); - } - if (LS.istable(val)) { - lua_pushvalue(L, val.index()); + case LUA_TNUMBER: { + double value = LS_.cknumber(val); + if (std::isnan(value)) { + (*output_) << "nan"; + } else { + int64_t ivalue = int64_t(value); + if (double(ivalue) == value) { + (*output_) << ivalue; + } else { + (*output_) << value; } } - LS.getmetatable(val, tab); - if (LS.istable(val)) { - lua_pushvalue(L, val.index()); + return; + } + case LUA_TBOOLEAN: { + (*output_) << (LS_.ckboolean(val) ? "true" : "false"); + return; + } + case LUA_TFUNCTION: { + (*output_) << ""; + return; + } + case LUA_TTHREAD: { + (*output_) << ""; + return; + } + case LUA_TLIGHTUSERDATA: { + LuaToken token = LS_.cktoken(val); + (*output_) << "[" << token.str() << "]"; + return; + } + case LUA_TT_GENERAL: { + (*output_) << ""; + return; + } + case LUA_TT_TANGIBLE: { + (*output_) << ""; + return; + } + case LUA_TT_CLASS: { + (*output_) << ""; + return; + } + case LUA_TT_GLOBALENV: { + (*output_) << ""; + return; + } + default: { + (*output_) << ""; + return; + }} + } + + void tabify(int level) { + if (indent_) { + (*output_) << std::endl; + for (int i = 0; i < level; i++) { + (*output_) << " "; } } else { - LS.rawset(tabcount, tab, -1); + (*output_) << " "; } } + + void pprint_r(int level, bool expand, LuaSlot value) { + lua_State *L = LS_.state(); + lua_checkstack(L, 20); + LuaVar loffset, pairs, key, val, lchpos, nextseq; + LuaStack LS(L, loffset, pairs, key, val, lchpos, nextseq); + + // Determine the extended type of the object. If the + // expand flag is true, try to coerce it to a general table. + int type = LS.xtype(value); + if (expand && (LS.istable(value))) { + type = LUA_TT_GENERAL; + } + + // If it's anything but a general table, use 'atomic_print' + // and return. + if (type != LUA_TT_GENERAL) { + atomic_print(type, value, true); + LS.result(); + return; + } + + // Find out whether it has a table number. If necessary, + // assign one. + int tabnum = 0; + LS.rawget(lchpos, tabchpos_, value); + if (!LS.isnumber(lchpos)) { + // First time. Record the character position where the + // table first appears in the output stream. + LS.rawset(tabchpos_, value, int((*output_).tellp())); + } else { + int chpos = LS.ckint(lchpos); + tabnum = chpos_to_tabnum_[chpos]; + if (tabnum == 0) { + // Second time. The table is already in the output, + // but it hasn't been assigned a number. Assign one. + tabnum = next_id_++; + chpos_to_tabnum_[chpos] = tabnum; + } + } + + // If the table has a number, that means we're visiting + // it for the second time. Just print an abbreviated version + // and return. + if (tabnum > 0) { + (*output_) << "
"; + if (lua_nkeys(L, value.index())==0) { + (*output_) << "{}"; + } else { + (*output_) << "{...}"; + } + return; + } + + // State variables. + bool needcomma = false; + bool multiline = false; + LS.set(nextseq, 1); + + // Open the brackets. + (*output_) << "{"; + + // Output the table keys. + table_getpairs(LS, value, pairs, true); + for (int i = 2; ; i+=2) { + LS.rawget(key, pairs, i); + if (LS.isnil(key)) break; + LS.rawget(val, pairs, i+1); + if (needcomma) (*output_) << ","; + needcomma = true; + if (LS.rawequal(key, nextseq)) { + (*output_) << " "; + pprint_r(level + 1, false, val); + LS.set(nextseq, LS.ckinteger(nextseq) + 1); + } else { + multiline = true; + tabify(level + 1); + if (LS.isstring(key) && sv::is_lua_id(LS.ckstring(key))) { + (*output_) << LS.ckstring(key); + } else { + (*output_) << "["; + pprint_r(level + 1, false, key); + (*output_) << "]"; + } + if (indent_) { + (*output_) << " = "; + } else { + (*output_) << "="; + } + pprint_r(level + 1, false, val); + } + } + + // Output the metatable. + LS.getmetatable(val, value); + if (LS.istable(val)) { + multiline = true; + if (needcomma) (*output_) << ","; + needcomma = true; + tabify(level + 1); + (*output_) << " = "; + pprint_r(level + 1, false, val); + } + + // Close the brackets. + if (multiline) { + tabify(level); + } else if (LS.ckinteger(nextseq) > 1) { + (*output_) << " "; + } + (*output_) << "}"; + + LS.result(); + } + + // Atomic print interface. + PrintMachine(LuaStack &LS0, LuaSlot root, bool quote, std::ostream *os) : + LS_(LS0.state(), tabchpos_) { + output_ = os; + atomic_print(LS_.xtype(root), root, quote); + LS_.result(); + } + + // Pretty print interface. + PrintMachine(LuaStack &LS0, LuaSlot root, bool indent, int level, bool expand, std::ostream *os) : + LS_(LS0.state(), tabchpos_) { + next_id_ = 1; + indent_ = indent; + LS_.newtable(tabchpos_); + util::ostringstream preoutput; + output_ = &preoutput; + pprint_r(level, expand, root); + std::string_view pre = preoutput.view(); + + // Output the results. We would just copy the characters + // one by one to the target stream, except that we have to + // insert
in front of all tables that got referenced. + chpos_to_tabnum_.emplace(0x7FFFFFFF, 0); + auto iter = chpos_to_tabnum_.begin(); + for (int i = 0; i < int(pre.size()); i++) { + if (i == iter->first) { + (*os) << "
second << ">"; + iter++; + } + (*os).put(pre[i]); + } + LS_.result(); + } +}; + +void PrettyPrintOptions::parse(LuaKeywordParser &kp) { + LuaVar option; + LuaStack LS(kp.state(), option); + if (kp.parse(option, "indent")) { + indent = LS.ckboolean(option); + } + if (kp.parse(option, "level")) { + level = LS.ckint(option); + } + if (kp.parse(option, "expand")) { + expand = LS.ckboolean(option); + } LS.result(); } -LuaDefine(table_findtables, "root", "recursively find tables (debugging only)") { - LuaArg root; - LuaRet tabcount; - LuaStack LS(L, root, tabcount); - findtables(LS, root, tabcount); +void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os) { + PrintMachine pm(LS, val, quote, os); +} + +void pprint(LuaStack &LS, LuaSlot val, const PrettyPrintOptions &opts, std::ostream *os) { + PrintMachine pm(LS, val, opts.indent, opts.level, opts.expand, os); +} + +LuaDefine(string_pprint, "obj1, obj2, ...", + "|Pretty-print the specified objects into a string." + "|" + "|See also: string.pprintx, which has a lot more options." + "|This function uses the default options: pretty print indented," + "|start at indentation level zero, and always expand the" + "|top-level table." + "|") { + int n = lua_gettop(L); + LuaRet result; + LuaStack LS(L, result); + util::ostringstream oss; + for (int i = 1; i <= n; i++) { + LuaSpecial root(i); + pprint(LS, root, PrettyPrintOptions(), &oss); + if (i < n) oss << "\n"; + } + oss << std::endl; + LS.set(result, oss.view()); return LS.result(); } -struct Inspector { - lua_State *L; - LuaVar ids; - int nextid; - bool indent; - int maxlen; - bool anyindent; - std::ostream *stream; -}; - -static void tabify(Inspector &insp, int level) { - if (insp.indent) { - (*insp.stream) << std::endl; - for (int i = 0; i < level; i++) { - (*insp.stream) << " "; - } - insp.anyindent = true; - } else { - (*insp.stream) << " "; +LuaDefine(string_pprintx, "options", + "|Pretty-print the specified object into a string, with options" + "|" + "|Options is a table with these fields:" + "|" + "| value - the object to pretty-print" + "| indent - if false, suppress newlines and indentation (default: true)" + "| level - base level of indentation (default: zero)" + "| expand - if true, force expansion of top-level table (default: false)" + "|" + "|About the expand flag: normally, when you print a class, it just " + "|prints '', and when you print a tangible, it just" + "|prints ''. But sometimes, you want to see the details." + "|The expand flag forces it to expand the top-level table, even if the" + "|top-level table is a tangible or class." + "|") { + LuaArg loptions; + LuaRet result; + LuaVar value; + LuaStack LS(L, loptions, result, value); + PrettyPrintOptions options; + LuaKeywordParser kp(LS, loptions); + options.parse(kp); + if (!kp.parse(value, "value")) { + LS.set(value, LuaNil); } + kp.final_check_throw(); + util::ostringstream oss; + pprint(LS, value, options, &oss); + LS.set(result, oss.view()); + return LS.result(); } -static void pprint_r(Inspector &insp, int level, LuaSlot root) { - lua_checkstack(insp.L, 20); - LuaVar idv, pairs, key, val, nextseq; - LuaStack LS(insp.L, idv, pairs, key, val, nextseq); - - // If it's anything but a table, use 'atomic_print'. - if (!LS.istable(root)) { - atomic_print(LS, root, true, insp.stream); - LS.result(); - return; - } - - // Determine the table's ID, allocating an ID if necessary. - LS.rawget(idv, insp.ids, root); - int id = LS.ckint(idv); - bool new_id = false; - if (id < 0) { - new_id = true; - id = insp.nextid++; - LS.rawset(insp.ids, root, id); - } - - // Print the table's name, if any. - bool is_class = false; - bool is_tangible = false; - eng::string cname = LS.classname(root); - if (cname != "") { - is_class = true; - (*insp.stream) << ""; - } else { - int64_t tid = LS.tanid(root); - if (tid > 0) { - is_tangible = true; - (*insp.stream) << ""; - } else if (id > 0) { - (*insp.stream) << "
"; - } - } - - // If this is a class, and we're not at the top level, truncate. - if ((is_class || is_tangible) && (level > 0)) { - LS.result(); - return; - } - - // If this is a table we've already printed, truncate it. - if ((id > 0) && (!new_id)) { - if (lua_nkeys(insp.L, root.index())==0) { - (*insp.stream) << "{}"; - } else { - (*insp.stream) << "{...}"; - } - LS.result(); - return; - } - - // State variables. - bool needcomma = false; - bool multiline = false; - LS.set(nextseq, 1); - - // Open the brackets. - (*insp.stream) << "{"; - - // Output the table keys. - table_getpairs(LS, root, pairs, true); - for (int i = 2; ; i+=2) { - LS.rawget(key, pairs, i); - if (LS.isnil(key)) break; - LS.rawget(val, pairs, i+1); - if (needcomma) (*insp.stream) << ","; - needcomma = true; - if (LS.rawequal(key, nextseq)) { - (*insp.stream) << " "; - pprint_r(insp, level + 1, val); - LS.set(nextseq, LS.ckinteger(nextseq) + 1); - } else { - multiline = true; - tabify(insp, level + 1); - if (LS.isstring(key) && sv::is_lua_id(LS.ckstring(key))) { - (*insp.stream) << LS.ckstring(key); - } else { - (*insp.stream) << "["; - pprint_r(insp, level + 1, key); - (*insp.stream) << "]"; - } - if (insp.indent) { - (*insp.stream) << " = "; - } else { - (*insp.stream) << "="; - } - pprint_r(insp, level + 1, val); - } - } - - // Output the metatable. - LS.getmetatable(val, root); - if (LS.istable(val)) { - multiline = true; - if (needcomma) (*insp.stream) << ","; - needcomma = true; - tabify(insp, level + 1); - (*insp.stream) << " = "; - pprint_r(insp, level + 1, val); - } - - // Close the brackets. - if (multiline) { - tabify(insp, level); - } else if (LS.ckinteger(nextseq) > 1) { - (*insp.stream) << " "; - } - (*insp.stream) << "}"; - - LS.result(); +LuaDefine(string_print, "obj", + "|Concise print the specified object into a string" + "|" + "|This prints a concise representation of obj into a string. Tables" + "|are not expanded: for that, use string.pprintx. The functions" + "|tostring and string.print are identical." + "|") { + LuaArg val; + LuaRet result; + LuaStack LS(L, val, result); + eng::ostringstream oss; + atomic_print(LS, val, false, &oss); + LS.set(result, oss.str()); + return LS.result(); } -void pprint(LuaStack &LS0, LuaSlot root, bool indent, std::ostream *os) { - Inspector insp; - LuaStack LS(LS0.state(), insp.ids); - findtables(LS, root, insp.ids); - insp.L = LS0.state(); - insp.nextid = 1; - insp.indent = indent; - insp.anyindent = false; - insp.stream = os; - pprint_r(insp, 0, root); - LS.result(); + +LuaDefine(tostring, "obj", + "|Concise print the specified object into a string" + "|" + "|This prints a concise representation of obj into a string. Tables" + "|are not expanded: for that, use string.pprint. The functions" + "|tostring and string.print are identical." + "|") { + LuaArg val; + LuaRet result; + LuaStack LS(L, val, result); + eng::ostringstream oss; + atomic_print(LS, val, false, &oss); + LS.set(result, oss.str()); + return LS.result(); } LuaDefine(string_isidentifier, "str", "return true if the string is a valid lua identifier") { @@ -271,33 +368,5 @@ LuaDefine(string_isidentifier, "str", "return true if the string is a valid lua return LS.result(); } -LuaDefine(string_print, "obj", "print the specified object into a string") { - LuaArg val; - LuaRet result; - LuaStack LS(L, val, result); - eng::ostringstream oss; - atomic_print(LS, val, false, &oss); - LS.set(result, oss.str()); - return LS.result(); -} -LuaDefine(string_pprint, "obj,indent", "pretty-print the specified object into a string") { - LuaArg root, indent; - LuaRet result; - LuaStack LS(L, root, indent, result); - bool ind = LS.ckboolean(indent); - eng::ostringstream oss; - pprint(LS, root, ind, &oss); - LS.set(result, oss.str()); - return LS.result(); -} -LuaDefine(tostring, "obj", "print the specified object into a string") { - LuaArg val; - LuaRet result; - LuaStack LS(L, val, result); - eng::ostringstream oss; - atomic_print(LS, val, false, &oss); - LS.set(result, oss.str()); - return LS.result(); -} diff --git a/luprex/cpp/core/pprint.hpp b/luprex/cpp/core/pprint.hpp index c39a1a82..10ca7066 100644 --- a/luprex/cpp/core/pprint.hpp +++ b/luprex/cpp/core/pprint.hpp @@ -22,6 +22,14 @@ #include "luastack.hpp" #include +struct PrettyPrintOptions { + bool indent; + int level; + bool expand; + PrettyPrintOptions() : indent(true), level(0), expand(true) {} + void parse(LuaKeywordParser &kp); +}; + // Atomic print to a stream. // // This prints an atomic value to a stream. If you give it a table, @@ -32,6 +40,6 @@ void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os); // Pretty print to a stream. // -void pprint(LuaStack &LS, LuaSlot val, bool indent, std::ostream *os); +void pprint(LuaStack &LS, LuaSlot val, const PrettyPrintOptions &opts, std::ostream *os); #endif // PPRINT_HPP \ No newline at end of file diff --git a/luprex/cpp/core/table.hpp b/luprex/cpp/core/table.hpp index a572b43e..dba1ad22 100644 --- a/luprex/cpp/core/table.hpp +++ b/luprex/cpp/core/table.hpp @@ -25,4 +25,5 @@ bool table_equal(LuaStack &LS0, LuaSlot tab1, LuaSlot tab2); // bool table_getpairs(LuaStack &LS0, LuaSlot tab, LuaSlot pairs, bool sort); + #endif // TABLE_HPP diff --git a/luprex/cpp/core/util.hpp b/luprex/cpp/core/util.hpp index 55b57f9f..438c008f 100644 --- a/luprex/cpp/core/util.hpp +++ b/luprex/cpp/core/util.hpp @@ -350,7 +350,32 @@ inline eng::string ss(const ARGS & ... args) { return oss.str(); } - +// util::ostringstream +// +// This is a variant of ostringstream in which it is possible +// to get the contents without copying. To get the contents +// without copying, use oss.size() and oss.c_str() +// +class ostringstream : public eng::ostringstream { + class rstringbuf : public std::basic_stringbuf { + public: + char *eback() const { return std::streambuf::eback(); } + char *pptr() const { return std::streambuf::pptr(); } + }; + rstringbuf rstringbuf_; +public: + ostringstream() { + std::basic_ostream::rdbuf(&rstringbuf_); + } + std::string_view view() const { + char *p = rstringbuf_.eback(); + size_t size = rstringbuf_.pptr() - p; + return std::string_view(p, size); + } + eng::string str() const { + return rstringbuf_.str(); + } +}; // dprintf // diff --git a/luprex/cpp/core/world-accessor.cpp b/luprex/cpp/core/world-accessor.cpp index fe18f595..f41f260e 100644 --- a/luprex/cpp/core/world-accessor.cpp +++ b/luprex/cpp/core/world-accessor.cpp @@ -711,21 +711,61 @@ LuaDefine(math_randomstate, "seed", LuaSandboxBuiltin(math_randomseed, "", ""); - -LuaDefine(pprint, "obj1,obj2,...", - "|Pretty-print object or objects.") { +LuaDefine(pprint, "obj1, obj2, ...", + "|Pretty-print the specified objects." + "|" + "|See also: pprintx, which has a lot more options." + "|This function uses the default options: pretty print indented," + "|start at indentation level zero, and always expand the" + "|top-level table." + "|") { World *w = World::fetch_global_pointer(L); std::ostream *ostream = w->lthread_print_stream(); + int n = lua_gettop(L); LuaStack LS(L); - for (int i = 1; i <= lua_gettop(L); i++) { + for (int i = 1; i <= n; i++) { LuaSpecial root(i); - pprint(LS, root, true, ostream); - (*ostream) << std::endl; + pprint(LS, root, PrettyPrintOptions(), ostream); + if (i < n) (*ostream) << "\n"; } + (*ostream) << std::endl; + return LS.result(); + +} + +LuaDefine(pprintx, "options", + "|Pretty-print the specified object, with options" + "|" + "|Options is a table with these fields:" + "|" + "| value - the object to pretty-print" + "| indent - if false, suppress newlines and indentation (default: true)" + "| level - base level of indentation (default: zero)" + "| expand - if true, force expansion of top-level table (default: false)" + "|" + "|About the expand flag: normally, when you print a class, it just " + "|prints '', and when you print a tangible, it just" + "|prints ''. But sometimes, you want to see the details." + "|The expand flag forces it to expand the top-level table, even if the" + "|top-level table is a tangible or class." + "|") { + World *w = World::fetch_global_pointer(L); + std::ostream *ostream = w->lthread_print_stream(); + LuaArg loptions; + LuaVar value; + LuaStack LS(L, loptions, value); + PrettyPrintOptions options; + LuaKeywordParser kp(LS, loptions); + options.parse(kp); + if (!kp.parse(value, "value")) { + LS.set(value, LuaNil); + } + kp.final_check_throw(); + pprint(LS, value, options, ostream); return LS.result(); } -LuaDefine(print, "obj1,obj2,...", +LuaDefine(print, "obj1, obj2, ...", "|Print object or objects.") { World *w = World::fetch_global_pointer(L); std::ostream *ostream = w->lthread_print_stream(); diff --git a/luprex/cpp/core/world-core.cpp b/luprex/cpp/core/world-core.cpp index c7ec45b7..4b7b27f2 100644 --- a/luprex/cpp/core/world-core.cpp +++ b/luprex/cpp/core/world-core.cpp @@ -297,7 +297,7 @@ eng::string World::probe_lua(int64_t actor_id, const eng::string &lua) { if (msg.empty()) { for (int i = top + 1; i <= lua_gettop(L); i++) { LuaSpecial root(i); - pprint(LS, root, true, ostream); + pprint(LS, root, PrettyPrintOptions(), ostream); // TODO: this endl is unnecessary if we just printed a newline. (*ostream) << std::endl; } @@ -876,7 +876,7 @@ void World::run_scheduled_threads() { LuaStack LSCO(CO); if (LS.ckboolean(print)) { for (int i = 1; i <= lua_gettop(CO); i++) { - pprint(LSCO, LuaSpecial(i), true, ostream); + pprint(LSCO, LuaSpecial(i), PrettyPrintOptions(), ostream); (*ostream) << std::endl; } } diff --git a/luprex/cpp/core/world-testing.cpp b/luprex/cpp/core/world-testing.cpp index 889fb748..4dff9b01 100644 --- a/luprex/cpp/core/world-testing.cpp +++ b/luprex/cpp/core/world-testing.cpp @@ -57,7 +57,9 @@ eng::string World::tangible_pprint(int64_t id) const { if (LS.istable(tan)) { LS.getmetatable(meta, tan); LS.clearmetatable(tan); - pprint(LS, tan, false, &oss); + PrettyPrintOptions opts; + opts.indent = false; + pprint(LS, tan, opts, &oss); LS.setmetatable(tan, meta); } else { oss << ""; diff --git a/luprex/cpp/drv/driver-common.cpp b/luprex/cpp/drv/driver-common.cpp index 934d094f..2affc465 100644 --- a/luprex/cpp/drv/driver-common.cpp +++ b/luprex/cpp/drv/driver-common.cpp @@ -158,7 +158,8 @@ class Driver { drvutil::ostringstream oss; std::string err = drvutil::package_lua_source(".", &oss); if_error_print_and_exit(err); - engw.play_set_lua_source(&engw, oss.size(), oss.c_str()); + std::string_view ossv = oss.view(); + engw.play_set_lua_source(&engw, ossv.size(), ossv.data()); } } @@ -590,7 +591,8 @@ class Driver { if_error_print_and_exit(srcpakerr); // Initialize the engine. - engw.play_initialize(&engw, argc, argv, srcpak.size(), srcpak.c_str(), replaylogfn.c_str()); + std::string_view srcpakv = srcpak.view(); + engw.play_initialize(&engw, argc, argv, srcpakv.size(), srcpakv.data(), replaylogfn.c_str()); if_error_print_and_exit(engw.error); // Set up listening ports. diff --git a/luprex/cpp/drv/drvutil.hpp b/luprex/cpp/drv/drvutil.hpp index 852d1216..eb4eb1e7 100644 --- a/luprex/cpp/drv/drvutil.hpp +++ b/luprex/cpp/drv/drvutil.hpp @@ -68,23 +68,27 @@ double get_monotonic_clock(); // without copying, use oss.size() and oss.c_str() // class ostringstream : public std::ostringstream { - class rstringbuf : public std::stringbuf { + class rstringbuf : public std::basic_stringbuf { public: - char *eback() { return std::streambuf::eback(); } + char *eback() const { return std::streambuf::eback(); } + char *pptr() const { return std::streambuf::pptr(); } }; - rstringbuf rsbuf_; + rstringbuf rstringbuf_; public: ostringstream() { - std::basic_ostream::rdbuf(&rsbuf_); + std::basic_ostream::rdbuf(&rstringbuf_); } - size_t size() { - return tellp(); + std::string_view view() const { + char *p = rstringbuf_.eback(); + size_t size = rstringbuf_.pptr() - p; + return std::string_view(p, size); } - const char *c_str() { - return rsbuf_.eback(); + std::string str() const { + return rstringbuf_.str(); } }; + // Remove items from a vector that are marked for deletion. // template diff --git a/luprex/cpp/wrap/wrap-sstream.hpp b/luprex/cpp/wrap/wrap-sstream.hpp index 785dfcbe..eb60b0ee 100644 --- a/luprex/cpp/wrap/wrap-sstream.hpp +++ b/luprex/cpp/wrap/wrap-sstream.hpp @@ -12,7 +12,10 @@ class basic_ostringstream : public std::basic_ostringstream>; using underlying::basic_ostringstream; }; +//template> +//using basic_stringbuf = std::basic_stringbuf>; using ostringstream = basic_ostringstream; +//using stringbuf = basic_stringbuf; } // namespace eng #endif // WRAP_SSTREAM_HPP diff --git a/luprex/lua/ut-globaldb.lua b/luprex/lua/ut-globaldb.lua new file mode 100644 index 00000000..431ce663 --- /dev/null +++ b/luprex/lua/ut-globaldb.lua @@ -0,0 +1,13 @@ +makeclass("unittests") + +function unittests.globaldb() + local g1a = global.table("unittest-g1") + local g2a = global.table("unittest-g2") + local g1b = global.table("unittest-g1") + local g2b = global.table("unittest-g2") + assert(g1a == g1b) + assert(g2a == g2b) + assert(g1a.__global == "unittest-g1") + assert(g2a.__global == "unittest-g2") +end + From a8c780563fae3ebc4594a818ec7781d86837d92e Mon Sep 17 00:00:00 2001 From: jyelon Date: Thu, 16 Mar 2023 23:31:29 -0400 Subject: [PATCH 6/7] Change the creation/deletion of tangibles so that the tangible database is never really deleted. --- luprex/cpp/core/luastack.cpp | 48 ++++++++++++++++++++++++++ luprex/cpp/core/luastack.hpp | 17 +++++++--- luprex/cpp/core/pprint.cpp | 24 +++++++------ luprex/cpp/core/world-accessor.cpp | 31 +++++++++++------ luprex/cpp/core/world-core.cpp | 54 +++++++++++++----------------- luprex/cpp/core/world-difftab.cpp | 7 +--- luprex/cpp/core/world-diffxmit.cpp | 40 +++++++++++++--------- luprex/cpp/core/world-testing.cpp | 9 ++--- luprex/cpp/core/world.hpp | 12 ++++--- 9 files changed, 152 insertions(+), 90 deletions(-) diff --git a/luprex/cpp/core/luastack.cpp b/luprex/cpp/core/luastack.cpp index 1f30cf7b..ccd09706 100644 --- a/luprex/cpp/core/luastack.cpp +++ b/luprex/cpp/core/luastack.cpp @@ -376,6 +376,54 @@ void LuaStack::makeclass(LuaSlot tab, std::string_view name) const { lua_pop(L_, 1); } +void LuaStack::maketan(LuaSlot tab, int64_t id) const { + LuaVar tangibles, metatab; + LuaStack LS(L_, tangibles, metatab); + + // Try to get the existing tangible. + LS.rawget(tangibles, LuaRegistry, "tangibles"); + LS.rawget(tab, tangibles, id); + + // If we succeeded, return it. + if (LS.istable(tab)) { + LS.result(); + return; + } + + // Create the tangible's database and metatable. + LS.set(tab, LuaNewTable); + LS.set(metatab, LuaNewTable); + LS.setmetatable(tab, metatab); + + // Mark the tangible using the tabletype field. + LS.settabletype(tab, LUA_TT_TANGIBLE); + LS.settabletype(metatab, LUA_TT_TANGIBLEMETA); + + // Store the tangible ID and lock the metatable. + LS.rawset(metatab, "id", id); + LS.rawset(metatab, "__metatable", false); + + // Store the database into the tangibles table. + LS.rawset(tangibles, id, tab); + + LS.result(); +} + + +bool LuaStack::tanblank(LuaSlot tab) const { + bool result = true; + if (istable(tab) && gettabletype(tab) == LUA_TT_TANGIBLE) { + if (lua_getmetatable(L_, tab.index())) { + lua_pushstring(L_, "threads"); + lua_rawget(L_, -2); + if (lua_type(L_, -1) == LUA_TTABLE) { + result = false; + } + lua_pop(L_, 2); + } + } + return result; +} int64_t LuaStack::tanid(LuaSlot tab) const { int64_t result = 0; diff --git a/luprex/cpp/core/luastack.hpp b/luprex/cpp/core/luastack.hpp index 01fae478..b68a66e5 100644 --- a/luprex/cpp/core/luastack.hpp +++ b/luprex/cpp/core/luastack.hpp @@ -222,9 +222,8 @@ int LuaTypeTagValue(lua_State *L) { return 0; } #define LUA_TT_GLOBALENV 18 #define LUA_TT_TANGIBLE 19 #define LUA_TT_TANGIBLEMETA 20 -#define LUA_TT_DEADTANGIBLE 21 -#define LUA_TT_GLOBALDB 22 -#define LUA_TT_CLASS 23 +#define LUA_TT_GLOBALDB 21 +#define LUA_TT_CLASS 22 // World types enum. @@ -458,8 +457,16 @@ public: void makeclass(LuaSlot tab, LuaSlot name) const; void makeclass(LuaSlot tab, std::string_view name) const; - // Get the ID of a tangible. It's a little weird to put this in - // this module. + // Create a tangible, or look up an existing tangible. + // This doesn't do the entire process of tangible creation. It + // just creates an empty table, marks it as a tangible, creates + // the metatable, stores the tangible ID, and stores it in the tangible table. + void maketan(LuaSlot tab, int64_t id) const; + + // Return true if a tangible is empty (deleted or not yet created). + bool tanblank(LuaSlot tab) const; + + // Get the ID of a tangible. int64_t tanid(LuaSlot tab) const; // Return true if the value is a sortable key (string, number, or boolean). diff --git a/luprex/cpp/core/pprint.cpp b/luprex/cpp/core/pprint.cpp index 80f74be3..ab778790 100644 --- a/luprex/cpp/core/pprint.cpp +++ b/luprex/cpp/core/pprint.cpp @@ -78,6 +78,10 @@ public: (*output_) << ""; return; } + case LUA_TT_TANGIBLEMETA: { + (*output_) << ""; + return; + } default: { (*output_) << ""; return; @@ -103,17 +107,15 @@ public: // Determine the extended type of the object. If the // expand flag is true, try to coerce it to a general table. - int type = LS.xtype(value); - if (expand && (LS.istable(value))) { - type = LUA_TT_GENERAL; - } + int xtype = LS.xtype(value); - // If it's anything but a general table, use 'atomic_print' - // and return. - if (type != LUA_TT_GENERAL) { - atomic_print(type, value, true); - LS.result(); - return; + // Print the atomic portion. + if (xtype != LUA_TT_GENERAL) { + atomic_print(xtype, value, true); + if ((xtype < LUA_TT_GENERAL) || (!expand)) { + LS.result(); + return; + } } // Find out whether it has a table number. If necessary, @@ -189,7 +191,7 @@ public: // Output the metatable. LS.getmetatable(val, value); - if (LS.istable(val)) { + if (LS.istable(val) && (LS.gettabletype(val) != LUA_TT_TANGIBLEMETA)) { multiline = true; if (needcomma) (*output_) << ","; needcomma = true; diff --git a/luprex/cpp/core/world-accessor.cpp b/luprex/cpp/core/world-accessor.cpp index f41f260e..b98107ac 100644 --- a/luprex/cpp/core/world-accessor.cpp +++ b/luprex/cpp/core/world-accessor.cpp @@ -26,7 +26,7 @@ LuaDefine(tangible_animstate, "tan", LuaRet graphic, plane, x, y, z, facing; LuaStack LS(L, tanobj, graphic, plane, x, y, z, facing); World *w = World::fetch_global_pointer(L); - Tangible *tan = w->tangible_get(LS, tanobj); + Tangible *tan = w->tangible_get(LS, tanobj, false); const AnimStep &aqback = tan->anim_queue_.back(); LS.set(graphic, aqback.graphic()); LS.set(plane, aqback.plane()); @@ -44,7 +44,7 @@ LuaDefine(tangible_xyz, "tan", LuaRet x, y, z; LuaStack LS(L, tanobj, x, y, z); World *w = World::fetch_global_pointer(L); - Tangible *tan = w->tangible_get(LS, tanobj); + Tangible *tan = w->tangible_get(LS, tanobj, false); const AnimStep &aqback = tan->anim_queue_.back(); LS.set(x, aqback.xyz().x); LS.set(y, aqback.xyz().y); @@ -60,7 +60,7 @@ LuaDefine(tangible_animate, "tan,configtable", LuaStack LS(L, tanobj, config); LuaKeywordParser kp(LS, config); World *w = World::fetch_global_pointer(L); - Tangible *tan = w->tangible_get(LS, tanobj); + Tangible *tan = w->tangible_get(LS, tanobj, false); int64_t id = w->alloc_id_predictable(); AnimStep step; step.configure(kp, tan->anim_queue_.back()); @@ -82,7 +82,7 @@ LuaDefine(tangible_setclass, "tan,class", LuaVar classtab, mt; LuaStack LS(L, tanobj, classname, classtab, mt); World *w = World::fetch_global_pointer(L); - w->tangible_get(LS, tanobj); + w->tangible_get(LS, tanobj, false); eng::string err = LS.getclass(classtab, classname); if (err != "") { luaL_error(L, "%s", err.c_str()); @@ -101,7 +101,7 @@ LuaDefine(tangible_getclass, "tan", LuaRet classname; LuaStack LS(L, tanobj, mt, classtab, classname); World *w = World::fetch_global_pointer(L); - w->tangible_get(LS, tanobj); + w->tangible_get(LS, tanobj, false); LS.getmetatable(mt, tanobj); LS.rawget(classtab, mt, "__index"); eng::string name = LS.classname(classtab); @@ -120,8 +120,10 @@ LuaDefine(tangible_delete, "tan", LuaArg tanobj; LuaStack LS(L, tanobj); World *w = World::fetch_global_pointer(L); - Tangible *tan = w->tangible_get(LS, tanobj); - assert(tan != nullptr); // this should be checked above. + Tangible *tan = w->tangible_get(LS, tanobj, true); + if (tan == nullptr) { + return LS.result(); + } if (tan->is_an_actor()) { luaL_error(L, "Cannot delete a player using tangible.delete, use tangible.redirect instead."); return 0; @@ -205,14 +207,14 @@ LuaDefine(tangible_redirect, "tan1,tan2,bulldozetan1", LuaStack LS(L, actor1, actor2, bldz); World *w = World::fetch_global_pointer(L); bool bulldoze = LS.ckboolean(bldz); - Tangible *tan1 = w->tangible_get(LS, actor1); + Tangible *tan1 = w->tangible_get(LS, actor1, false); if (!tan1->is_an_actor()) { luaL_error(L, "redirect source is not an actor"); } if (LS.isnil(actor2)) { w->redirects_[tan1->id()] = 0; } else { - Tangible *tan2 = w->tangible_get(LS, actor2); + Tangible *tan2 = w->tangible_get(LS, actor2, false); tan2->configure_id_pool_for_actor(); w->redirects_[tan1->id()] = tan2->id(); } @@ -268,7 +270,7 @@ LuaDefine(tangible_near, "tan,radius,omit_nowhere,omit_self", LuaRet list; LuaStack LS(L, ltan, lradius, lomit_nowhere, lomit_self, list); World *w = World::fetch_global_pointer(L); - Tangible *tan = w->tangible_get(LS, ltan); + Tangible *tan = w->tangible_get(LS, ltan, false); const AnimStep &aqback = tan->anim_queue_.back(); PlaneScan scan; @@ -460,13 +462,20 @@ LuaDefine(tangible_start, "tangible,function,arg1,arg2...", } } + // Check the entire tangible list for validity. + for (int i = 1; ; i++) { + LS.rawget(place, tanlist, i); + if (LS.isnil(place)) break; + w->tangible_get(LS, place, false); + } + + // Start threads on all the tangibles. for (int i = 1; ; i++) { LS.rawget(place, tanlist, i); if (LS.isnil(place)) break; // Confirm that the place is a valid tangible, // and get the tangible ID. - w->tangible_get(LS, place); place_id = LS.tanid(place); // Get place's metatable and threads table. diff --git a/luprex/cpp/core/world-core.cpp b/luprex/cpp/core/world-core.cpp index 4b7b27f2..62c5dd1b 100644 --- a/luprex/cpp/core/world-core.cpp +++ b/luprex/cpp/core/world-core.cpp @@ -134,14 +134,16 @@ World::TanVector World::tangible_get_all(const IdVector &ids) const { return result; } -Tangible *World::tangible_get(const LuaStack &LS, LuaSlot tab) { +Tangible *World::tangible_get(const LuaStack &LS, LuaSlot tab, bool allowdel) { int64_t id = LS.tanid(tab); if (id == 0) { luaL_error(LS.state(), "parameter is not a tangible"); } Tangible *result = tangible_get(id); - if (result == nullptr) { - luaL_error(LS.state(), "parameter is not a tangible"); + if (!allowdel) { + if (result == nullptr) { + luaL_error(LS.state(), "argument is a deleted tangible, which is not allowed here"); + } } return result; } @@ -154,9 +156,9 @@ Tangible *World::tangible_make(lua_State *L, int64_t id, const eng::string &plan } assert(id != 0); - LuaVar tangibles, metatab; + LuaVar metatab; LuaRet database; - LuaStack LS(L, tangibles, database, metatab); + LuaStack LS(L, database, metatab); // Create the C++ part of the structure. UniqueTangible &t = tangibles_[id]; @@ -167,25 +169,14 @@ Tangible *World::tangible_make(lua_State *L, int64_t id, const eng::string &plan t->anim_queue_.clear(plane); t->update_plane_item(); - // Create the tangible's database and metatable. - LS.set(database, LuaNewTable); - LS.set(metatab, LuaNewTable); - LS.setmetatable(database, metatab); + // Fetch the tangible's Lua database and metatable. + LS.maketan(database, id); + LS.getmetatable(metatab, database); - // Mark the tangible using the tabletype field. - LS.settabletype(database, LUA_TT_TANGIBLE); - LS.settabletype(metatab, LUA_TT_TANGIBLEMETA); - - // Store the database into the tangibles table. - LS.rawget(tangibles, LuaRegistry, "tangibles"); - LS.rawset(tangibles, id, database); - - // Populate the database and metatable with initial stuff. + // Set up the inventory and thread table. LS.rawset(database, "inventory", LuaNewTable); - LS.rawset(metatab, "id", id); LS.rawset(metatab, "threads", LuaNewTable); - // LS.rawset(metatab, "__metatable", LuaNil); - + LS.result(); if (!pushdb) lua_pop(L, 1); return t.get(); @@ -193,8 +184,8 @@ Tangible *World::tangible_make(lua_State *L, int64_t id, const eng::string &plan void World::tangible_delete(int64_t id) { lua_State *L = state(); - LuaVar tangibles, database; - LuaStack LS(L, tangibles, database); + LuaVar tangibles, database, metatab; + LuaStack LS(L, tangibles, database, metatab); // Fetch the C++ side of the tangible. auto iter = tangibles_.find(id); @@ -204,16 +195,17 @@ void World::tangible_delete(int64_t id) { } // Fetch the lua side of the tangible. - LS.rawget(tangibles, LuaRegistry, "tangibles"); - LS.rawget(database, tangibles, id); + LS.maketan(database, id); assert(LS.istable(database)); - - // Clear out the database. - LS.cleartable(database, true); - LS.settabletype(database, LUA_TT_DEADTANGIBLE); + LS.getmetatable(metatab, database); - // Remove the lua portion from the tangibles table. - LS.rawset(tangibles, id, LuaNil); + // Clear out the database and the metatable. + LS.cleartable(database, false); + LS.cleartable(metatab, true); + + // Now put the bare minimum info back into the metatable. + LS.rawset(metatab, "id", id); + LS.rawset(metatab, "__metatable", false); // Remove the C++ portion from the tangibles table. tangibles_.erase(iter); diff --git a/luprex/cpp/core/world-difftab.cpp b/luprex/cpp/core/world-difftab.cpp index d7410895..c286191f 100644 --- a/luprex/cpp/core/world-difftab.cpp +++ b/luprex/cpp/core/world-difftab.cpp @@ -304,12 +304,7 @@ static void set_transmitted_value(LuaStack &LS, LuaSlot tangibles, LuaSlot ntmap case LUA_TT_TANGIBLE: { int64_t id = sb->read_int64(); DebugLine(dbc) << dbinfo << "tan " << id; - LS.rawget(target, tangibles, id); - if (LS.isnil(target)) { - World *w = World::fetch_global_pointer(LS.state()); - w->tangible_make(LS.state(), id, "nowhere", true); - lua_replace(LS.state(), target.index()); - } + LS.maketan(target, id); return; } case LUA_TT_GLOBALENV: { diff --git a/luprex/cpp/core/world-diffxmit.cpp b/luprex/cpp/core/world-diffxmit.cpp index abb60d52..edc76abf 100644 --- a/luprex/cpp/core/world-diffxmit.cpp +++ b/luprex/cpp/core/world-diffxmit.cpp @@ -1,10 +1,11 @@ -#include "world.hpp" #include +#include "world.hpp" + util::IdVector World::get_visible_union(int64_t actor_id, World *master) { return util::sort_union_id_vectors( - master->get_near(actor_id, RadiusVisibility, true, false, false), - get_near(actor_id, RadiusVisibility, true, false, false)); + master->get_near(actor_id, RadiusVisibility, true, false, false), + get_near(actor_id, RadiusVisibility, true, false, false)); } int64_t World::patch_actor(StreamBuffer *sb, DebugCollector *dbc) { @@ -27,14 +28,14 @@ int64_t World::patch_actor(StreamBuffer *sb, DebugCollector *dbc) { return actor_id; } -void World::diff_actor(int64_t actor_id, World *master, StreamBuffer *xsb) { +void World::diff_actor(int64_t actor_id, World *master, StreamBuffer *xsb) { StreamBuffer tsb; // Get the actor in both models. s_actor may be nil. const Tangible *s_actor = tangible_get(actor_id); const Tangible *m_actor = master->tangible_get(actor_id); assert(m_actor != nullptr); - + // Calculate diffs. tsb.write_int64(actor_id); if (s_actor == nullptr) { @@ -85,15 +86,17 @@ void World::patch_visible(StreamBuffer *sb, DebugCollector *dbc) { } } -void World::diff_visible(const util::IdVector &visible, World *master, StreamBuffer *xsb) { +void World::diff_visible(const util::IdVector &visible, World *master, + StreamBuffer *xsb) { StreamBuffer tsb; // Get the specified tangibles in both models. - // Some tangibles may be missing in the master, some may be missing in the sync. + // Some tangibles may be missing in the master, some may be missing in the + // sync. TanVector mvis = master->tangible_get_all(visible); TanVector svis = tangible_get_all(visible); assert(mvis.size() == svis.size()); - + // For each tangible that exists in the master, but not // in the synchronous model, send a create message. tsb.write_int32(0); @@ -169,11 +172,13 @@ void World::patch_luatabs(StreamBuffer *sb, DebugCollector *dbc) { int64_t actor_id = sb->read_int64(); util::HashValue closehash = sb->read_hashvalue(); int ncreate = sb->read_int32(); - util::IdVector closetans = get_near(actor_id, RadiusClose, true, false, true); + util::IdVector closetans = + get_near(actor_id, RadiusClose, true, false, true); assert(closehash == util::hash_id_vector(closetans)); number_lua_tables(closetans); create_new_tables(ncreate); - // DebugLine(dbc) << "lua tables: " << nt << " existing " << ncreate << " new"; + // DebugLine(dbc) << "lua tables: " << nt << " existing " << ncreate << " + // new"; patch_tangible_databases(sb, dbc); patch_numbered_tables(sb, dbc); unnumber_lua_tables(); @@ -183,7 +188,8 @@ void World::diff_luatabs(int64_t actor_id, World *master, StreamBuffer *xsb) { StreamBuffer tsb; // Calculate the set of close tangibles. - util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false, true); + util::IdVector closetans = + master->get_near(actor_id, RadiusClose, true, false, true); assert(get_near(actor_id, RadiusClose, true, false, true) == closetans); util::HashValue closehash = util::hash_id_vector(closetans); @@ -199,7 +205,7 @@ void World::diff_luatabs(int64_t actor_id, World *master, StreamBuffer *xsb) { tsb.write_int32(ncreate); diff_tangible_databases(closetans, master->state(), &tsb); diff_numbered_tables(master->state(), &tsb); - + // Forward to client, and apply to server-synchronous. tsb.copy_into(xsb); assert(tsb.read_int64() == actor_id); @@ -207,7 +213,7 @@ void World::diff_luatabs(int64_t actor_id, World *master, StreamBuffer *xsb) { assert(tsb.read_int32() == ncreate); patch_tangible_databases(&tsb, nullptr); patch_numbered_tables(&tsb, nullptr); - assert(tsb.empty()); + assert(tsb.empty()); // Unnumber tables in both models. unnumber_lua_tables(); @@ -249,8 +255,10 @@ void World::diff_tanclass(int64_t actor_id, World *master, StreamBuffer *xsb) { MLS.rawget(mtangibles, LuaRegistry, "tangibles"); // Calculate the set of close tangibles. - // TODO: we've already calculated this in an earlier function. This is wasteful. - util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false, true); + // TODO: we've already calculated this in an earlier function. This is + // wasteful. + util::IdVector closetans = + master->get_near(actor_id, RadiusClose, true, false, true); tsb.write_int32(0); int write_count_after = tsb.total_writes(); @@ -310,7 +318,7 @@ int64_t World::patch_everything(StreamBuffer *sb, DebugCollector *dbc) { return actor_id; } -void World::diff_everything(int64_t actor_id, World *master, StreamBuffer *sb) { +void World::diff_everything(int64_t actor_id, World *master, StreamBuffer *sb) { diff_actor(actor_id, master, sb); util::IdVector visible = get_visible_union(actor_id, master); diff_visible(visible, master, sb); diff --git a/luprex/cpp/core/world-testing.cpp b/luprex/cpp/core/world-testing.cpp index 4dff9b01..8fe81c0f 100644 --- a/luprex/cpp/core/world-testing.cpp +++ b/luprex/cpp/core/world-testing.cpp @@ -55,14 +55,11 @@ eng::string World::tangible_pprint(int64_t id) const { LS.rawget(tan, tangibles, id); eng::ostringstream oss; if (LS.istable(tan)) { - LS.getmetatable(meta, tan); - LS.clearmetatable(tan); PrettyPrintOptions opts; opts.indent = false; pprint(LS, tan, opts, &oss); - LS.setmetatable(tan, meta); } else { - oss << ""; + oss << "{}"; } LS.result(); return oss.str(); @@ -384,7 +381,7 @@ LuaDefine(unittests_world3diffluatab, "", "some unit tests") { // The data in the master model should now look like this: const char *expect_123 = - "{ " + "{ " "bacon='crispy', " "inventory={ gold='wealthy' }, " "skills={ " @@ -393,7 +390,7 @@ LuaDefine(unittests_world3diffluatab, "", "some unit tests") { "} " "}"; const char *expect_345 = - "{ " + "{ " "inventory={ gold='poor' }, " "phone='867-5309' " "}"; diff --git a/luprex/cpp/core/world.hpp b/luprex/cpp/core/world.hpp index 6c416105..bf584fc8 100644 --- a/luprex/cpp/core/world.hpp +++ b/luprex/cpp/core/world.hpp @@ -137,17 +137,21 @@ public: // Get a pointer to the specified tangible. // - // If there's no such tangible, returns nullptr. + // If there's no such tangible, or if the tangible is deleted, + // returns nullptr. // Tangible *tangible_get(int64_t id); const Tangible *tangible_get(int64_t id) const; // Get a pointer to the specified tangible. // - // The value on the lua stack should be a valid lua tangible. If not, - // a lua error is generated. + // The value on the lua stack should be a lua tangible. // - Tangible *tangible_get(const LuaStack &LS, LuaSlot slot); + // If the 'allowdel' flag is true, then it is valid to pass in + // a deleted tangible. In that case, this function returns nullptr, + // but this is not a Lua error. + // + Tangible *tangible_get(const LuaStack &LS, LuaSlot slot, bool allowdel); // Get pointers to many tangibles. // From 5ede3f507c8d9588c532052d65e653953bf84f40 Mon Sep 17 00:00:00 2001 From: jyelon Date: Fri, 17 Mar 2023 13:31:10 -0400 Subject: [PATCH 7/7] Change how class tables are created and recreated for greater robustness --- luprex/cpp/core/luastack.cpp | 76 +++++++++++++++++++++++----------- luprex/cpp/core/luastack.hpp | 8 ++-- luprex/cpp/core/source.cpp | 28 ++++++------- luprex/cpp/core/world-core.cpp | 5 +-- 4 files changed, 71 insertions(+), 46 deletions(-) diff --git a/luprex/cpp/core/luastack.cpp b/luprex/cpp/core/luastack.cpp index ccd09706..47e8007d 100644 --- a/luprex/cpp/core/luastack.cpp +++ b/luprex/cpp/core/luastack.cpp @@ -75,7 +75,20 @@ static void *l_alloc(void *ud, void *ptr, size_t osize, size_t nsize) { lua_State *LuaStack::newstate (lua_Alloc allocf) { if (allocf == nullptr) allocf = l_alloc; lua_State *L = lua_newstate(allocf, NULL); - if (L) lua_atpanic(L, &panicf); + assert(L != nullptr); + lua_atpanic(L, &panicf); + + // We want all states to have a classes table and + // a tangibles table so that LS.makeclass and LS.maketan + // work out-of-the-box. + + lua_pushstring(L, "classes"); + lua_newtable(L); + lua_rawset(L, LUA_REGISTRYINDEX); + lua_pushstring(L, "tangibles"); + lua_newtable(L); + lua_rawset(L, LUA_REGISTRYINDEX); + return L; } @@ -256,27 +269,35 @@ bool LuaStack::validclassname(LuaSlot slot) const { eng::string LuaStack::classname(LuaSlot tab) const { eng::string result; - if (istable(tab)) { - lua_pushstring(L_, "__class"); - lua_rawget(L_, tab); - if (lua_type(L_, -1) == LUA_TSTRING) { - lua_pushglobaltable(L_); // cname table - lua_pushvalue(L_, -2); // cname table cname - lua_rawget(L_, -2); // cname table ctab - if (lua_rawequal(L_, -1, tab)) { - size_t len; - const char *s = lua_tolstring(L_, -3, &len); - result = eng::string(s, len); - if (!validclassname(result)) { - result = ""; - } + if ((istable(tab)) && (gettabletype(tab) == LUA_TT_CLASS)) { + LuaVar classes, name, dup; + LuaStack LS(L_, classes, name, dup); + // Get the classes table from the registry. + LS.rawget(classes, LuaRegistry, "classes"); + + // Try the efficient approach: get the class name from + // the class, and confirm it by checking the classes table. + LS.rawget(name, tab, "__class"); + if (LS.isstring(name)) { + LS.rawget(dup, classes, name); + if (LS.rawequal(dup, tab)) { + result = LS.ckstring(name); + LS.result(); + return result; + } + } + + // Do it the brute force way: scan the classes table. + LS.set(name, LuaNil); + while (LS.next(classes, name, dup)) { + if (LS.rawequal(dup, tab)) { + result = LS.ckstring(name); + LS.result(); + return result; } - lua_pop(L_, 3); - } else { - lua_pop(L_, 1); } } - return result; + return ""; } eng::string LuaStack::getclass(LuaSlot classtab, LuaSlot classname) const { @@ -342,24 +363,31 @@ eng::string LuaStack::getclass(LuaSlot tab, std::string_view name) const { void LuaStack::makeclass(LuaSlot classtab, LuaSlot classname) const { lua_checkstack(L_, 20); - LuaVar globtab, cname; - LuaStack LS(L_, globtab, cname); + LuaVar classes, globtab, cname; + LuaStack LS(L_, classes, globtab, cname); // Validate the class name. assert(LS.validclassname(classname)); + // Fetch the classes table from the registry. + LS.rawget(classes, LuaRegistry, "classes"); + assert(LS.istable(classes)); + // Fetch the global environment from the registry. LS.getglobaltable(globtab); - // Get the classtab from the global environment. - LS.rawget(classtab, globtab, classname); + // Get the classtab from the classes table. + LS.rawget(classtab, classes, classname); // Make a new table if necessary. if (!LS.istable(classtab)) { LS.newtable(classtab); - LS.rawset(globtab, classname, classtab); + LS.rawset(classes, classname, classtab); } + // Put the table into the global environment. + LS.rawset(globtab, classname, classtab); + // Repair the special fields. LS.settabletype(classtab, LUA_TT_CLASS); LS.rawset(classtab, "__class", classname); diff --git a/luprex/cpp/core/luastack.hpp b/luprex/cpp/core/luastack.hpp index b68a66e5..a5a4daeb 100644 --- a/luprex/cpp/core/luastack.hpp +++ b/luprex/cpp/core/luastack.hpp @@ -458,9 +458,11 @@ public: void makeclass(LuaSlot tab, std::string_view name) const; // Create a tangible, or look up an existing tangible. - // This doesn't do the entire process of tangible creation. It - // just creates an empty table, marks it as a tangible, creates - // the metatable, stores the tangible ID, and stores it in the tangible table. + // If the tangible doesn't exist yet, this creates a tangible stub. + // It is possible to use World::tangible_make to transform a tangible + // stub into a full blown tangible, and World::tangible_delete to turn + // a full-blown tangible back into a stub. A stub doesn't have a + // class or a thread table. void maketan(LuaSlot tab, int64_t id) const; // Return true if a tangible is empty (deleted or not yet created). diff --git a/luprex/cpp/core/source.cpp b/luprex/cpp/core/source.cpp index ccb9142e..8aa824b6 100644 --- a/luprex/cpp/core/source.cpp +++ b/luprex/cpp/core/source.cpp @@ -245,26 +245,24 @@ void SourceDB::update(const util::LuaSourceVec &source) { LS.result(); } -// Delete everything from the global environment except -// the class tables. +// Delete everything from the global environment. +// Clear all the classes in the registry classes table. // static void source_clear_globals(lua_State *L) { - LuaVar classname, classtab, key, globtab; - LuaStack LS(L, classname, classtab, key, globtab); + LuaVar classname, classtab, key, globtab, classes; + LuaStack LS(L, classname, classtab, key, globtab, classes); LS.getglobaltable(globtab); - LS.rawset(globtab, "_G", LuaNil); - LS.set(classname, LuaNil); - while (LS.next(globtab, classname, classtab) != 0) { - if (LS.rawequal(globtab, classtab)) { - LS.rawset(globtab, classname, LuaNil); - } else if (LS.istable(classtab)) { - LS.cleartable(classtab, true); - } else { - LS.rawset(globtab, classname, LuaNil); - } - } + LS.cleartable(globtab, true); LS.rawset(globtab, "_G", globtab); + + LS.rawget(classes, LuaRegistry, "classes"); + assert(LS.istable(classes)); + LS.set(classname, LuaNil); + while (LS.next(classes, classname, classtab) != 0) { + assert(LS.istable(classtab)); + LS.cleartable(classtab, true); + } LS.result(); } diff --git a/luprex/cpp/core/world-core.cpp b/luprex/cpp/core/world-core.cpp index 62c5dd1b..11866fc1 100644 --- a/luprex/cpp/core/world-core.cpp +++ b/luprex/cpp/core/world-core.cpp @@ -60,9 +60,6 @@ World::World(WorldType wt) { LS.getglobaltable(globtab); LS.settabletype(globtab, LUA_TT_GLOBALENV); - // Create the tangibles table in the registry. - LS.rawset(LuaRegistry, "tangibles", LuaNewTable); - // Store the world type in the registry. LS.rawset(LuaRegistry, "worldtype", wt); @@ -592,7 +589,7 @@ void World::invoke_flush_prints(int64_t actor_id, int64_t place_id, const eng::s void World::invoke_lua(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data) { assert(stack_is_clear()); - // Make sure that actor and place exist. + // Make sure that actor and place exist and are not stubs. Tangible *tactor = tangible_get(actor_id); Tangible *tplace = tangible_get(place_id); if ((tactor == nullptr) || (tplace == nullptr)) {