Merge branch 'main' of https://github.com/jyelon/luprex into main

luprex/core/Makefile

@@ -77,6 +77,7 @@ CORE_OBJ_FILES=\
 	obj/idalloc.o\
 	obj/globaldb.o\
 	obj/sched.o\
+	obj/http.o\
 	obj/table.o\
 	obj/gui.o\
 	obj/luasnap.o\

luprex/core/cpp/animqueue.cpp

@@ -1,6 +1,6 @@
 #include "wrap-sstream.hpp"
 #include "wrap-map.hpp"
-
+#include "util.hpp"
 #include "animqueue.hpp"
 #include "luastack.hpp"
 #include "streambuffer.hpp"
@@ -297,25 +297,25 @@ bool AnimStep::from_string(const eng::string &config) {
         if (key == "action") {
             action_ = val;
         } else if (key == "id") {
-            int64_t id = util::strtoint(val, -1);
+            int64_t id = sv::to_int64(val, -1);
             if (id < 0) return false;
             id_ = id;
         } else if (key == "plane") {
             set_plane(val);
         } else if (key == "x") {
-            double v = util::strtodouble(val);
+            double v = sv::to_double(val);
             if (std::isnan(v)) return false;
             set_x(v);
         } else if (key == "y") {
-            double v = util::strtodouble(val);
+            double v = sv::to_double(val);
             if (std::isnan(v)) return false;
             set_y(v);
         } else if (key == "z") {
-            double v = util::strtodouble(val);
+            double v = sv::to_double(val);
             if (std::isnan(v)) return false;
             set_z(v);
         } else if (key == "facing") {
-            double v = util::strtodouble(val);
+            double v = sv::to_double(val);
             if (std::isnan(v)) return false;
             set_facing(v);
         } else if (key == "graphic") {

luprex/core/cpp/drivenengine.cpp

@@ -1,12 +1,12 @@
 #include "wrap-string.hpp"
 #include "wrap-vector.hpp"
+#include "util.hpp"
+#include "drivenengine.hpp"
 
 #include <utility>
 #include <iostream>
 #include <cstring>
 
-#include "drivenengine.hpp"
-
 DrivenEngineReg *DrivenEngineReg::All;
 
 DrivenEngineReg::DrivenEngineReg(const char *n, DrivenEngineMaker fn) {
@@ -66,7 +66,7 @@ void Channel::echo_command() {
     }
 
     // Find out how much of the command matches.
-    int match = util::common_prefix_length(current_command_, desired_command_);
+    int match = sv::common_prefix_length(current_command_, desired_command_);
 
     // Echo backspaces to remove the non-matching part.
     int remove = current_command_.size() - match;

luprex/core/cpp/driver-ssl.cpp

@@ -63,26 +63,31 @@ const char *dummy_key =
 
 std::string errors_string(bool lastonly) {
     std::string err;
-    const char *file, *data, *func;
+    const char *file, *data;
     int line, flags;
+    // const char *func;
 
     while (true) {
+        // Newer versions of the SSL API support this.
+        // unsigned long code =
+        //    ERR_get_error_all(&file, &line, &func, &data, &flags);
+        // Older versions of the SSL API support this.
         unsigned long code =
-            ERR_get_error_all(&file, &line, &func, &data, &flags);
+            ERR_get_error_line_data(&file, &line, &data, &flags);
         if (code == 0) break;
         std::string reason;
-        if (ERR_SYSTEM_ERROR(code)) {
-            reason = strerror_str(ERR_GET_REASON(code));
+        const char *rc = ERR_reason_error_string(code);
+        if (rc != nullptr) {
+            reason = rc;
         } else {
-            const char *rc = ERR_reason_error_string(code);
-            reason = (rc == nullptr) ? "unknown" : rc;
+            reason = "sys:" + strerror_str(ERR_GET_REASON(code));
         }
         if (err.empty() || lastonly) {
             err = reason;
         } else {
             err = err + ", " + reason;
         }
-        if (data != nullptr) {
+        if ((data != nullptr) && (data[0] != 0)) {
             err = err + " " + data;
         }
     }

luprex/core/cpp/driver-util.cpp

@@ -1,17 +1,16 @@
 
 #include "wrap-string.hpp"
 #include "wrap-vector.hpp"
+#include "util.hpp"
+#include "spookyv2.hpp"
+#include "driver-util.hpp"
+#include "luastack.hpp"
 
 #include <string_view>
 #include <fstream>
 #include <ios>
 #include <iostream>
 
-#include "spookyv2.hpp"
-#include "driver-util.hpp"
-#include "luastack.hpp"
-#include "util.hpp"
-
 #define RLOG_BUFSIZE (1024 * 1024)
 #define MAX_ARGC 1024
 
@@ -47,8 +46,8 @@ void split_target(std::string_view target, std::string &cert, std::string &host,
 std::vector<std::string> parse_control_lst(std::string_view ctrl) {
     std::vector<std::string> result;
     while (!ctrl.empty()) {
-        std::string_view line = util::sv_read_line(ctrl);
-        std::string_view trimmed = util::sv_trim(line);
+        std::string_view line = sv::read_to_line(ctrl);
+        std::string_view trimmed = sv::trim(line);
         if ((trimmed.size() > 0) && (trimmed[0] != '#')) {
             result.emplace_back(trimmed);
         }

luprex/core/cpp/eng-malloc.hpp

@@ -142,6 +142,9 @@ namespace eng {
 
 // eng::nevernew.  A class containing private operator new and
 // operator delete, making it impossible to 'new' the class.
+// This means the class must be embedded as a field in some other
+// class, and it gets allocated when its enclosing object gets
+// allocated.
 namespace eng {
     class nevernew {
     private:

luprex/core/cpp/globaldb.cpp

@@ -45,14 +45,15 @@ LuaDefine(global_table, "globalname", "get a table where global data can be stor
     LuaRet globaltab;
     LuaVar globaldb;
     LuaStack LS(L, globalname, globaltab, globaldb);
+    LS.checkstring(globalname);
 
     // Get a pointer to the globaldb.
     LS.rawget(globaldb, LuaRegistry, "globaldb");
-    if (!LS.istable(globaldb)) {
-        return lua_yield(L, 0); // donotpredict
-    }
 
-    LS.checkstring(globalname);
+    // nopredict
+    if (lua_isyieldable(L) && (!LS.istable(globaldb))) {
+        return lua_yield(L, 0);
+    }
 
     // Get the globaltab from the globaldb, sanity check it.
     LS.rawget(globaltab, globaldb, globalname);

luprex/core/cpp/gui.cpp

@@ -1,7 +1,7 @@
 #include "wrap-string.hpp"
 #include "wrap-map.hpp"
 #include "wrap-vector.hpp"
-
+#include "util.hpp"
 #include "gui.hpp"
 #include "luastack.hpp"
 
@@ -39,8 +39,8 @@ bool Gui::has_action(const eng::string &action) const {
     return false;
 }
 
-eng::string Gui::get_action(int index) {
-    if ((index < 0) || (index >= int(elts_.size()))) {
+eng::string Gui::get_action(int64_t index) {
+    if ((index < 0) || (index >= int64_t(elts_.size()))) {
         return "";
     }
     return elts_[index].action();
@@ -62,7 +62,7 @@ LuaDefine(gui_menu_item, "action,label", "add a menu item to the current gui") {
     LuaStack LS(L, laction, llabel);
     eng::string action = LS.ckstring(laction);
     eng::string label = LS.ckstring(llabel);
-    if (!util::has_prefix(action, "cb_")) {
+    if (!sv::has_prefix(action, "cb_")) {
         luaL_error(L, "menuitem callbacks must start with cb_");
     }
     gui->menu_item(action, label);

luprex/core/cpp/gui.hpp

@@ -40,7 +40,7 @@ public:
     void clear(int64_t p) { place_ = p; elts_.clear(); }
     bool has_action(const eng::string &action) const;
     void menu_item(const eng::string &action, const eng::string &label);
-    eng::string get_action(int index);
+    eng::string get_action(int64_t index);
     eng::string menu_debug_string() const;
 
     // Put a pointer to a gui into the lua registry.

luprex/core/cpp/http.cpp

@@ -1,26 +1,987 @@
+//
+// Things to worry about:
+//   Expect: 100-Continue
 
+
+
+#include "http.hpp"
 #include "wrap-sstream.hpp"
 #include "wrap-string.hpp"
+#include "util.hpp"
+#include "luastack.hpp"
 
-static eng::string url_encode(const eng::string &value) {
-    eng::ostringstream escaped;
-    escaped.fill('0');
-    escaped << hex;
+#include <cstdint>
 
+using string_view = std::string_view;
+
+bool words_separated_by_dashes(string_view v) {
+    while (true) {
+        if (!sv::ascii_isalpha(sv::zfront(v))) return false;
+        v.remove_prefix(1);
+        while (sv::ascii_isalnum(sv::zfront(v))) v.remove_prefix(1);
+        if (v.empty()) return true;
+        if (sv::zfront(v) != '-') return false;
+        v.remove_prefix(1);
+    }
+}
+
+// Technically, this is a true, correct URL encode routine.
+static eng::string url_encode_param(string_view value) {
+    eng::ostringstream result;
+    const char *hexdigits = "0123456789ABCDEF";
     for (int i = 0; i < int(value.size()); i++) {
         char c = value[i];
 
-        // Keep alphanumeric and other accepted characters intact
-        // Any other characters are percent-encoded
-        if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
-            escaped << c;
+        if (sv::ascii_isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
+            result << c;
+        } else if (c == ' ') {
+            result << '+';
         } else {
-            escaped << std::uppercase;
-            escaped << '%' << std::setw(2) << int((unsigned char) c);
-            escaped << std::nouppercase;
+            result << '%' << hexdigits[c>>4] << hexdigits[c&15];
+        }
+    }
+    return result.str();
+}
+
+// This URL encode routine leaves slashes intact.  That's not
+// technically correct, but it's really what you want for paths.
+static eng::string url_encode_path(string_view value) {
+    eng::ostringstream result;
+    const char *hexdigits = "0123456789ABCDEF";
+    for (int i = 0; i < int(value.size()); i++) {
+        char c = value[i];
+
+        if (sv::ascii_isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' || c == '/') {
+            result << c;
+        } else if (c == ' ') {
+            result << '+';
+        } else {
+            result << '%' << hexdigits[c>>4] << hexdigits[c&15];
+        }
+    }
+    return result.str();
+}
+
+static eng::string url_decode(string_view eurl) {
+    eng::ostringstream result;
+    int i = 0;
+    int len = eurl.size();
+    while (i < len) {
+        char c = eurl[i];
+        if (c == '+') {
+            result << ' ';
+            i += 1;
+        } else if ((c == '%') && (i + 2 < len)) {
+            std::string_view code = eurl.substr(i + 1, 2);
+            uint64_t value = sv::to_hex64(code);
+            if (value > 255) {
+                result << '?';
+            } else {
+                result << char(value);
+            }
+            i += 3;
+        } else {
+            result << c;
+            i += 1;
+        }
+    }
+    return result.str();
+}
+
+static void send_encoded_path(std::string_view path, const UrlParameters &params, StreamBuffer *sb) {
+    sb->write_bytes(url_encode_path(path));
+    bool first_param = true;
+    for (const auto &pair : params) {
+        sb->write_char(first_param ? '?' : '&');
+        sb->write_bytes(url_encode_param(pair.first));
+        sb->write_char('=');
+        sb->write_bytes(url_encode_param(pair.second));
+        first_param = false;
+    }
+}
+
+static void send_host_and_port(std::string_view host, int port, StreamBuffer *sb) {
+    sb->write_bytes(host);
+    if (port != 0) {
+        sb->write_char(':');
+        sb->ostream() << port;
+    }
+}
+
+// In a properly-formed url, the hostname and path are url encoded.
+// This parser expects an encoded URL.
+
+struct ParsedURL {
+public:
+    bool valid;
+    eng::string proto;
+    eng::string host;
+    int port;
+    eng::string path;
+    UrlParameters params;
+
+public:
+    void clear() {
+        valid = false;
+        proto.clear();
+        host.clear();
+        port = 0;
+        path.clear();
+        params.clear();
+    }
+
+    eng::string str() {
+        StreamBuffer sb;
+        sb.write_bytes(proto);
+        sb.write_bytes("://");
+        send_host_and_port(host, port, &sb);
+        send_encoded_path(path, params, &sb);
+        return eng::string(sb.view());
+    }
+
+    ParsedURL(std::string_view url) {
+        clear();
+
+        proto = util::ascii_tolower(sv::read_to_sep(url, ':'));
+        if (!sv::has_prefix(url, "//")) { clear(); return; }
+        url.remove_prefix(2);
+        if (!words_separated_by_dashes(proto)) { clear(); return; }
+        
+        // Extract the host and port as a single string.
+        string_view turl = url;
+        string_view hostport = sv::read_to_sep(turl, '/');
+        url.remove_prefix(hostport.size());
+
+        // Split the host and port from each other and parse them.
+        host = util::ascii_tolower(sv::read_to_sep(hostport, ':'));
+        if (host.empty()) { clear(); return; }
+        if (!hostport.empty()) {
+            int64_t iport = sv::to_int64(hostport);
+            if ((iport < 1) || (iport > 65535)) {
+                clear(); return;
+            }
+            port = iport;
+        }
+
+        // Split off the path.
+        path = url_decode(sv::read_to_sep(url, '?'));
+        if (path.empty()) {
+            path = "/";
+        }
+
+        // Process url parameters.
+        while (!sv::isnull(url)) {
+            std::string_view keyval = sv::read_to_sep(url, '&');
+            if (keyval.empty()) { clear(); return; }
+            std::string_view key = sv::read_to_sep(keyval, '=');
+            if (key.empty()) { clear(); return; }
+            if (sv::isnull(keyval)) { clear(); return; }
+            eng::string dkey = url_decode(key);
+            eng::string dval = url_decode(keyval);
+            params[dkey] = dval;
+        }
+
+        // If we made it here, we have a valid URL
+        valid = true;
+    }
+};
+
+HttpClientRequest::HttpClientRequest() {
+    verify_certificate_ = false;
+    port_ = 0;
+    request_id_ = 0;
+    place_id_ = 0;
+    thread_id_ = 0;
+}
+
+void HttpClientRequest::fail(string_view s) {
+    if (error_.empty()) {
+        error_ = s;
+    }
+}
+
+eng::string HttpClientRequest::target() const {
+    assert(check().empty());
+    eng::ostringstream oss;
+    oss << (verify_certificate_ ? "cert" : "nocert");
+    oss << ':' << host_ << ':' << port_;
+    return oss.str();
+}
+
+void HttpClientRequest::set_verify_certificate(bool flag) {
+    verify_certificate_ = flag;
+}
+
+void HttpClientRequest::set_method(const eng::string &s) {
+    eng::string method = util::ascii_toupper(s);
+    if ((method != "GET") && (method != "HEAD")) {
+        fail(util::ss("HTTP method not implemented: ", method, ".",
+                    "Currently, only HEAD and GET are implemented."));
+        return;
+    }
+    if ((!method_.empty()) && (method_ != method)) {
+        fail(util::ss("HTTP method specified twice: ", method_, " and ", method));
+        return;
+    }
+    method_ = method;
+}
+
+void HttpClientRequest::set_host(const eng::string &s) {
+    eng::string host = util::ascii_tolower(s);
+    if (host.empty()) {
+        fail(util::ss("HTTP hostname cannot be empty string."));
+        return;
+    }
+    // This is not quite strict, but it's close.  I believe
+    // the DNS lookup will fail for invalid hostnames anyway.
+    for (char c : host) {
+        if ((c != '-') && (c != '.') && (!sv::ascii_isalnum(c))) {
+            fail(util::ss("HTTP hostnames can only contain letters, digits, and hyphen: ", host));
+            return;
+        }
+    }
+    if (!host_.empty()) {
+        fail(util::ss("HTTP hostname specified twice: ", host_, " and ", host));
+        return;
+    }
+    host_ = host;
+}
+
+void HttpClientRequest::set_port(int port) {
+    if ((port < 1) || (port > 65535)) {
+        fail(util::ss("HTTP port must be between 1 and 65535: ", port));
+        return;
+    }
+    if (port_ != 0) {
+        fail(util::ss("HTTP port specified twice: ", port_, " and ", port));
+        return;
+    }
+    port_ = port;
+}
+
+void HttpClientRequest::set_path(string_view path) {
+    if (!sv::has_prefix(path, "/")) {
+        fail(util::ss("HTTP path must start with slash"));
+        return;
+    }
+    if (!path_.empty()) {
+        fail(util::ss("HTTP path specified twice: ", path_, " and ", path));
+        return;
+    }
+    path_ = path;
+}
+
+void HttpClientRequest::set_param(const eng::string &key, const eng::string &val) {
+    if (params_.find(key) != params_.end()) {
+        fail(util::ss("HTTP url parameter specified twice: ", key));
+        return;
+    }
+    if (key.empty()) {
+        fail(util::ss("HTTP parameter key cannot be empty"));
+        return;
+    }
+    params_[key] = val;
+}
+
+void HttpClientRequest::set_url(string_view url) {
+    ParsedURL parsed_url(url);
+    if (!parsed_url.valid) {
+        fail(util::ss("syntactically invalid URL: ", url));
+        return;
+    }
+    if (parsed_url.proto != "https") {
+        fail(util::ss("unsupported protocol: ", parsed_url.proto));
+        return;
+    }
+    set_host(parsed_url.host);
+    if (parsed_url.port) set_port(parsed_url.port);
+    set_path(parsed_url.path);
+    for (const auto &pair : parsed_url.params) {
+        set_param(pair.first, pair.second);
+    }
+}
+
+void HttpClientRequest::set_verify_certificate(LuaStack &LS, LuaSlot val) {
+    if (!LS.isboolean(val)) {
+        fail(util::ss("HTTP verify_certificate must be a boolean"));
+        return;
+    }
+    set_verify_certificate(LS.ckboolean(val));
+}
+
+void HttpClientRequest::set_method(LuaStack &LS, LuaSlot val) {
+    if (!LS.isstring(val)) {
+        fail(util::ss("HTTP method must be a string"));
+        return;
+    }
+    set_method(LS.ckstring(val));
+}
+
+void HttpClientRequest::set_host(LuaStack &LS, LuaSlot val) {
+    if (!LS.isstring(val)) {
+        fail(util::ss("HTTP host must be a string"));
+        return;
+    }
+    set_host(LS.ckstring(val));
+}
+
+void HttpClientRequest::set_port(LuaStack &LS, LuaSlot val) {
+    if (!LS.isint(val)) {
+        fail(util::ss("HTTP port must be an int"));
+        return;
+    }
+    set_port(LS.ckint(val));
+}
+
+void HttpClientRequest::set_path(LuaStack &LS, LuaSlot val) {
+    if (!LS.isstring(val)) {
+        fail(util::ss("HTTP path must be a string"));
+        return;
+    }
+    set_path(LS.ckstring(val));
+}
+
+void HttpClientRequest::set_param(LuaStack &LS, LuaSlot key, LuaSlot val) {
+    if (!LS.isstring(key)) {
+        fail(util::ss("HTTP url parameter key must be a string"));
+        return;
+    }
+    if (!LS.isstring(val)) {
+        fail(util::ss("HTTP url parameter val must be a string"));
+        return;
+    }
+    set_param(LS.ckstring(key), LS.ckstring(val));
+}
+
+void HttpClientRequest::set_params(LuaStack &LS0, LuaSlot tab) {
+    if (!LS0.istable(tab)) {
+        fail(util::ss("HTTP params must be a table"));
+        return;
+    }
+    LuaVar key, val;
+    LuaStack LS(LS0.state(), key, val);
+    LS.set(key, LuaNil);
+    while (LS.next(tab, key, val)) {
+        set_param(LS, key, val);
+    }
+}
+
+void HttpClientRequest::set_url(LuaStack &LS, LuaSlot val) {
+    if (!LS.isstring(val)) {
+        fail(util::ss("HTTP url must be a string"));
+        return;
+    }
+    set_url(LS.ckstring(val));
+}
+
+void HttpClientRequest::set_defaults() {
+    if (method_.empty()) {
+        method_ = "GET";
+    }
+    if (port_ == 0) {
+        port_ = 443;
+    }
+}
+
+void HttpClientRequest::set_config(LuaStack &LS0, LuaSlot tab) {
+    LuaVar key, val;
+    LuaStack LS(LS0.state(), key, val);
+    LS.set(key, LuaNil);
+    while (LS.next(tab, key, val)) {
+        eng::string kstr;
+        if (LS.isstring(key)) kstr = LS.ckstring(key);
+        if (kstr == "method") {
+            set_method(LS, val);
+        } else if (kstr == "host") {
+            set_host(LS, val);
+        } else if (kstr == "port") {
+            set_port(LS, val);
+        } else if (kstr == "path") {
+            set_path(LS, val);
+        } else if (kstr == "params") {
+            set_params(LS, val);
+        } else if (kstr == "url") {
+            set_url(LS, val);
+        } else if (kstr == "verifycertificate") {
+            set_verify_certificate(LS, val);
+        } else if (kstr == "") {
+            fail(util::ss("HTTP config parameter names must be strings."));
+        } else {
+            fail(util::ss("HTTP unrecognized config parameter: ", kstr));
+        }
+    }
+}
+
+eng::string HttpClientRequest::check() const {
+    if (!error_.empty()) {
+        return error_;
+    }
+    if (method_.empty()) {
+        return "HTTP method has not been set";
+    }
+    if (host_.empty()) {
+        return "HTTP host has not been set";
+    }
+    if (port_ == 0) {
+        return "HTTP port has not been set";
+    }
+    if (path_.empty()) {
+        return "HTTP url has not been set";
+    }
+    return "";
+}
+
+
+void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const {
+    // If there's an error in the request, handle it.  In debug string mode,
+    // we just put the error into the output.  In production mode, we assert
+    // fail.
+    eng::string error = check();
+    if (debug_string) {
+        if (!error.empty()) {
+            sb->write_bytes(error);
+            return;
+        }
+    } else {
+        assert(error.empty());
+    }
+
+    // Choose a linebreak.
+    eng::string linebreak = (debug_string) ? "\n" : "\r\n";
+
+    // Send the command.
+    sb->write_bytes(method_);
+    sb->write_char(' ');
+    send_encoded_path(path_, params_, sb);
+    sb->write_bytes(" HTTP/1.1");
+    sb->write_bytes(linebreak);
+
+    // Send the host header.
+    sb->write_bytes("Host: ");
+    send_host_and_port(host_, port_, sb);
+    sb->write_bytes(linebreak);
+
+    // The empty accept-encoding header notifies the
+    // server that we don't support gzip, deflate, or
+    // other content compression.
+    sb->write_bytes("Accept-encoding:");
+    sb->write_bytes(linebreak);
+
+    // Add a user-agent header.  Not sure why.
+    sb->write_bytes("User-agent: Mozilla 5.0 (luprex)");
+    sb->write_bytes(linebreak);
+
+    // Add the requester IDs (debug string only)
+    if (debug_string && ((request_id_ != 0) || (place_id_ != 0) || (thread_id_ != 0))) {
+        sb->write_bytes("X-Requester-Ids: ");
+        sb->ostream() << "rid=" << request_id_ << "; pid=" << place_id_ << "; tid=" << thread_id_;
+        sb->write_bytes(linebreak);
+    }
+
+    // Send the extra linebreak.
+    if (!debug_string) {
+        sb->write_bytes(linebreak);
+    }
+}
+
+void HttpClientRequest::serialize(StreamBuffer *sb) const {
+    sb->write_int64(request_id_);
+    sb->write_int64(place_id_);
+    sb->write_int64(thread_id_);
+    sb->write_string(error_);
+    sb->write_bool(verify_certificate_);
+    sb->write_string(method_);
+    sb->write_string(host_);
+    sb->write_int32(port_);
+    sb->write_string(path_);
+    sb->write_int32(params_.size());
+    for (const auto &pair : params_) {
+        sb->write_string(pair.first);
+        sb->write_string(pair.second);
+    }
+}
+
+void HttpClientRequest::deserialize(StreamBuffer *sb) {
+    request_id_ = sb->read_int64();
+    place_id_ = sb->read_int64();
+    thread_id_ = sb->read_int64();
+    error_ = sb->read_string();
+    verify_certificate_ = sb->read_bool();
+    method_ = sb->read_string();
+    host_ = sb->read_string();
+    port_ = sb->read_int32();
+    path_ = sb->read_string();
+    int32_t nparams = sb->read_int32();
+    params_.clear();
+    for (int i = 0; i < nparams; i++) {
+        eng::string k = sb->read_string();
+        eng::string v = sb->read_string();
+        params_[k] = v;
+    }
+}
+
+eng::string HttpClientRequest::DebugString() {
+    StreamBuffer sb;
+    send_internal(&sb, true);
+    return eng::string(sb.view());
+}
+
+void HttpClientRequestMap::serialize(StreamBuffer *sb) const {
+    sb->write_int32(size());
+    for (const auto &pair : *this) {
+        pair.second.serialize(sb);
+    }
+}
+
+void HttpClientRequestMap::deserialize(StreamBuffer *sb) {
+    int32_t count = sb->read_int32();
+    clear();
+    HttpClientRequest req;
+    for (int i = 0; i < count; i++) {
+        req.deserialize(sb);
+        (*this)[req.request_id()] = req;
+    }
+}
+
+HttpClientResponse::HttpClientResponse() {
+    request_id_ = 0;
+    status_code_ = 0;
+    response_length_ = 0;
+    mime_type_ = "";
+    content_length_ = -1;
+}
+
+eng::string HttpClientResponse::DebugString() const {
+    eng::ostringstream oss;
+    oss << "HttpClientResponse:" << std::endl;
+    oss << "  status_code: " << status_code_ << std::endl;
+    oss << "  error: " << error_ << std::endl;
+    oss << "  content_length: " << content_length_ << std::endl;
+    oss << "  transfer_encoding: " << transfer_encoding_ << std::endl;
+    oss << "  location: " << location_ << std::endl;
+    oss << "  mime_type: " << mime_type_ << std::endl;
+    oss << "  charset: " << charset_ << std::endl;
+    oss << "  content: " << content_ << std::endl;
+    oss << "  response_length: " << response_length_ << std::endl;
+    return oss.str();
+}
+
+void HttpClientResponse::fail(int code, string_view message) {
+    status_code_ = code;
+    error_ = message;
+    mime_type_ = "";
+    charset_ = "";
+    content_ = "";
+}
+
+void HttpClientResponse::incomplete(bool closed) {
+    if (closed) {
+        fail(500, "internal server error: response truncated");
+    } else {
+        fail(0, "response not yet fully received");
+    }
+}
+
+void HttpClientResponse::parse_content_encoding(string_view value) {
+    content_encoding_ = util::ascii_tolower(value);
+}
+
+void HttpClientResponse::parse_content_length(string_view value) {
+    int64_t code = sv::to_int64(value);
+    if ((code < 0) || (code > INT_MAX)) {
+        fail(500, util::ss("internal server error: unparseable content-length: ", value));
+    }
+    content_length_ = code;
+}
+
+void HttpClientResponse::parse_content_type(string_view value) {
+    eng::string ctype = util::ascii_tolower(value);
+    string_view ctview(ctype);
+    mime_type_ = sv::trim(sv::read_to_sep(ctview, ';'));
+    if (mime_type_.empty()) {
+        fail(500, util::ss("internal server error: unparseable content-type: ", value));
+        return;
+    }
+    while (true) {
+        string_view feature = sv::trim(sv::read_to_sep(ctview, ';'));
+        if (feature.empty()) {
+            return;
+        }
+        string_view ftype = sv::trim(sv::read_to_sep(feature, '='));
+        if (ftype == "charset") {
+            charset_ = sv::trim(feature);
+        }
+    }
+}
+
+void HttpClientResponse::parse_location(string_view value) {
+    location_ = url_decode(value);
+}
+
+void HttpClientResponse::parse_transfer_encoding(string_view value) {
+    transfer_encoding_ = util::ascii_tolower(value);
+}
+
+void HttpClientResponse::parse_header(string_view header, string_view value) {
+    if (header == "content-encoding") {
+        parse_content_encoding(value);
+    } else if (header == "content-length") {
+        parse_content_length(value);
+    } else if (header == "content-type") {
+        parse_content_type(value);
+    } else if (header == "location") {
+        parse_location(value);
+    } else if (header == "transfer-encoding") {
+        parse_transfer_encoding(value);
+    } else if (header == "content-range") {
+        fail(406, util::ss("not acceptable: unsupported response header: ", header));
+    }
+}
+
+bool HttpClientResponse::parse_content_basic(std::string_view &view, bool closed) {
+    if (content_length_ >= 0) {
+        if (content_length_ > MAX_CONTENT_LENGTH) {
+            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            return false;
+        }
+        if (int(view.size()) < content_length_) {
+            incomplete(closed);
+            return false;
+        }
+        content_ = sv::read_nbytes(view, content_length_);
+    } else {
+        if (int64_t(view.size()) > MAX_CONTENT_LENGTH) {
+            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            return false;
+        }
+        if (!closed) {
+            incomplete(closed);
+            return false;
+        }
+        content_ = sv::read_nbytes(view, view.size());
+    }
+    return true;
+}
+
+bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool closed) {
+    int64_t total_size = 0;
+    std::vector<string_view> chunks;
+    while (true) {
+        std::string_view chunk_header = sv::trim(sv::read_to_line(view));
+        if (sv::isnull(view)) {
+            incomplete(closed);
+            return false;
+        }
+        int64_t chunk_size = sv::to_hex64(chunk_header, -1);
+        if (chunk_size < 0) {
+            fail(500, "internal server error: unparseable chunk header");
+            return false;
+        }
+        if (chunk_size > MAX_CONTENT_LENGTH) {
+            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            return false;
+        }
+        if (chunk_size == 0) break;
+        total_size += chunk_size;
+        if (total_size > MAX_CONTENT_LENGTH) {
+            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            return false;
+        }
+        std::string_view chunk = sv::read_nbytes(view, chunk_size);
+        if (int64_t(chunk.size()) != chunk_size) {
+            incomplete(closed);
+            return false;
+        }
+        std::string_view newline = sv::read_to_line(view);
+        if (!newline.empty()) {
+            fail(500, "internal server error: corrupted chunk encoding");
+            return false;
+        }
+        if (sv::isnull(view)) {
+            incomplete(closed);
+            return false;
+        }
+        chunks.push_back(chunk);
+    }
+    content_.resize(total_size);
+    size_t offset = 0;
+    for (string_view chunk : chunks) {
+        content_.replace(offset, chunk.size(), chunk);
+        offset += chunk.size();
+    }
+    return true;
+}
+
+void HttpClientResponse::parse(const StreamBuffer *sb, bool closed) {
+    // We're not going to modify the StreamBuffer at all.
+    // Instead, we work entirely on a view.
+    string_view view = sb->view();
+
+    // Get the status line.
+    string_view status = sv::trim(sv::read_to_line(view));
+    if (sv::isnull(view)) {
+        incomplete(closed);
+        return;
+    }
+
+    // Parse the status line.
+    string_view protoversion = sv::read_to_space(status);
+    if (!sv::has_prefix(protoversion, "HTTP/")) {
+        fail(500, util::ss("internal server error: status line appears corrupt"));
+        return;
+    }
+    string_view scode = sv::read_to_space(status);
+    int64_t code = sv::to_int64(scode, 0);
+    if ((code < 100) || (code > 599)) {
+        fail(500, util::ss("internal server error: invalid response code: ", scode));
+        return;
+    }
+    status_code_ = code;
+
+    // Responses outside the range 200-299 are errors,
+    // and therefore must store an error message.
+    if ((code < 200) || (code > 299)) {
+        if (status.empty()) {
+            error_ = util::ss("error code ", code);
+        } else {
+            error_ = status;
         }
     }
 
-    return escaped.str();
+    // Parse the headers.
+    while (true) {
+        string_view header = sv::read_to_line(view);
+        if (sv::isnull(view)) {
+            incomplete(closed);
+            return;
+        }
+        if (header.empty()) {
+            break;
+        }
+        eng::string command = util::ascii_tolower(sv::trim(sv::read_to_sep(header, ':')));
+        if (sv::isnull(header)) {
+            fail(500, util::ss("internal server error: no colon in header line: ", command));
+            return;
+        }
+        if (!words_separated_by_dashes(command)) {
+            fail(500, util::ss("internal server error: invalid header: ", command));
+            return;
+        }
+        parse_header(command, sv::trim(header));
+    }
+
+    // Process the content using the transfer encoding.
+    if (transfer_encoding_ == "") {
+        if (!parse_content_basic(view, closed)) return;
+    } else if (transfer_encoding_ == "chunked") {
+        if (!parse_content_chunked(view, closed)) return;
+    } else {
+        fail(500, util::ss("unsupported transfer-encoding: ", transfer_encoding_));
+        return;
+    }
+
+    // Calculate the response length.
+    response_length_ = sb->fill() - view.size();
+
+    // If it's not a redirect, disallow 'location'.
+    if ((status_code_ < 300) || (status_code_ > 399)) {
+        if (!location_.empty()) {
+            fail(500, util::ss("internal server error: location specified, but result code not 300-399: ", code));
+            return;
+        }
+    }
+
+    // If the server didn't specify content-type, make a guess.
+    if (mime_type_.empty()) {
+        if (sv::valid_utf8(content_)) {
+            mime_type_ = "text/plain";
+            charset_ = "utf-8";
+        } else {
+            mime_type_ = "application/octet-stream";
+            charset_ = "";
+        }
+    }
+
+    // If it's multipart, reject it.
+    if (sv::has_prefix(mime_type_, "multipart/")) {
+        fail(406, "not acceptable: multipart messages not supported");
+        return;
+    }
+
+    // If it's text, demand a reasonable charset.  Otherwise,
+    // ignore the charset.
+    if (sv::has_prefix(mime_type_, "text/")) {
+        if (charset_.empty() || (charset_ == "ascii")) {
+            charset_ = "utf-8";
+        }
+        if (charset_ != "utf-8") {
+            fail(406, util::ss("not acceptable: charset not supported: ", charset_));
+            return;
+        }
+    } else {
+        charset_.clear();
+    }
+
+    // Uncompress the content.
+    if ((content_encoding_ == "") || (content_encoding_ == "identity")) {
+    } else {
+        fail(406, util::ss("not acceptable: content-encoding not supported: ", content_encoding_));
+        return;
+    }
+
+    // If there's an error code, throw out the content.
+    if ((status_code_ < 200) || (status_code_ > 299)) {
+        mime_type_.clear();
+        charset_.clear();
+        content_.clear();
+    }
+}
+
+void HttpClientResponse::store(LuaStack &LS0, LuaSlot tab) const {
+    LuaStack LS(LS0.state());
+
+    LS.newtable(tab);
+    LS.rawset(tab, "responsecode", status_code_);
+    if (!error_.empty()) {
+        LS.rawset(tab, "error", error_);
+    }
+    if (!location_.empty()) {
+        LS.rawset(tab, "location", location_);
+    }
+    if (!mime_type_.empty()) {
+        LS.rawset(tab, "mimetype", mime_type_);
+        LS.rawset(tab, "content", content_);
+    }
+
+    // Debugging fields.  Do not use for lua programming.
+    if (content_length_ >= 0) {
+        LS.rawset(tab, "dbg_contentlength", content_length_);
+    }
+    if (!transfer_encoding_.empty()) {
+        LS.rawset(tab, "dbg_transferencoding", transfer_encoding_);
+    }
+    if (!charset_.empty()) {
+        LS.rawset(tab, "dbg_charset", charset_);
+    }
+    if (response_length_ != 0) {
+        LS.rawset(tab, "dbg_responselength", response_length_);
+    }
+}
+
+void HttpClientResponse::store_fail(LuaStack &LS, LuaSlot tab, int status_code, std::string_view error) {
+    HttpClientResponse response;
+    response.fail(status_code, error);
+    response.store(LS, tab);
+}
+
+LuaDefine(http_fixurl, "url", "validate URL and repair minor flaws in the URL syntax") {
+    LuaArg url;
+    LuaRet fixed;
+    LuaStack LS(L, url, fixed);
+    ParsedURL parsed(LS.ckstring(url));
+    if (!parsed.valid) {
+        luaL_error(L, "invalid URL, not fixable");
+        return LS.result();
+    }
+    LS.set(fixed, parsed.str());
+    return LS.result();
+}
+
+
+LuaDefine(http_request, "request",
+    "|Takes an HTTP request in the form of a lua table."
+    "|The table may contain these fields:"
+    "|"
+    "|  method             (ie, 'GET', 'POST', etc)"
+    "|  host               (ie, 'google.com')"
+    "|  port               (default: 443)"
+    "|  path               (ie, '/index.html')"
+    "|  params             (a table of url parameters)"
+    "|  verifycertificate  (default: true)"
+    "|  url                (ie, 'https://host:port/path.html?a=b&c=d')"
+    "|"
+    "|You can specify url components separately (host, port, path,"
+    "|and params), or you can specify the entire url as a unit. "
+    "|If you specify components, they must not be url-encoded. "
+    "|If you specify the url as a whole, it must already be url-encoded."
+    "|"
+    "|You can omit the port, in which case it defaults to the"
+    "|standard https port.  You can omit verifycertificate, in which"
+    "|case it defaults to true.  You can omit the method if the"
+    "|method is implied by the function you called (eg, 'http.get')."
+    "|"
+    "|Note that unencrypted http is not supported - we only allow https."
+    "|However, you can talk to a server that has a dummy certificate"
+    "|by specifying verifycertificate=false."
+    "|"
+    "|This routine, http.request, returns a debug string for the "
+    "|request.  The debug string looks like the actual http headers"
+    "|that would be sent.") {
+    LuaArg tab;
+    LuaRet str;
+    LuaStack LS(L, tab, str);
+    HttpClientRequest req;
+    req.set_config(LS, tab);
+    req.set_defaults();
+    eng::string error = req.check();
+    if (!error.empty()) {
+        luaL_error(L, "%s", error.c_str());
+        return 0;
+    }
+    LS.set(str, req.DebugString());
+    return LS.result();
+}
+
+LuaDefine(http_response, "response",
+    "|Returns an HTTP response in the form of a lua table."
+    "|The table will contain these important fields:"
+    "|"
+    "|  responsecode - 3-digit HTTP response code."
+    "|  error        - an error message, or nil if no error."
+    "|  content      - on success, the content, as a string."
+    "|  mimetype     - on success, the mime type of the content."
+    "|  location     - for HTTP redirects, the target url."
+    "|"
+    "|If the mimetype is a text mimetype, then the content"
+    "|is automatically converted to utf-8."
+    "|"
+    "|The table may also contain these debugging-only fields."
+    "|"
+    "|  dbg_transferencoding - If there was a Transfer-Encoding header."
+    "|  dbg_contentlength    - If there was a Content-length header."
+    "|  dbg_charset          - Original character set for text mime types."
+    "|  dbg_responselength   - Total bytes in the response."
+    "|"
+    "|None of the dbg fields is needed to understand the response."
+    "|For example, consider dbg_charset.  When text content is"
+    "|passed to lua, the content is automatically converted to utf-8."
+    "|So dbg_charset only tells you what the character set used"
+    "|to be, before it was converted to utf-8."
+    "|"
+    "|If an http routine generates an error, that error will be"
+    "|expressed as a status code.  These locally-generated status"
+    "|codes can be:"
+    "|"
+    "|   400 (bad request)          - the request was malformed"
+    "|   503 (service unavailable)  - dns fail, connect fail, or ssl fail"
+    "|   500 (internal server error)- the response contains invalid HTTP"
+    "|   406 (not acceptable)       - the response used a feature we don't support yet"
+    "|   413 (payload too large)    - we refuse to download something so big"
+    "|   425 (can't resume)         - reloaded a save game with a pending request"
+    "|"
+    "|In this case, the error message will be the stock"
+    "|error message, followed by more information."
+    "|"
+    "|This routine, http.response, generates a response by parsing"
+    "|an actual HTTP response string.  This is for debugging only.") {
+    LuaArg text;
+    LuaRet tab;
+    LuaStack LS(L, text, tab);
+    HttpClientResponse resp;
+    StreamBuffer sb;
+    sb.write_bytes(LS.ckstring(text));
+    resp.parse(&sb, true);
+    resp.store(LS, tab);
+    return LS.result();
 }
 

luprex/core/cpp/http.hpp

@@ -16,26 +16,242 @@
 #include "eng-malloc.hpp"
 #include "wrap-string.hpp"
 #include "wrap-vector.hpp"
+#include "wrap-map.hpp"
 #include "luastack.hpp"
+#include "streambuffer.hpp"
+#include "drivenengine.hpp"
 
-class HttpRequest : public eng::nevernew {
-public:
-    enum Method { INVALID, GET, POST, HEAD };
+using UrlParameters = eng::map<eng::string, eng::string>;
 
-public:
+class HttpClientRequest : public eng::nevernew {    
+private:
+    // Request IDs.
+    int64_t request_id_;
+    int64_t place_id_;
+    int64_t thread_id_;
+    
+    // If the request contains an error, the error
+    // message is stored here.
+    eng::string error_;
+
+    // If true, verify the server's certificate.
+    // True is the default.
+    bool verify_certificate_;
+
+    // Method: GET, HEAD, POST, etc.  Must be all-caps.
+    eng::string method_;
+
+    // The hostname. Not yet url-encoded.
     eng::string host_;
+
+    // Port number.
     int port_;
-    Method method_;
-    eng::string encoded_url_;
-    eng::string content_type_;
+
+    // The path.  Not yet url-encoded.  Can not include URL parameters.
+    eng::string path_;
+    
+    // URL parameters to append to the path.  Not yet url-encoded.
+    UrlParameters params_;
+
+private:
+    void fail(std::string_view error);
+    void send_internal(StreamBuffer *target, bool debug_string) const;
+
+public:
+    // Construct an empty HTTP request.
+    // All of the fields have empty values.
+    HttpClientRequest();
+    
+    // Get request-related fields.
+    //
+    const eng::string &error() const { return error_; }
+    bool verify_certificate() const { return verify_certificate_; }
+    const eng::string &method() const { return method_; }
+    const eng::string &host() const { return host_; }
+    int port() const { return port_; }
+    const eng::string &path() const { return path_; }
+    
+    // Populate an request-related fields one piece at a time.
+    // If you pass an invalid value, or if the field is
+    // already set, the routine will generate an error message
+    // and store it in the error field.  In that case, the set
+    // will not happen.  If there's already an error in the error
+    // field, it will not be overwritten.
+    //
+    void set_verify_certificate(bool flag);
+    void set_method(const eng::string &method);
+    void set_host(const eng::string &host);
+    void set_port(int port);
+    void set_path(std::string_view path);
+    void set_param(const eng::string &key, const eng::string &value);
+    void set_url(std::string_view url);
+ 
+    void set_verify_certificate(LuaStack &LS, LuaSlot val);
+    void set_method(LuaStack &LS, LuaSlot val);
+    void set_host(LuaStack &LS, LuaSlot val);
+    void set_port(LuaStack &LS, LuaSlot val);
+    void set_path(LuaStack &LS, LuaSlot path);
+    void set_param(LuaStack &LS, LuaSlot key, LuaSlot val);
+    void set_params(LuaStack &LS, LuaSlot tab);
+    void set_url(LuaStack &LS, LuaSlot val);
+
+    // Set default values for method and port.
+    // This must be done after setting regular values.
+    void set_defaults();
+
+    // Populate request-related fields from a Lua table.
+    void set_config(LuaStack &LS0, LuaSlot tab);
+
+    // Get or Set the request IDs.
+    // This class does just stores these.
+    int64_t request_id() const { return request_id_; }
+    int64_t place_id() const { return place_id_; }
+    int64_t thread_id() const { return thread_id_; }
+    void set_request_id(int64_t request_id) { request_id_ = request_id; }
+    void set_place_id(int64_t place_id) { place_id_ = place_id; }
+    void set_thread_id(int64_t thread_id) { thread_id_ = thread_id; }
+    
+    // Get the network target, eg, "cert:host:port"
+    eng::string target() const;
+
+    // Verify that the request is error free and that
+    // defaults have been set.
+    eng::string check() const;
+
+    // Put the request into the stream, assuming HTTP/1.1
+    void send(StreamBuffer *target) const { send_internal(target, false); }
+
+    // Serialize and deserialize.
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+
+    // Get the request as a debug string.
+    eng::string DebugString();
 };
 
-class HttpReply : public eng::nevernew {
-public:
-    int error_code_;
-    eng::string error_message_;
+class HttpClientResponse {
+private:
+    // The request ID.
+    int64_t request_id_;
+
+    // The HTTP response status code.
+    int status_code_;
+
+    // If the HTTP response contains an error, the
+    // error message is stored here.  If the HTTP response
+    // is a success such as "200 OK" or "201 Created", this
+    // is the empty string, not "OK" or "Created".
+    eng::string error_;
+
+    // Only if content-length header present, otherwise, -1.
+    int64_t content_length_;
+
+    // If empty, it means there was no transfer-encoding header.
+    eng::string transfer_encoding_;
+
+    // If empty, it means there was no content-encoding header.
+    eng::string content_encoding_;
+
+    // Only if location header present.
+    eng::string location_;
+
+    // MIME type of the content.
     eng::string mime_type_;
+
+    // Charset of the content.  Hopefully utf-8.
+    eng::string charset_;
+
+    // The content as string.
     eng::string content_;
+
+    // The length in bytes of the entire response.
+    // May be zero, which means that the response
+    // was so garbled that we couldn't determine the length.
+    int response_length_;
+    
+private:
+    // Store a message indicating that we haven't received enough
+    // bytes yet.  If the connection is closed and we still haven't
+    // received enough bytes, that's a fatal error.
+    void incomplete(bool closed);
+
+    // Parse a response header.  Most headers are ignored.
+    // If the header contains an error, the error is stored.
+    void parse_header(std::string_view header, std::string_view value);
+
+    // Parse specific headers.
+    // For several headers, all we do is verify that they aren't
+    // invoking unsupported features.
+    void parse_content_encoding(std::string_view value);
+    void parse_content_length(std::string_view value);
+    void parse_content_type(std::string_view value);
+    void parse_location(std::string_view value);
+    void parse_transfer_encoding(std::string_view value);
+
+    // parse the body
+    bool parse_content_basic(std::string_view &view, bool closed);
+    bool parse_content_chunked(std::string_view &view, bool closed);
+public:
+    const int64_t MAX_CONTENT_LENGTH = 1000000;
+
+    // Construct a blank response.
+    HttpClientResponse();
+
+    // Store a result code and an error message, and clear the content.
+    // This is generally used when the client detects an error,
+    // such as a DNS lookup fail, a connection failed, an SSL negotiation
+    // failed, or the like.
+    void fail(int status_code, std::string_view error);
+
+    // Parse the HTTP response.  The closed flag is to be set to true if the
+    // remote has closed the connection.
+    //
+    // If the request is incomplete, generates a status code of zero. In that
+    // case, loading more data from the server might improve the situation.
+    //
+    // Note that the response is not ever removed from the StreamBuffer, which
+    // is always unmodified. If you want to remove the response from the
+    // StreamBuffer, see response_length.
+    //
+    void parse(const StreamBuffer *sb, bool closed);
+
+    // Return true if the response is complete.
+    bool complete() const { return status_code_ != 0; }
+
+    // Get or Set the request ID.
+    // This class does nothing with the request ID, it just stores it.
+    int64_t request_id() const { return request_id_; }
+    void set_request_id(int64_t v) { request_id_ = v; }
+    
+    // Convert the HTTP response to a lua table.
+    void store(LuaStack &LS, LuaSlot tab) const;
+
+    // Convert to a debug string.
+    eng::string DebugString() const;
+
+    // Synthesize an error response and store it in lua.
+    static void store_fail(LuaStack &LS, LuaSlot tab, int status_code, std::string_view error);
 };
 
+class HttpClientRequestMap : public eng::map<int64_t, HttpClientRequest> {
+public:
+    void serialize(StreamBuffer *sb) const;
+    void deserialize(StreamBuffer *sb);
+};
+
+using HttpClientResponseVec = eng::vector<HttpClientResponse>;
+
+// This class associates an HTTP request to an actual communication
+// channel that is executing that request.
+
+class HttpClientChannel {
+public:
+    SharedChannel channel_;
+    int64_t parsed_bytes_;
+
+    HttpClientChannel() : parsed_bytes_(0) {}
+};
+
+using HttpClientChannelMap = eng::map<int64_t, HttpClientChannel>;
+
 #endif // HTTP_HPP

luprex/core/cpp/lpxclient.cpp

@@ -125,13 +125,13 @@ public:
 
     void do_menu_command(const StringVec &cmd) {
         world_to_asynchronous();
-        int64_t place = util::strtoint(cmd[1], actor_id_);
+        int64_t place = sv::to_int64(cmd[1], actor_id_);
         world_->update_gui(actor_id_, place, &gui_);
         stdostream() << gui_.menu_debug_string();
     }
 
     void do_choose_command(const StringVec &cmd) {
-        eng::string action = gui_.get_action(util::strtoint(cmd[1], -1));
+        eng::string action = gui_.get_action(sv::to_int64(cmd[1]));
         if (action == "") {
             stdostream() << "Invalid menu item #" << std::endl;
             return;

luprex/core/cpp/lpxserver.cpp

@@ -20,11 +20,14 @@ using ClientVector = eng::vector<UniqueClient>;
 
 class LpxServer : public DrivenEngine {
 public:
+    using StringVec = LuaConsole::StringVec;
     UniqueWorld master_;
     LuaConsole console_;
     ClientVector clients_;
     PrintChanneler print_channeler_;
+    HttpClientChannelMap http_client_channels_;
     int64_t admin_id_;
+    Gui gui_;
 
 public:
     virtual void event_init(int argc, char *argv[]) {
@@ -61,6 +64,22 @@ public:
         stdostream() << "Syntax Error: " << words[1] << std::endl;
     }
 
+    void do_menu_command(const StringVec &cmd) {
+        int64_t place = sv::to_int64(cmd[1], admin_id_);
+        master_->update_gui(admin_id_, place, &gui_);
+        stdostream() << gui_.menu_debug_string();
+    }
+
+    void do_choose_command(const StringVec &cmd) {
+        eng::string action = gui_.get_action(sv::to_int64(cmd[1]));
+        if (action == "") {
+            stdostream() << "Invalid menu item #" << std::endl;
+            return;
+        }
+        stdostream() << "Invoking plan: " << action << std::endl;
+        master_->invoke(Invocation(Invocation::KIND_PLAN, admin_id_, gui_.place(), action));
+    }
+
     void do_tick_command(const util::StringVec &words) {
         master_->invoke(Invocation(Invocation::KIND_TICK, admin_id_, admin_id_, ""));
     }
@@ -73,14 +92,21 @@ public:
         stop_driver();
     }
 
+    void do_aborthttp_command(const util::StringVec &words) {
+        master_->abort_all_http_requests(425, "http requests aborted from the server command line");
+    }
+
     void do_command(const util::StringVec &words) {
         if (words.empty()) return;
         else if (words[0] == "luainvoke") do_luainvoke_command(words);
         else if (words[0] == "luaprobe") do_luaprobe_command(words);
         else if (words[0] == "syntax") do_syntax_command(words);
+        else if (words[0] == "menu") do_menu_command(words);
+        else if (words[0] == "choose") do_choose_command(words);
         else if (words[0] == "tick") do_tick_command(words);
         else if (words[0] == "cpl") do_cpl_command(words);
         else if (words[0] == "quit") do_quit_command(words);
+        else if (words[0] == "aborthttp") do_aborthttp_command(words);
         else {
             stdostream() << "Unsupported command: " << words[0] << std::endl;
         }
@@ -176,6 +202,41 @@ public:
             while (handle_invocation(client));
         }
         util::remove_nullptrs(clients_);
+
+        // Look for new outgoing HTTP client requests.
+        for (const auto &pair : master_->http_requests()) {
+            const HttpClientRequest &request = pair.second;
+            HttpClientChannel &channel = http_client_channels_[request.request_id()];
+            if (channel.channel_ == nullptr) {
+                channel.channel_ = new_outgoing_channel(request.target());
+                channel.parsed_bytes_ = 0;
+                request.send(channel.channel_->out());
+            }
+        }
+
+        // Maintain existing outgoing HTTP client requests.
+        HttpClientResponseVec http_responses;
+        for (auto &pair : http_client_channels_) {
+            HttpClientChannel &htchan = pair.second;
+            Channel &channel = *htchan.channel_;
+            if (channel.closed() || (channel.in()->fill() > htchan.parsed_bytes_)) {
+                HttpClientResponse response;
+                if (!channel.error().empty()) {
+                    response.fail(503, util::ss("Service Unavailable: ", channel.error()));
+                } else {
+                    htchan.parsed_bytes_ = channel.in()->fill();
+                    response.parse(channel.in(), channel.closed());
+                }
+                if (response.complete()) {
+                    response.set_request_id(pair.first);
+                    http_responses.push_back(response);
+                }
+            }
+        }
+        for (const HttpClientResponse &response : http_responses) {
+            http_client_channels_.erase(response.request_id());
+        }
+        master_->http_responses(http_responses);
     }
 };
 

luprex/core/cpp/luaconsole.cpp

@@ -1,8 +1,7 @@
-#include "eng-malloc.hpp"
-#include "luastack.hpp"
 #include "wrap-string.hpp"
 #include "wrap-vector.hpp"
-
+#include "eng-malloc.hpp"
+#include "luastack.hpp"
 #include "luaconsole.hpp"
 #include "util.hpp"
 
@@ -53,7 +52,7 @@ void LuaConsole::simplify(const StringVec &words) {
     words_ = words;
     if (words.size() == 0) {
         return;
-    } else if (util::validinteger(words[0])) {
+    } else if (sv::valid_int64(words[0])) {
         if (words.size() == 1) {
             words_.clear();
             words_.push_back("choose");
@@ -62,7 +61,7 @@ void LuaConsole::simplify(const StringVec &words) {
             synerr("/choose command takes no arguments");
         }
     } else if (words[0] == "choose") {
-        if ((words.size() == 2)&&(util::validinteger(words[1]))) {
+        if ((words.size() == 2)&&(sv::valid_int64(words[1]))) {
             // OK
         } else {
             synerr("/choose [menu-line-number]");
@@ -74,7 +73,7 @@ void LuaConsole::simplify(const StringVec &words) {
     } else if (words[0] == "menu") {
         if (words.size() == 1) {
             words_.push_back("-");
-        } else if ((words.size() == 2)&&(util::validinteger(words[1]))) {
+        } else if ((words.size() == 2)&&(sv::valid_int64(words[1]))) {
             // OK
         } else {
             synerr("/menu [optional-tangible-id]");
@@ -95,6 +94,10 @@ void LuaConsole::simplify(const StringVec &words) {
         if (words.size() != 1) {
             synerr("/work takes no arguments");
         }
+    } else if (words[0] == "aborthttp") {
+        if (words.size() != 1) {
+            synerr("/aborthttp takes no arguments");
+        }
     } else {
         synerr("unrecognized command");
     }
@@ -126,13 +129,13 @@ void LuaConsole::add(eng::string line) {
     // Translate lua expression, deal with initial prefix.
     eng::string partial;
     eng::string lua_mode;
-    if (util::has_prefix(raw_input_, "?=")) {
+    if (sv::has_prefix(raw_input_, "?=")) {
         lua_mode = "luaprobe";
         partial = eng::string("return ") + raw_input_.substr(2);
-    } else if (util::has_prefix(raw_input_, "?")) {
+    } else if (sv::has_prefix(raw_input_, "?")) {
         lua_mode = "luaprobe";
         partial = raw_input_.substr(1);
-    } else if (util::has_prefix(raw_input_, "=")) {
+    } else if (sv::has_prefix(raw_input_, "=")) {
         lua_mode = "luainvoke";
         partial = eng::string("return ") + raw_input_.substr(1);
     } else {
@@ -157,7 +160,7 @@ void LuaConsole::add(eng::string line) {
         }
     } else {
         words_.push_back(lua_mode);
-        words_.push_back(util::rtrim(partial));
+        words_.emplace_back(sv::rtrim(partial));
         clear_raw_input();
     }
     lua_settop(lua_state_, top);

luprex/core/cpp/luastack.cpp

@@ -52,6 +52,22 @@ lua_State *LuaStack::newstate (lua_Alloc allocf) {
     return L;
 }
 
+bool LuaStack::isinteger(LuaSlot s) const {
+    if (lua_type(L_, s) == LUA_TNUMBER) {
+        lua_Number result = lua_tonumber(L_, s);
+        if (lua_Integer(result) == result) return true;
+    }
+    return false;
+}
+
+bool LuaStack::isint(LuaSlot s) const {
+    if (lua_type(L_, s) == LUA_TNUMBER) {
+        lua_Number result = lua_tonumber(L_, s);
+        if (int(result) == result) return true;
+    }
+    return false;
+}
+
 bool LuaStack::ckboolean(LuaSlot s) const {
     luaL_checktype(L_, s, LUA_TBOOLEAN);
     return lua_toboolean(L_, s) ? true:false;
@@ -59,12 +75,24 @@ bool LuaStack::ckboolean(LuaSlot s) const {
 
 lua_Integer LuaStack::ckinteger(LuaSlot s) const {
     luaL_checktype(L_, s, LUA_TNUMBER);
-    return lua_tointeger(L_, s);
+    lua_Number result = lua_tonumber(L_, s);
+    lua_Integer iresult(result);
+    if (iresult != result) {
+        luaL_error(L_, "not a valid integer");
+        return 0;
+    }
+    return iresult;
 }
 
 int LuaStack::ckint(LuaSlot s) const {
     luaL_checktype(L_, s, LUA_TNUMBER);
-    return (int)lua_tointeger(L_, s);
+    lua_Number result = lua_tonumber(L_, s);
+    int iresult(result);
+    if (iresult != result) {
+        luaL_error(L_, "not a valid int");
+        return 0;
+    }
+    return iresult;
 }
 
 lua_Number LuaStack::cknumber(LuaSlot s) const {

luprex/core/cpp/luastack.hpp

@@ -348,6 +348,8 @@ public:
     bool istable(LuaSlot s) const { return lua_type(L_, s) == LUA_TTABLE; }
     bool isstring(LuaSlot s) const { return lua_type(L_, s) == LUA_TSTRING; }
     bool isnumber(LuaSlot s) const { return lua_type(L_, s) == LUA_TNUMBER; }
+    bool isinteger(LuaSlot s) const;
+    bool isint(LuaSlot s) const;
     bool isthread(LuaSlot s) const { return lua_type(L_, s) == LUA_TTHREAD; }
     bool isfunction(LuaSlot s) const { return lua_type(L_, s) == LUA_TFUNCTION; }
     bool isboolean(LuaSlot s) const { return lua_type(L_, s) == LUA_TBOOLEAN; }

luprex/core/cpp/pprint.cpp

@@ -120,8 +120,8 @@ static void tabify(Inspector &insp, int level) {
 
 static void pprint_r(Inspector &insp, int level, LuaSlot root) {
     lua_checkstack(insp.L, 20);
-    LuaVar idv, pairs, key, val;
-    LuaStack LS(insp.L, idv, pairs, key, val);
+    LuaVar idv, pairs, key, val, nextseq;
+    LuaStack LS(insp.L, idv, pairs, key, val, nextseq);
 
     // If it's anything but a table, use 'atomic_print'.
     if (!LS.istable(root)) {
@@ -175,9 +175,9 @@ static void pprint_r(Inspector &insp, int level, LuaSlot root) {
     }
 
     // State variables.
-    int nextseq = 1;
     bool needcomma = false;
     bool multiline = false;
+    LS.set(nextseq, 1);
 
     // Open the brackets.
     (*insp.stream) << "{";
@@ -190,14 +190,14 @@ static void pprint_r(Inspector &insp, int level, LuaSlot root) {
         LS.rawget(val, pairs, i+1);
         if (needcomma) (*insp.stream) << ",";
         needcomma = true;
-        if (LS.isnumber(key) && (LS.ckint(key) == nextseq)) {
+        if (LS.rawequal(key, nextseq)) {
             (*insp.stream) << " ";
             pprint_r(insp, level + 1, val);
-            nextseq = nextseq + 1;
+            LS.set(nextseq, LS.ckinteger(nextseq) + 1);
         } else {
             multiline = true;
             tabify(insp, level + 1);
-            if (LS.isstring(key) && util::is_identifier(LS.ckstring(key))) {
+            if (LS.isstring(key) && sv::is_lua_id(LS.ckstring(key))) {
                 (*insp.stream) << LS.ckstring(key);
             } else {
                 (*insp.stream) << "[";
@@ -227,7 +227,7 @@ static void pprint_r(Inspector &insp, int level, LuaSlot root) {
     // Close the brackets.
     if (multiline) {
         tabify(insp, level);
-    } else if (nextseq > 1) {
+    } else if (LS.ckinteger(nextseq) > 1) {
         (*insp.stream) << " ";
     }
     (*insp.stream) << "}";
@@ -254,7 +254,7 @@ LuaDefine(string_isidentifier, "str", "return true if the string is a valid lua
     LuaStack LS(L, str, result);
     if (LS.isstring(str)) {
         eng::string s = LS.ckstring(str);
-        LS.set(result, util::is_identifier(s));
+        LS.set(result, sv::is_lua_id(s));
     } else {
         LS.set(result, false);
     }

luprex/core/cpp/sched.cpp

@@ -24,6 +24,8 @@ eng::string SchedEntry::debug_string() const {
 }
 
 void Schedule::add(int64_t clk, int64_t thid, int64_t plid) {
+    assert(plid != 0);
+    assert(thid != 0);
     schedule_.insert(SchedEntry(clk, thid, plid));
 }
 

luprex/core/cpp/source.cpp

@@ -7,7 +7,6 @@
 #include "wrap-map.hpp"
 #include "wrap-set.hpp"
 #include "wrap-sstream.hpp"
-
 #include "util.hpp"
 #include "luastack.hpp"
 #include "traceback.hpp"
@@ -329,9 +328,10 @@ static eng::string source_load_lfunctions(lua_State *L) {
         
         // Call the closure.  If there's an error, collect it.
         lua_pushvalue(L, closure.index());
-        if (traceback_pcall(L, 0, 0) != 0) {
-            lua_replace(L, err.index());
-            errss << LS.ckstring(err);
+        eng::string msg = traceback_pcall(L, 0, 0);
+        if (!msg.empty()) {
+            LS.set(err, msg);
+            errss << msg << std::endl;
         }
     }
     LS.result();
@@ -381,9 +381,10 @@ void SourceDB::run_unittests() {
         LS.rawget(func, unittests, name);
 
         lua_pushvalue(L, func.index());
-        if (traceback_pcall(L, 0, 0) != 0) {
-            lua_replace(L, err.index());
-            std::cerr << LS.ckstring(err);
+        eng::string msg = traceback_pcall(L, 0, 0);
+        if (!msg.empty()) {
+            LS.set(err, msg);
+            std::cerr << msg << std::endl;
             any = true;
         }
     }
@@ -569,7 +570,7 @@ eng::string SourceDB::function_docs(const LuaStack &LS0, LuaSlot fn) {
 
         // Incorporate the function comment.
         int linelo = linehi;
-        while ((linelo > 0) && (util::is_lua_comment(lines[linelo-1]))) linelo -= 1;
+        while ((linelo > 0) && (sv::is_lua_comment(lines[linelo-1]))) linelo -= 1;
         
         // Output the docs.
         eng::ostringstream result;

luprex/core/cpp/streambuffer.cpp

@@ -263,6 +263,12 @@ void StreamBuffer::write_uint64(uint64_t vv) {
     write_cursor_ += 8;
 }
 
+void StreamBuffer::write_char(char c) {
+    make_space(1);
+    write_cursor_[0] = c;
+    write_cursor_ += 1;
+}
+
 void StreamBuffer::write_float(float f) {
     make_space(4);
     memcpy(write_cursor_, &f, 4);
@@ -307,6 +313,13 @@ int64_t StreamBuffer::read_int64() {
     return v;
 }
 
+char StreamBuffer::read_char() {
+    check_available(1);
+    char c = read_cursor_[0];
+    read_cursor_ += 1;
+    return c;
+}
+
 float StreamBuffer::read_float() {
     check_available(4);
     float f;

luprex/core/cpp/streambuffer.hpp

@@ -310,6 +310,7 @@ public:
     void write_uint16(uint64_t v);
     void write_uint32(uint64_t v);
     void write_uint64(uint64_t v);
+    void write_char(char c);
     void write_float(float f);
     void write_double(double d);
 
@@ -325,6 +326,7 @@ public:
     uint16_t read_uint16() { return read_int16(); }
     uint32_t read_uint32() { return read_int32(); }
     uint64_t read_uint64() { return read_int64(); }
+    char read_char();
     float read_float();
     double read_double();
 

luprex/core/cpp/textgame.cpp

@@ -1,7 +1,6 @@
 
 #include "wrap-vector.hpp"
 #include "wrap-string.hpp"
-
 #include "luastack.hpp"
 #include "util.hpp"
 #include "gui.hpp"
@@ -47,13 +46,13 @@ private:
     }
 
     void do_menu_command(const StringVec &cmd) {
-        int64_t place = util::strtoint(cmd[1], actor_id_);
+        int64_t place = sv::to_int64(cmd[1], actor_id_);
         world_->update_gui(actor_id_, place, &gui_);
         stdostream() << gui_.menu_debug_string();
     }
 
     void do_choose_command(const StringVec &cmd) {
-        eng::string action = gui_.get_action(util::strtoint(cmd[1], -1));
+        eng::string action = gui_.get_action(sv::to_int64(cmd[1]));
         if (action == "") {
             stdostream() << "Invalid menu item #" << std::endl;
             return;

luprex/core/cpp/traceback.cpp

@@ -1,6 +1,7 @@
 #include "traceback.hpp"
 
 #include <cstring>
+#include <cassert>
 
 #define TRACEBACK_LEVELS1 12
 #define TRACEBACK_LEVELS2 10
@@ -76,12 +77,21 @@ int traceback_coroutine(lua_State *L) {
 }
 
 
-int traceback_pcall(lua_State *L, int narg, int nret) {
+eng::string traceback_pcall(lua_State *L, int narg, int nret) {
     int status;
     int base = lua_gettop(L) - narg; /* function index */
     lua_pushcfunction(L, traceback_coroutine); /* push traceback function */
     lua_insert(L, base);             /* put it under chunk and args */
     status = lua_pcall(L, narg, nret, base);
     lua_remove(L, base); /* remove traceback function */
-    return status;
+    if (status != LUA_OK) {
+        const char *msg = lua_tostring(L, -1);
+        if ((msg == NULL) || (msg[0] == 0)) {
+            msg = "unknown error";
+        }
+        eng::string result = msg;
+        assert(result != "attempt to yield from outside a coroutine");
+        return result;
+    }
+    return "";
 }

luprex/core/cpp/traceback.hpp

@@ -22,9 +22,12 @@ int traceback_coroutine(lua_State *L);
 
 // traceback_pcall
 //
-// same as lua_pcall, except that it automatically supplies 
-// traceback_coroutine as a message handler.
+// Similar to lua_pcall, except that it automatically supplies 
+// traceback_coroutine as a message handler.  It also automatically
+// returns any error message.  Returns empty string if there's
+// no error.  Also checks for a yield inside a pcall, which is 
+// not allowed, and does an assert-fail in that case.
 //
-int traceback_pcall(lua_State *L, int narg, int nret);
+eng::string traceback_pcall(lua_State *L, int narg, int nret);
 
 #endif // TRACEBACK_HPP

luprex/core/cpp/util.cpp

@@ -1,16 +1,18 @@
 #include "wrap-string.hpp"
 #include "wrap-vector.hpp"
+#include "util.hpp"
+#include "fast-float.hpp"
+
 
 #include <algorithm>
-
-#include "util.hpp"
-
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <iomanip>
 #include <cassert>
 #include <cstdlib>
 #include <cmath>
+#include <charconv>
+
 
 #ifdef WIN32
 #endif
@@ -20,17 +22,131 @@
 #include <unistd.h>
 #endif
 
-namespace util {
+namespace sv {
 
-static bool ascii_isalpha(char c) {
-    return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
+bool valid_int64(string_view value) {
+    int64_t result;
+    const char *last = value.data() + value.size();
+    auto r = std::from_chars(value.data(), last, result, 10);
+    if (r.ec != std::errc()) return false;
+    if (r.ptr != last) return false;
+    return true;
 }
 
-static bool ascii_isdigit(char c) {
-    return ((c >= '0') && (c <= '9'));
+bool valid_hex64(string_view value) {
+    int64_t result;
+    const char *last = value.data() + value.size();
+    auto r = std::from_chars(value.data(), last, result, 16);
+    if (r.ec != std::errc()) return false;
+    if (r.ptr != last) return false;
+    return true;
 }
 
-bool is_identifier(const eng::string &str) {
+bool valid_double(string_view value) {
+    double result;
+    const char *last = value.data() + value.size();
+    auto r = fast_float::from_chars(value.data(), last, result);
+    if (r.ec != std::errc()) return false;
+    if (r.ptr != last) return false;
+    return true;
+}
+
+int64_t to_int64(string_view value, int64_t errval) {
+    int64_t result;
+    const char *last = value.data() + value.size();
+    auto r = std::from_chars(value.data(), last, result, 10);
+    if (r.ec != std::errc()) return errval;
+    if (r.ptr != last) return errval;
+    return result;
+}
+
+uint64_t to_hex64(string_view value, uint64_t errval) {
+    uint64_t result;
+    const char *last = value.data() + value.size();
+    auto r = std::from_chars(value.data(), last, result, 16);
+    if (r.ec != std::errc()) return errval;
+    if (r.ptr != last) return errval;
+    return result;
+}
+
+double to_double(string_view value, double errval) {
+    double result;
+    const char *last = value.data() + value.size();
+    auto r = fast_float::from_chars(value.data(), last, result);
+    if (r.ec != std::errc()) return errval;
+    if (r.ptr != last) return errval;
+    return result;
+}
+
+string_view ltrim(string_view v) {
+    while ((!v.empty()) && (ascii_isspace(v.front()))) {
+        v.remove_prefix(1);
+    }
+    return v;
+}
+
+string_view rtrim(string_view v) {
+    while ((!v.empty()) && (ascii_isspace(v.back()))) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+string_view trim(string_view v) {
+    while ((!v.empty()) && (ascii_isspace(v.front()))) {
+        v.remove_prefix(1);
+    }
+    while ((!v.empty()) && (ascii_isspace(v.back()))) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+string_view ltrim(string_view v, char c) {
+    while ((!v.empty()) && (v.front() == c)) {
+        v.remove_prefix(1);
+    }
+    return v;
+}
+
+string_view rtrim(string_view v, char c) {
+    while ((!v.empty()) && (v.back() == c)) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+string_view trim(string_view v, char c) {
+    while ((!v.empty()) && (v.front() == c)) {
+        v.remove_prefix(1);
+    }
+    while ((!v.empty()) && (v.back() == c)) {
+        v.remove_suffix(1);
+    }
+    return v;
+}
+
+bool has_prefix(string_view s, string_view prefix) {
+    return 0 == s.compare(0, prefix.size(), prefix);
+}
+
+bool has_suffix(string_view s, string_view suffix) {
+    if (s.length() >= suffix.length()) {
+        return (0 == s.compare (s.length() - suffix.length(), suffix.length(), suffix));
+    } else {
+        return false;
+    }
+}
+
+int common_prefix_length(string_view a, string_view b) {
+    int minlen = std::min(a.size(), b.size());
+    for (int i = 0; i < minlen; i++) {
+        if (a[i] != b[i]) return i;
+    }
+    return minlen;
+}
+
+bool is_lua_id(string_view str) {
     if (str.size() == 0) return false;
     char c=str[0];
     if ((!ascii_isalpha(c)) && (c!='_')) return false;
@@ -41,6 +157,143 @@ bool is_identifier(const eng::string &str) {
     return true;
 }
 
+bool is_lua_comment(string_view s) {
+    int start = 0;
+    while ((start < int(s.size())) && ((s[start]==' ') || (s[start]=='\t'))) start++;
+    return s.substr(start, 2) == "--";
+}
+
+string_view read_to_sep(string_view &source, char sep) {
+    size_t pos = source.find(sep);
+    string_view result;
+    if (pos == string_view::npos) {
+        result = source;
+        source = string_view();
+    } else {
+        result = source.substr(0, pos);
+        source = source.substr(pos + 1);
+    }
+    return result;
+}
+
+string_view read_to_line(string_view &source) {
+    size_t pos = source.find('\n');
+    string_view result;
+    if (pos == string_view::npos) {
+        result = source;
+        source = string_view();
+    } else {
+        result = source.substr(0, pos);
+        source = source.substr(pos + 1);
+    }
+    if ((!result.empty()) && (result.back() == '\r')) {
+        result.remove_suffix(1);
+    }
+    return result;
+}
+
+string_view read_to_space(string_view &source) {
+    size_t pos1 = 0;
+    while ((pos1 < source.size()) && (!ascii_isspace(source[pos1]))) {
+        pos1 += 1;
+    }
+    string_view result = source.substr(0, pos1);
+    if (pos1 == source.size()) {
+        source = string_view();
+        return result;
+    }
+    size_t pos2 = pos1 + 1;
+    while ((pos2 < source.size()) && (ascii_isspace(source[pos2]))) {
+        pos2 += 1;
+    }
+    source = source.substr(pos2);
+    return result;
+}
+
+string_view read_nbytes(string_view &source, int nbytes) {
+    if (nbytes < 0) nbytes = 0;
+    if (nbytes > int(source.size())) nbytes = source.size();
+    string_view result = source.substr(0, nbytes);
+    source = source.substr(nbytes);
+    return result;
+}
+
+bool valid_utf8(string_view s)
+{
+    const unsigned char *bytes = (const unsigned char *)s.data();
+    const unsigned char *tail = bytes + s.size();
+    unsigned int codepoint;
+    int seqlen;
+
+    while (bytes < tail) {
+        if ((bytes[0] & 0x80) == 0x00) {
+            // U+0000 to U+007F
+            codepoint = (bytes[0] & 0x7F);
+            seqlen = 1;
+        } else if ((bytes[0] & 0xE0) == 0xC0) {
+            // U+0080 to U+07FF
+            codepoint = (bytes[0] & 0x1F);
+            seqlen = 2;
+        } else if ((bytes[0] & 0xF0) == 0xE0) {
+            // U+0800 to U+FFFF
+            codepoint = (bytes[0] & 0x0F);
+            seqlen = 3;
+        } else if ((bytes[0] & 0xF8) == 0xF0) {
+            // U+10000 to U+10FFFF
+            codepoint = (bytes[0] & 0x07);
+            seqlen = 4;
+        } else {
+            return false;
+        }
+
+        if (bytes + seqlen > tail) {
+            return false;
+        }
+
+        for (int i = 1; i < seqlen; ++i) {
+            if ((bytes[i] & 0xC0) != 0x80) return false;
+            codepoint = (codepoint << 6) | (bytes[i] & 0x3F);
+        }
+
+        if ((codepoint > 0x10FFFF) ||
+            ((codepoint >= 0xD800) && (codepoint <= 0xDFFF)) ||
+            ((codepoint <= 0x007F) && (seqlen != 1)) ||
+            ((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) ||
+            ((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) ||
+            ((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) {
+            return false;
+        }
+
+        bytes += seqlen;
+    }
+    return true;
+}
+
+} // namespace sv
+
+
+namespace util {
+
+eng::string ascii_tolower(std::string_view s) {
+    eng::string mod(s);
+    for (int i = 0; i < int(mod.size()); i++) {
+        if (sv::ascii_isupper(mod[i])) {
+            mod[i] += 'a' - 'A';
+        }
+    }
+    return mod;
+}
+
+eng::string ascii_toupper(std::string_view s) {
+    eng::string mod(s);
+    for (int i = 0; i < int(mod.size()); i++) {
+        if (sv::ascii_islower(mod[i])) {
+            mod[i] += 'A' - 'a';
+        }
+    }
+    return mod;
+}
+
 void quote_string(const eng::string &s, std::ostream *os) {
     bool anysq = false;
     bool anydq = false;
@@ -157,6 +410,10 @@ uint64_t hash_ints(uint64_t a, uint64_t b, uint64_t c, uint64_t d) {
     return h1;
 }
 
+double hash_to_double(uint64_t hash) {
+    return (hash >> (64-53)) * 0x1p-53;
+}
+
 StringVec split(const eng::string &s, char sep) {
     StringVec result;
     int start = 0;
@@ -232,14 +489,6 @@ eng::string repeat_string(const eng::string &a, int n) {
     }
     return result;
 }
-
-int common_prefix_length(const eng::string &a, const eng::string &b) {
-    int minlen = std::min(a.size(), b.size());
-    for (int i = 0; i < minlen; i++) {
-        if (a[i] != b[i]) return i;
-    }
-    return minlen;
-}
     
 eng::string tolower(eng::string input) {
     for (int i = 0; i < int(input.size()); i++) {
@@ -255,117 +504,12 @@ eng::string toupper(eng::string input) {
     return input;
 }
 
-bool has_prefix(const eng::string &s, const eng::string &prefix) {
-    return 0 == s.compare(0, prefix.size(), prefix);
-}
-
-bool has_suffix(const eng::string &s, const eng::string &suffix) {
-    if (s.length() >= suffix.length()) {
-        return (0 == s.compare (s.length() - suffix.length(), suffix.length(), suffix));
-    } else {
-        return false;
-    }
-}
-
-bool validinteger(const eng::string &value) {
-    char *endptr;
-    if (value.size() == 0) return false;
-    strtoll(value.c_str(), &endptr, 10);
-    return (endptr == value.c_str() + value.size());
-}
-
-int64_t strtoint(const eng::string &value, int64_t errval) {
-    char *endptr;
-    if (value.size() == 0) return errval;
-    int64_t result = strtoll(value.c_str(), &endptr, 10);
-    if (endptr == value.c_str() + value.size()) {
-        return result;
-    } else {
-        return errval;
-    }
-}
-
-double strtodouble(const eng::string &value) {
-    char *endptr;
-    if (value.size() == 0) return std::nan("");
-    double result = strtod(value.c_str(), &endptr);
-    if (endptr == value.c_str() + value.size()) {
-        return result;
-    } else {
-        return std::nan("");
-    }
-}
-
-std::string_view sv_ltrim(std::string_view v) {
-    const char *b = v.data();
-    const char *e = v.data() + v.size();
-    while ((e > b) && (std::isspace(b[0]))) {
-        b++;
-    }
-    return std::string_view(b, e-b);
-}
-
-std::string_view sv_rtrim(std::string_view v) {
-    const char *b = v.data();
-    const char *e = v.data() + v.size();
-    while ((e > b) && (std::isspace(e[-1]))) {
-        e--;
-    }
-    return std::string_view(b, e-b);
-}
-
-std::string_view sv_trim(std::string_view v) {
-    const char *b = v.data();
-    const char *e = v.data() + v.size();
-    while ((e > b) && (std::isspace(b[0]))) {
-        b++;
-    }
-    while ((e > b) && (std::isspace(e[-1]))) {
-        e--;
-    }
-    return std::string_view(b, e-b);
-}
-
-eng::string ltrim(std::string_view v) {
-    return eng::string(sv_ltrim(v));
-}
-
-eng::string rtrim(std::string_view v) {
-    return eng::string(sv_rtrim(v));
-}
-
-eng::string trim(std::string_view v) {
-    return eng::string(sv_trim(v));
-}
-
-std::string_view sv_read_line(std::string_view &source) {
-    size_t pos = source.find('\n');
-    std::string_view result;
-    if (pos == std::string_view::npos) {
-        result = source;
-        source = "";
-    } else {
-        result = source.substr(0, pos);
-        source = source.substr(pos + 1);
-    }
-    int fsize = result.size();
-    if ((fsize >= 1) && (result[fsize - 1] == '\r')) {
-        result.remove_suffix(1);
-    }
-    return result;
-}
-
-
 double distance_squared(double x1, double y1, double x2, double y2) {
     double dx = x1 - x2;
     double dy = y1 - y2;
     return dx*dx + dy*dy;
 }
 
-bool world_type_authoritative(util::WorldType wt) {
-    return (wt == WORLD_TYPE_MASTER) || (wt == WORLD_TYPE_STANDALONE);
-}
-
 LuaSourcePtr make_lua_source(const eng::string &code) {
     LuaSourcePtr result(new LuaSourceVec);
     eng::string fn = "file.lua";
@@ -373,18 +517,13 @@ LuaSourcePtr make_lua_source(const eng::string &code) {
     return result;
 }
 
-bool is_lua_comment(const eng::string &s) {
-    int start = 0;
-    while ((start < int(s.size())) && ((s[start]==' ') || (s[start]=='\t'))) start++;
-    return s.substr(start, 2) == "--";
-}
-
 eng::string XYZ::debug_string() const {
     eng::ostringstream oss;
     oss << "(" << x << "," << y << "," << z << ")";
     return oss.str();
 }
 
+
 } // namespace util
 
 std::ostream &operator<<(std::ostream &oss, const util::hex64 &v) {
@@ -408,6 +547,57 @@ std::ostream &operator<<(std::ostream &oss, const util::hex8 &v) {
 }
 
 LuaDefine(unittests_util, "", "some unit tests") {
+        // test str_to_int64, str_to_double
+    LuaAssert(L, sv::to_int64("123") == 123);
+    LuaAssert(L, sv::to_int64("123.4") == INT64_MIN);
+    LuaAssert(L, sv::to_int64("12ab") == INT64_MIN);
+    LuaAssert(L, sv::to_int64("") == INT64_MIN);
+    LuaAssert(L, sv::to_double("123.5") == 123.5);
+    LuaAssert(L, std::isnan(sv::to_double("12ab")));
+    LuaAssert(L, std::isnan(sv::to_double("")));
+    
+    // Test trim, ltrim, rtrim
+    LuaAssert(L, sv::ltrim("  foo  ") == "foo  ");
+    LuaAssert(L, sv::rtrim("  foo  ") == "  foo");
+    LuaAssert(L, sv::trim("  foo  ") == "foo");
+    LuaAssert(L, sv::trim("foo") == "foo");
+    LuaAssert(L, sv::trim("") == "");
+    LuaAssert(L, sv::ltrim("**foo**", '*') == "foo**");
+    LuaAssert(L, sv::rtrim("**foo**", '*') == "**foo");
+    LuaAssert(L, sv::trim("**foo**", '*') == "foo");
+    LuaAssert(L, sv::trim("foo", '*') == "foo");
+    LuaAssert(L, sv::trim("", '*') == "");
+    
+    // Test read_to_line
+    std::string_view v = "foo\nbar\r\nbaz";
+    
+    std::string_view v1 = sv::read_to_line(v);
+    LuaAssertStrEq(L, v1, "foo");
+    LuaAssertStrEq(L, v, "bar\r\nbaz");
+
+    std::string_view v2 = sv::read_to_line(v);
+    LuaAssertStrEq(L, v2, "bar");
+    LuaAssertStrEq(L, v, "baz");
+
+    std::string_view v3 = sv::read_to_line(v);
+    LuaAssertStrEq(L, v3, "baz");
+    LuaAssert(L, sv::isnull(v));
+
+    // Test read_to_space
+    v = "foo  bar  baz";
+
+    std::string_view s1 = sv::read_to_space(v);
+    LuaAssertStrEq(L, s1, "foo");
+    LuaAssertStrEq(L, v, "bar  baz");
+
+    std::string_view s2 = sv::read_to_space(v);
+    LuaAssertStrEq(L, s2, "bar");
+    LuaAssertStrEq(L, v, "baz");
+
+    std::string_view s3 = sv::read_to_space(v);
+    LuaAssertStrEq(L, s3, "baz");
+    LuaAssert(L, sv::isnull(v));
+
     // Test the unioning of ID vectors.
     util::IdVector idv1,idv2;
     idv1.push_back(1);
@@ -452,35 +642,6 @@ LuaDefine(unittests_util, "", "some unit tests") {
     LuaAssert(L, util::toupper("fooBar") == "FOOBAR");
     LuaAssert(L, util::tolower("fooBar") == "foobar");
 
-    // test validinteger, strtoint, strtodouble
-    LuaAssert(L, util::validinteger("123") == true);
-    LuaAssert(L, util::validinteger("123.4") == false);
-    LuaAssert(L, util::validinteger("12ab") == false);
-    LuaAssert(L, util::validinteger("") == false);
-    LuaAssert(L, util::strtoint("123", -5) == 123);
-    LuaAssert(L, util::strtoint("123.4", -5) == -5);
-    LuaAssert(L, util::strtoint("12ab", -5) == -5);
-    LuaAssert(L, util::strtoint("", -5) == -5);
-    LuaAssert(L, util::strtodouble("123.5") == 123.5);
-    LuaAssert(L, std::isnan(util::strtodouble("12ab")));
-    LuaAssert(L, std::isnan(util::strtodouble("")));
-    
-    // Test trim, ltrim, rtrim
-    LuaAssert(L, util::ltrim("  foo  ") == "foo  ");
-    LuaAssert(L, util::rtrim("  foo  ") == "  foo");
-    LuaAssert(L, util::trim("  foo  ") == "foo");
-    LuaAssert(L, util::trim("foo") == "foo");
-    LuaAssert(L, util::trim("") == "");
-    
-    // Test sv_read_line
-    std::string_view v = "foo\nbar\r\n";
-    std::string_view v1 = util::sv_read_line(v);
-    std::string_view v2 = util::sv_read_line(v);
-    std::string_view v3 = util::sv_read_line(v);
-    LuaAssertStrEq(L, v1, "foo");
-    LuaAssertStrEq(L, v2, "bar");
-    LuaAssertStrEq(L, v3, "");
-
     // Test distance_squared
     LuaAssert(L, util::distance_squared(1, 1, 5, 4) == 25.0);
     LuaAssert(L, util::distance_squared(5, 4, 1, 1) == 25.0);
@@ -498,6 +659,10 @@ LuaDefine(unittests_util, "", "some unit tests") {
     LuaAssertStrEq(L, util::hash_to_hex(util::HashValue(0x1234,0x789a)),
         "0000000000001234000000000000789a");
     
+    // Test hash_to_double
+    LuaAssert(L, util::hash_to_double(0x1000000000000000) == 1.0/16.0);
+    LuaAssert(L, util::hash_to_double(0x7000000000000000) == 7.0/16.0);
+    LuaAssert(L, util::hash_to_double(0xF000000000000000) == 15.0/16.0);
     return 0;
 }
 

luprex/core/cpp/util.hpp

@@ -1,3 +1,18 @@
+///////////////////////////////////////////////////////////////////////
+//
+// NAMESPACE SV
+//
+//  * Operate on string_view or just characters.
+//  * Do not allocate memory.
+//  * Do not copy strings.
+//
+// NAMESPACE UTIL
+//
+//  * General purpose utility functions.
+//  * Sort of a catch-all.
+//
+///////////////////////////////////////////////////////////////////////
+
 #ifndef UTIL_HPP
 #define UTIL_HPP
 
@@ -15,6 +30,107 @@
 #include "luastack.hpp"
 #include "spookyv2.hpp"
 
+namespace sv {
+
+// Bring this into our namespace.
+using string_view = std::string_view;
+
+// Test character class, ignoring current locale and unicode issues.
+inline bool ascii_isupper(char c) { return (c >= 'A') && (c <= 'Z'); }
+inline bool ascii_islower(char c) { return (c >= 'a') && (c <= 'z'); }
+inline bool ascii_isdigit(char c) { return (c >= '0') && (c <= '9'); }
+inline bool ascii_isalpha(char c) { return ascii_isupper(c) || ascii_islower(c); }
+inline bool ascii_isalnum(char c) { return ascii_isalpha(c) || ascii_isdigit(c); }
+inline bool ascii_isspace(char c) { return (c==' ')||(c=='\t')||(c=='\r')||(c=='\n')||(c=='\f')||(c=='\v'); }
+
+// Check for the null string_view
+//
+// Note that the null string view is an empty string, 
+// but not every empty string is the null string view.
+//
+inline bool isnull(string_view v) { return v.data() == nullptr; }
+
+// Check if numbers can be parsed as int64/double
+bool valid_double(string_view v);
+bool valid_int64(string_view v);
+bool valid_hex64(string_view v);
+
+// Parse numbers as int32, int64, or double.  Returns errval on failure.
+double to_double(string_view v, double errval = std::numeric_limits<double>::quiet_NaN());
+int64_t to_int64(string_view v, int64_t errval = std::numeric_limits<int64_t>::min());
+uint64_t to_hex64(string_view v, uint64_t errval = std::numeric_limits<uint64_t>::max());
+
+// Trim whitspace from a string_view.
+string_view ltrim(string_view v);
+string_view rtrim(string_view v);
+string_view trim(string_view v);
+
+// Trim specific character (all occurrences) from a string_view.
+string_view ltrim(string_view v, char c);
+string_view rtrim(string_view v, char c);
+string_view trim(string_view v, char c);
+
+// Return true if the string has the specified prefix or suffix.
+bool has_prefix(string_view s, string_view prefix);
+bool has_suffix(string_view s, string_view suffix);
+
+// Return the length of the common prefix of A and B.
+int common_prefix_length(string_view a, string_view b);
+
+// Return true if the string is a lua identifier.
+bool is_lua_id(string_view s);
+
+// Return true if the line of code is a lua comment.
+bool is_lua_comment(string_view s);
+
+// Return the first character, but if the view is empty,
+// return zero.
+inline char zfront(string_view &s) {
+    return s.empty() ? char(0) : s.front();
+}
+
+// Read from a string_view until separator is reached.
+//
+// If the separator appears in the source, returns everything
+// before the separator, and updates the source to everything
+// after the separator.
+//
+// If the separator doesn't appear in the source, returns
+// the entire source, and replaces source with the null string_view.
+//
+string_view read_to_sep(string_view &source, char sep);
+
+// Read from a string_view until newline is reached.
+//
+// If there's a line-break in the source (newline or CRLF),
+// returns the text before the line-break, and updates the
+// source to the text after the line-break.
+//
+// If there's no line-break in the source, returns the entire source,
+// and updates source to the null string_view.
+//
+string_view read_to_line(string_view &source);
+
+// Read from a string_view until whitespace is reached.
+//
+// If there's any whitespace in the source, returns the text
+// before the whitespace, and update the source to the text
+// after the whitespace.
+//
+// If there's no whitespace in the source, returns the entire
+// source, and updates the source to the null string_view.
+//
+string_view read_to_space(string_view &source);
+
+// Read up to nbytes from a string_view.
+//
+string_view read_nbytes(string_view &source, int nbytes);
+
+// Return true if the string is valid utf-8.
+bool valid_utf8(string_view s);
+
+} // namespace sv
+
 namespace util {
 
 enum WorldType {
@@ -39,12 +155,13 @@ using LuaSourcePtr = std::unique_ptr<LuaSourceVec>;
 using HashValue = std::pair<uint64_t, uint64_t>;
 using IdVector = eng::vector<int64_t>;
 
+// Ascii uppercase and lowercase.
+eng::string ascii_tolower(std::string_view c);
+eng::string ascii_toupper(std::string_view c);
+
 // Return seconds elapsed, for profiling purposes.
 double profiling_clock();
 
-// Return true if the string is a valid lua identifier.
-bool is_identifier(const eng::string &str);
-
 // Output a string to a stream using Lua string escaping and quoting.
 void quote_string(const eng::string &str, std::ostream *os);
 
@@ -70,6 +187,9 @@ eng::string hash_to_hex(const HashValue &hash);
 // This is a good hash, but not cryptographically good.
 uint64_t hash_ints(uint64_t n1, uint64_t n2, uint64_t n3, uint64_t n4);
 
+// Convert a 64-bit hash value into a floating point number between 0 and 1.
+double hash_to_double(uint64_t hash);
+
 // Split a string into multiple strings
 StringVec split(const eng::string &s, char sep);
 
@@ -85,51 +205,16 @@ eng::string join(const StringVec &strs, eng::string sep);
 // Return N repetitions of string A
 eng::string repeat_string(const eng::string &a, int n);
 
-// Return the length of the common prefix of A and B.
-int common_prefix_length(const eng::string &a, const eng::string &b);
-
 // String to lowercase/uppercase.  Ascii only, no unicode.
 eng::string tolower(eng::string input);
 eng::string toupper(eng::string input);
 
-// Return true if the string has the specified prefix or suffix.
-bool has_prefix(const eng::string &s, const eng::string &prefix);
-bool has_suffix(const eng::string &s, const eng::string &suffix);
-
-// Return true if the string can be parsed as an integer.
-bool validinteger(const eng::string &value);
-
-// String to integer.  Returns errval if the number is not parseable.
-int64_t strtoint(const eng::string &value, int64_t errval);
-
-// String to double.  Returns NAN if the number is not parseable.
-double strtodouble(const eng::string &value);
-
-// Trim a string_view
-std::string_view sv_ltrim(std::string_view v);
-std::string_view sv_rtrim(std::string_view v);
-std::string_view sv_trim(std::string_view v);
-
-// Trim strings: left end, right end, both ends.
-eng::string ltrim(std::string_view s);
-eng::string rtrim(std::string_view s);
-eng::string trim(std::string_view s);
-
-// Read a line from a string_view
-std::string_view sv_read_line(std::string_view &source);
-
 // Calculate distance between two points
 double distance_squared(double x1, double y1, double x2, double y2);
 
-// Return true if a world type is authoritative.
-bool world_type_authoritative(util::WorldType wt);
-
 // Make a LuaSourceVec with one element, for unit testing.
 LuaSourcePtr make_lua_source(const eng::string &code);
 
-// Return true if the line of code is a lua comment.
-bool is_lua_comment(const eng::string &line);
-
 // Remove nullptrs from a vector of unique pointers.
 template<class T>
 void remove_nullptrs(eng::vector<std::unique_ptr<T>> &vec) {
@@ -160,6 +245,22 @@ public:
   int overflow(int c) { return c; }
 };
 
+// send_to_stream: send all arguments to the specified stream.
+inline void send_to_stream(std::ostream &os) {}
+template <class ARG, class... REST>
+inline void send_to_stream(std::ostream &os, ARG arg, REST & ... rest) {
+    os << arg;
+    send_to_stream(os, rest...);
+}
+
+// ss: convert all arguments to a string by sending them to a stringstream.
+template <class... ARGS>
+inline eng::string ss(ARGS & ... args) {
+    eng::ostringstream oss;
+    send_to_stream(oss, args...);
+    return oss.str();
+}
+
 } // namespace util
 
 std::ostream &operator<<(std::ostream &oss, const util::hex64 &v);

luprex/core/cpp/world-accessor.cpp

@@ -294,23 +294,32 @@ LuaDefine(tangible_scan, "plane,x,y,radius,omit_nowhere",
 
 LuaDefine(wait, "nticks",
     "|Wait the specified number of ticks.") {
-    if ((lua_gettop(L) != 1) || (lua_type(L, -1) != LUA_TNUMBER)) {
-        luaL_error(L, "Argument to wait must be a number.");
+    World *w = World::fetch_global_pointer(L);
+    w->guard_blockable(L, "wait");
+
+    // Parse the argument.
+    LuaArg seconds;
+    LuaStack LS(L, seconds);
+    int64_t n = LS.ckinteger(seconds);
+    if ((n < 0) || (n > 1000000)) {
+        luaL_error(L, "Argument to wait must be between 0 and 1000000");
+        return LS.result();
     }
-    return lua_yield(L, 1);
+
+    // Schedule a continuation.
+    w->schedule(w->clock_ + n, w->lthread_thread_id_, w->lthread_place_id_);
+    return lua_yield(L, 0);
 }
 
-LuaDefine(tangible_nopredict, "",
+LuaDefine(nopredict, "",
     "|Stop predictive execution of this thread.") {
-    if (lua_gettop(L) != 0) {
-        luaL_error(L, "tangible.nopredict takes no arguments");
-    }
     World *w = World::fetch_global_pointer(L);
-    if (util::world_type_authoritative(w->world_type_)) {
-        return 0;
-    } else {
-        return lua_yield(L, 0);
+    w->guard_nopredict(L, "nopredict");
+
+    if (lua_gettop(L) != 0) {
+        luaL_error(L, "nopredict takes no arguments");
     }
+    return 0;
 }
 
 LuaDefine(math_random, "(args...)",
@@ -412,12 +421,15 @@ LuaDefine(math_random, "(args...)",
         }
     }
 
-    // Generate the hash and convert to a lua_Number.
-    uint64_t hash = util::hash_ints(seed, count, salt, 456);
     if (!have_range) {
-        double result = (hash & LuaStack::MAXINT) * 0x1p-53;
-        lua_pushnumber(L, result);
+        // Generate the hash and convert to a double.
+        uint64_t hash = util::hash_ints(seed, count, salt, 456);
+        lua_pushnumber(L, util::hash_to_double(hash));
     } else {
+        // Generate the hash and scale it into the desired range.
+        // This code is not quite right: the results are not quite
+        // uniform, this is especially true for very long ranges.
+        uint64_t hash = util::hash_ints(seed, count, salt, 456);
         uint64_t range = (high - low) + 1;
         uint64_t offset = (hash & 0x7FFFFFFFFFFFFFFF) % range;
         int64_t result = low + int64_t(offset);
@@ -426,15 +438,16 @@ LuaDefine(math_random, "(args...)",
     return 1;
 }
 
-LuaDefine(math_randomstate, "(seed)",
+LuaDefine(math_randomstate, "seed",
     "|Create and return a randomstate table."
     "|This is a lua table that stores the state for a random"
     "|number generator.  A randomstate table can be passed"
     "|to math.random."
     "|"
-    "|You can optionally omit the seed, in which case it will"
-    "|pick a seed randomly.  Automatically-generated seeds are"
-    "|guaranteed never to be the same as user-specified seeds.") {
+    "|You can optionally omit the seed, in which case a"
+    "|seed will be chosen randomly.  Automatically-generated"
+    "|seeds are guaranteed never to be the same as"
+    "|user-specified seeds.") {
     double seed;
     if (lua_gettop(L) == 0) {
         World *w = World::fetch_global_pointer(L);
@@ -508,4 +521,36 @@ LuaDefine(doc, "function",
     }
     (*ostream) << doc;
     return LS.result();
+}
+
+LuaDefine(http_get, "request",
+    "|Make an HTTP GET request.  Returns an HTTP response."
+    "|See doc(http.request) and doc(http.response).") {
+    World *w = World::fetch_global_pointer(L);
+    w->guard_blockable(L, "http.get");
+
+    LuaArg request;
+    LuaRet response;
+    LuaStack LS(L, request, response);
+    HttpClientRequest req;
+    
+    // Parse the request and make sure it's valid.
+    req.set_config(LS, request);
+    req.set_defaults();
+    eng::string error = req.check();
+    if (!error.empty()) {
+        HttpClientResponse::store_fail(LS, response, 400, util::ss("bad request: ", error));
+        return LS.result();
+    }
+
+    // Give the request an ID.
+    req.set_request_id(w->id_global_pool_.get_one());
+    req.set_place_id(w->lthread_place_id_);
+    req.set_thread_id(w->lthread_thread_id_);
+
+    // Store it in the global request table.
+    w->http_requests_[req.request_id()] = req;
+
+    // Block.
+    return lua_yield(L, 0);
 }

luprex/core/cpp/world-core.cpp

@@ -5,6 +5,8 @@
 #include "gui.hpp"
 #include "traceback.hpp"
 #include "pprint.hpp"
+#include "util.hpp"
+
 #include <iostream>
 
 void World::store_global_pointer(lua_State *L, World *v) {
@@ -62,7 +64,7 @@ World::World(util::WorldType wt) {
     LS.rawset(LuaRegistry, "tangibles", LuaNewTable);
 
     // Create the globaldb and oncedb in the registry.
-    if (util::world_type_authoritative(wt)) {
+    if ((wt == util::WORLD_TYPE_MASTER) || (wt == util::WORLD_TYPE_STANDALONE)) {
         LS.rawset(LuaRegistry, "globaldb", LuaNewTable);
         LS.rawset(LuaRegistry, "oncedb", LuaNewTable);
     }
@@ -289,23 +291,19 @@ eng::string World::probe_lua(int64_t actor_id, const eng::string &lua) {
     // Call the closure.
     int top = lua_gettop(L);
     lua_pushvalue(L, closure.index());
-    open_lthread_state(actor_id, actor_id, false, true);
-    status = traceback_pcall(L, 0, LUA_MULTRET);
+    open_lthread_state(actor_id, actor_id, 0, false, true);
+    eng::string msg = traceback_pcall(L, 0, LUA_MULTRET);
     
     // If there's an error message, print it.
     // Otherwise, pretty-print the results.
     std::ostream *ostream = lthread_print_stream();
-    if (status == LUA_OK) {
+    if (msg.empty()) {
         for (int i = top + 1; i <= lua_gettop(L); i++) {
             LuaSpecial root(i);
             pprint(LS, root, true, ostream);
             (*ostream) << std::endl;
         }
     } else {
-        const char *msg = lua_tostring(L, -1);
-        if (msg == NULL) {
-            msg = "(error object is not a string)";
-        }
         (*ostream) << msg << std::endl;
     }
 
@@ -361,13 +359,13 @@ void World::update_gui(int64_t actor_id, int64_t place_id, Gui *gui) {
     lua_pushvalue(L, actor.index());
     lua_pushvalue(L, place.index());
     Gui::store_global_pointer(L, gui);
-    open_lthread_state(actor_id, place_id, false, false);
-    int status = traceback_pcall(L, 2, 0);
+    open_lthread_state(actor_id, place_id, 0, false, false);
+    eng::string msg = traceback_pcall(L, 2, 0);
     close_lthread_state();
     Gui::store_global_pointer(L, nullptr);
-    if (status != 0) {
+    if (!msg.empty()) {
         gui->clear(0);
-        std::cerr << lua_tostring(L, -1);
+        std::cerr << msg << std::endl;
         LS.result();
         return;
     }
@@ -395,6 +393,71 @@ void World::update_source(const util::LuaSourceVec &source) {
     assert(stack_is_clear());
 }
 
+void World::http_response(const HttpClientResponse &response) {
+    // Find the request.
+    auto iter = http_requests_.find(response.request_id());
+    if (iter == http_requests_.end()) {
+        return;
+    }
+    HttpClientRequest request = iter->second;
+    http_requests_.erase(iter);
+
+    // Get the place and thread as lua objects.
+    LuaVar tangibles, place, mt, threads, thinfo, thread;
+    LuaStack LS(state(), tangibles, place, mt, threads, thinfo, thread);
+    LS.rawget(tangibles, LuaRegistry, "tangibles");
+    LS.rawget(place, tangibles, request.place_id());
+    if (!LS.istable(place)) {
+        return;
+    }
+    LS.getmetatable(mt, place);
+    if (!LS.istable(mt)) {
+        return;
+    }
+    LS.rawget(threads, mt, "threads");
+    if (!LS.istable(threads)) {
+        return;
+    }
+    LS.rawget(thinfo, threads, request.thread_id());
+    if (!LS.istable(thinfo)) {
+        return;
+    }
+    LS.rawget(thread, thinfo, "thread");
+    if (!LS.isthread(thread)) {
+        return;
+    }
+    lua_State *CO = LS.ckthread(thread);
+
+    // Push the response onto the awakening thread.
+    LuaRet responsetable;
+    LuaStack LSCO(CO, responsetable);
+    response.store(LSCO, responsetable);
+
+    // Clean up lua stacks.
+    LSCO.result();
+    LS.result();
+    assert(stack_is_clear());
+
+    // Awaken the thread, with its new return value.
+    schedule(0, request.thread_id(), request.place_id());
+    run_scheduled_threads();
+}
+
+void World::http_responses(const HttpClientResponseVec &responses) {
+    for (const HttpClientResponse &response : responses) {
+        http_response(response);
+    }
+}
+
+void World::abort_all_http_requests(int status_code, std::string_view error) {
+    HttpClientResponse abortresponse;
+    abortresponse.fail(status_code, error);
+    while (!http_requests_.empty()) {
+        abortresponse.set_request_id(http_requests_.begin()->first);
+        http_response(abortresponse);
+    }
+}
+
 void World::run_unittests() {
     assert(stack_is_clear());
     source_db_.run_unittests();
@@ -434,8 +497,8 @@ void World::invoke_flush_prints(int64_t actor_id, int64_t place_id, const eng::s
     if (actor_id != place_id) {
         return;
     }
-    int line = util::strtoint(action, -1);
-    if (line < 0) {
+    int64_t line = sv::to_int64(action, -1);
+    if ((line < 0)||(line > INT_MAX)) {
         return;
     }
     Tangible *tactor = tangible_get(actor_id);
@@ -501,7 +564,7 @@ void World::invoke_lua(int64_t actor_id, int64_t place_id, const eng::string &ac
     LS.newtable(thinfo);
     LS.rawset(thinfo, "thread", thread);
     LS.rawset(thinfo, "actorid", actor_id);
-    LS.rawset(thinfo, "nargs", 0);
+    LS.rawset(thinfo, "isnew", true);
     LS.rawset(thinfo, "useppool", true);
     LS.rawset(thinfo, "print", true);
     
@@ -514,7 +577,7 @@ void World::invoke_lua(int64_t actor_id, int64_t place_id, const eng::string &ac
     LS.rawset(threads, tid, thinfo);
     LS.result();
 
-    thread_sched_.add(0, tid, place_id);
+    schedule(0, tid, place_id);
     run_scheduled_threads();
     assert(stack_is_clear());
 }
@@ -530,7 +593,7 @@ void World::invoke_plan(int64_t actor_id, int64_t place_id, const eng::string &a
     }
 
     // Make sure the action starts with "cb_"
-    if (!util::has_prefix(action, "cb_")) {
+    if (!sv::has_prefix(action, "cb_")) {
         return;
     }
 
@@ -594,7 +657,7 @@ void World::invoke_plan(int64_t actor_id, int64_t place_id, const eng::string &a
     LS.newtable(thinfo);
     LS.rawset(thinfo, "thread", thread);
     LS.rawset(thinfo, "actorid", actor_id);
-    LS.rawset(thinfo, "nargs", 3); // actor, place, invdata
+    LS.rawset(thinfo, "isnew", true);
     LS.rawset(thinfo, "useppool", true);
     LS.rawset(thinfo, "print", false);
     
@@ -609,13 +672,13 @@ void World::invoke_plan(int64_t actor_id, int64_t place_id, const eng::string &a
     
     // Push the thread's ID into the runnable thread queue,
     // then run the thread queue.
-    thread_sched_.add(0, tid, place_id);
+    schedule(0, tid, place_id);
     run_scheduled_threads();
     assert(stack_is_clear());
 }
 
 void World::invoke_tick(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data) {
-    if (!util::world_type_authoritative(world_type_)) {
+    if (!is_authoritative()) {
         return;
     }
     clock_ += 1;
@@ -623,7 +686,7 @@ void World::invoke_tick(int64_t actor_id, int64_t place_id, const eng::string &a
 }
 
 void World::invoke_lua_source(int64_t actor_id, int64_t place_id, const eng::string &action, const InvocationData &data) {
-    if (!util::world_type_authoritative(world_type_)) {
+    if (!is_authoritative()) {
         return;
     }
     // We need some kind of authentication here.
@@ -637,11 +700,40 @@ void World::invoke_lua_source(int64_t actor_id, int64_t place_id, const eng::str
     }
 }
 
+void World::guard_blockable(lua_State *L, const char *fn) {
+    if (lthread_thread_id_ == 0) {
+        // in a probe, http.get throws an error.
+        luaL_error(L, "cannot %s in a probe", fn);
+        assert(false);
+    }
+    if (!is_authoritative()) {
+        // in a nonauth model, http.get is converted to nopredict.
+        lua_yield(L, 0);
+        luaL_error(L, "unexplained nopredict failure in %s", fn);
+        assert(false);
+    }
+}
+
+void World::guard_nopredict(lua_State *L, const char *fn) {
+    if (lthread_thread_id_ == 0) {
+        return;
+    }
+    if (!is_authoritative()) {
+        lua_yield(L, 0);
+        luaL_error(L, "unexplained nopredict failure in %s", fn);
+    }
+}
+
+void World::schedule(int64_t clk, int64_t thid, int64_t plid) {
+    assert(is_authoritative());
+    thread_sched_.add(clk, thid, plid);
+}
+
 void World::run_scheduled_threads() {
     assert(stack_is_clear());
     lua_State *L = state();
-    LuaVar tangibles, place, mt, threads, thinfo, actorid, nargs, useppool, thread, print;
-    LuaStack LS(L, tangibles, place, mt, threads, thinfo, actorid, nargs, useppool, thread, print);
+    LuaVar tangibles, place, mt, threads, thinfo, actorid, isnew, useppool, thread, print;
+    LuaStack LS(L, tangibles, place, mt, threads, thinfo, actorid, isnew, useppool, thread, print);
 
     LS.rawget(tangibles, LuaRegistry, "tangibles");
     while (thread_sched_.ready(clock_)) {
@@ -666,8 +758,8 @@ void World::run_scheduled_threads() {
         if (!LS.isnumber(actorid)) {
             continue;
         }
-        LS.rawget(nargs, thinfo, "nargs");
-        if (!LS.isnumber(nargs)) {
+        LS.rawget(isnew, thinfo, "isnew");
+        if (!LS.isboolean(isnew)) {
             continue;
         }
         LS.rawget(useppool, thinfo, "useppool");
@@ -681,35 +773,16 @@ void World::run_scheduled_threads() {
 
         // Resume the coroutine.
         lua_State *CO = LS.ckthread(thread);
-        open_lthread_state(LS.ckinteger(actorid), sched.place_id(), LS.ckboolean(useppool), true);
-        int status = lua_resume(CO, nullptr, LS.ckint(nargs));
+        open_lthread_state(LS.ckinteger(actorid), sched.place_id(), sched.thread_id(), LS.ckboolean(useppool), true);
+        int nargs = LS.ckboolean(isnew) ? (lua_gettop(CO) - 1) : lua_gettop(CO);
+        int status = lua_resume(CO, nullptr, nargs);
         std::ostream *ostream = lthread_print_stream();
 
-        // Three possible outcomes: finished, yielded, or errored.
-        if (!util::world_type_authoritative(world_type_)) {
-            LS.rawset(threads, sched.thread_id(), LuaNil);
-        } else if (status == LUA_YIELD) {
-            // If there's nothing on the stack, infer that tangible.nopredict yielded.
-            if (lua_gettop(CO) == 0) {
-                LS.rawset(threads, sched.thread_id(), LuaNil);
-            }
-            // If there's a single number on the stack, infer that 'wait' yielded.
-            else if ((lua_gettop(CO) == 1) && (lua_isnumber(CO, 1))) {
-                lua_Number delay = lua_tonumber(CO, 1);
-                lua_settop(CO, 0);
-                LS.rawset(thinfo, "nargs", 0);
-                LS.rawset(thinfo, "useppool", false);
-                thread_sched_.add(clock_ + int64_t(delay), sched.thread_id(), sched.place_id());
-            }
-            // In any other case, generate an error and kill the coroutine.
-            else {
-                std::cerr << "Thread yielded incorrectly.  Killing it." << std::endl;
-                LS.rawset(threads, sched.thread_id(), LuaNil);
-            }
-        } else if (status == LUA_OK) {
+        if (status == LUA_OK) {
             // Successfully ran to completion.  Print any return values.
             // Remove from thread table.
             LS.rawget(print, thinfo, "print");
+            LS.rawset(threads, sched.thread_id(), LuaNil);
             LuaStack LSCO(CO);
             if (LS.ckboolean(print)) {
                 for (int i = 1; i <= lua_gettop(CO); i++) {
@@ -717,12 +790,21 @@ void World::run_scheduled_threads() {
                     (*ostream) << std::endl;
                 }
             }
-            LS.rawset(threads, sched.thread_id(), LuaNil);
+        } else if (status == LUA_YIELD) {
+            if (is_authoritative()) {
+                LS.rawset(thinfo, "isnew", false);
+                LS.rawset(thinfo, "useppool", false);
+            } else {
+                // In a nonauth model, a yield is converted to a 'nopredict'.
+                LS.rawset(threads, sched.thread_id(), LuaNil);
+            }
         } else {
             // Generated an error.  Add a traceback, print, and kill the coroutine.
             // Currently, the error is sent to the actor.  That seems... not right in the long run.
-            traceback_coroutine(CO);
-            (*ostream) << lua_tostring(CO, -1);
+            if (is_authoritative()) {
+                traceback_coroutine(CO);
+                (*ostream) << lua_tostring(CO, -1);
+            }
             LS.rawset(threads, sched.thread_id(), LuaNil);
         }
         close_lthread_state();
@@ -755,12 +837,14 @@ void World::clear_lthread_state() {
     lthread_prints_.reset();
     lthread_actor_id_ = 0;
     lthread_place_id_ = 0;
+    lthread_thread_id_ = 0;
     lthread_use_ppool_ = false;
 }
 
-void World::open_lthread_state(int64_t actor, int64_t place, bool ppool, bool prints) {
+void World::open_lthread_state(int64_t actor, int64_t place, int64_t thread, bool ppool, bool prints) {
     lthread_actor_id_ = actor;
     lthread_place_id_ = place;
+    lthread_thread_id_ = thread;
     lthread_use_ppool_ = ppool;
     if (prints) {
         lthread_prints_.reset(new eng::ostringstream);
@@ -778,8 +862,7 @@ void World::close_lthread_state() {
         const eng::string &output = lthread_prints_->str();
         Tangible *actor = tangible_get(lthread_actor_id_);
         if (actor != nullptr) {
-            bool auth = util::world_type_authoritative(world_type_);
-            actor->print_buffer_.add_string(output, auth);
+            actor->print_buffer_.add_string(output, is_authoritative());
         }
     }
     // Now clean up everything.
@@ -802,6 +885,7 @@ void World::serialize(StreamBuffer *sb) {
     id_global_pool_.serialize(sb);
     sb->write_int64(clock_);
     thread_sched_.serialize(sb);
+    http_requests_.serialize(sb);
     sb->write_uint32(tangibles_.size());
     for (const auto &p : tangibles_) {
         sb->write_int64(p.first);
@@ -819,6 +903,7 @@ void World::deserialize(StreamBuffer *sb) {
     id_global_pool_.deserialize(sb);
     clock_ = sb->read_int64();
     thread_sched_.deserialize(sb);
+    http_requests_.deserialize(sb);
     // Mark all tangibles for deletion by setting ID to zero.
     for (const auto &p : tangibles_) {
         p.second->plane_item_.set_id(0);
@@ -843,6 +928,8 @@ void World::deserialize(StreamBuffer *sb) {
             ++iter;
         }
     }
+    // After a save and load, http requests no longer should exist
+    abort_all_http_requests(425, "http requests aborted by loading a save game");
     assert(stack_is_clear());
 }
 

luprex/core/cpp/world.hpp

@@ -18,6 +18,7 @@
 #include "debugcollector.hpp"
 #include "printbuffer.hpp"
 #include "sched.hpp"
+#include "http.hpp"
 #include "source.hpp"
 #include "gui.hpp"
 #include "luasnap.hpp"
@@ -188,6 +189,9 @@ public:
     // This is the primary dispatcher for all operations that mutate a world model.
     // To mutate a world model, create an invocation, then invoke it.
     //
+    // It is legal to mutate a world model without using 'Invoke', but
+    // only in authoritative world models.
+    //
     void invoke(const Invocation &inv);
     
     // Get the PrintBuffer of the actor.
@@ -201,6 +205,15 @@ public:
     void update_source(const util::LuaSourcePtr &source);
     void update_source(const util::LuaSourceVec &source);
     
+    // Supply an HTTP response to an outstanding HTTP request.
+    void http_response(const HttpClientResponse &response);
+    void http_responses(const HttpClientResponseVec &responses);
+    
+    // Abort all HTTP requests.  This is typically used after
+    // reloading a world from a save-game.  The http requests that
+    // were in progress are long-since dead.
+    void abort_all_http_requests(int status_code, std::string_view error);
+
     // Run all unit tests.
     //
     void run_unittests();
@@ -212,6 +225,14 @@ public:
     //
     static World *fetch_global_pointer(lua_State *L);
 
+    // Check if the world is authoritative.
+    //
+    bool is_authoritative() const { return (world_type_ == util::WORLD_TYPE_MASTER) || (world_type_ == util::WORLD_TYPE_STANDALONE); }
+
+    // Get a table showing all outstanding HTTP requests.
+    //
+    const HttpClientRequestMap &http_requests() const { return http_requests_; }
+
     // Serialize and deserialize.
     //
     void serialize(StreamBuffer *sb);
@@ -245,7 +266,7 @@ public:
     // cleared.
     //
     void clear_lthread_state();
-    void open_lthread_state(int64_t actor_id, int64_t place_id, bool ppool, bool prints);
+    void open_lthread_state(int64_t actor_id, int64_t place_id, int64_t thread_id, bool ppool, bool prints);
     void close_lthread_state();
 
     std::ostream *lthread_print_stream() const;
@@ -259,7 +280,20 @@ public:
     //
     int64_t alloc_id_predictable();
     
+    // If we're in a probe, generate an error.
+    // If we're in a nonauthoritative model, do a nopredict yield.
+    // Otherwise, return.
+    void guard_blockable(lua_State *L, const char *fn);
+    
+    // If we're in a probe, return.
+    // If we're in a nonauthoritative model, do a nopredict yield.
+    // Otherwise, return.
+    void guard_nopredict(lua_State *L, const char *fn);
+
 private:
+    // Add a thread to the scheduler queue.
+    //
+    void schedule(int64_t clk, int64_t thid, int64_t plid);
 
     // Store a pointer to a world model into a lua registry.
     //
@@ -471,6 +505,7 @@ private:
     eng::unordered_map<int64_t, UniqueTangible> tangibles_;
 
     // Current time.
+    //
     int64_t clock_;
 
     // Thread schedule: must include every thread, except
@@ -478,7 +513,13 @@ private:
     //
     Schedule thread_sched_;
 
+    // Outstanding HTTP requests, indexed by request ID.
+    // Authoritative models only.
+    //
+    HttpClientRequestMap http_requests_;
+
     // Serialized snapshot of world model.
+    //
     StreamBuffer snapshot_;
 
     // Redirects.
@@ -489,6 +530,7 @@ private:
     //
     int64_t lthread_actor_id_;
     int64_t lthread_place_id_;
+    int64_t lthread_thread_id_;
     int64_t lthread_use_ppool_;
     std::unique_ptr<eng::ostringstream> lthread_prints_;
 
@@ -503,6 +545,9 @@ private:
     friend int lfn_tangible_scan(lua_State *L);
     friend int lfn_math_random(lua_State *L);
     friend int lfn_math_randomstate(lua_State *L);
+    friend int lfn_wait(lua_State *L);
+    friend int lfn_nopredict(lua_State *L);
+    friend int lfn_http_get(lua_State *L);
 };
 
 using UniqueWorld = std::unique_ptr<World>;

luprex/core/lua/login.lua

@@ -3,6 +3,7 @@ makeclass('login')
 function login.interface(actor, place)
     gui.menu_item("cb_becomeplayer", "Become a Player")
     gui.menu_item("cb_p123", "Print 1, 2, 3")
+    gui.menu_item("cb_p123_nopredict", "Print 1, 2, 3 nopredict")
     gui.menu_item("cb_uglytimedaemon","Start the Time Daemon")
     end
 
@@ -23,12 +24,12 @@ function login.cb_p123(actor, place, dialog)
     print(3)
     end
 
--- this is function documentation for setfoo.
-function setfoo(n)
-    tangible.nopredict()
-    tangible.actor().inventory.foo = n
+function login.cb_p123_nopredict(actor, place, dialog)
+    nopredict()
+    print(1)
+    wait(1)
+    print(2)
+    wait(1)
+    print(3)
     end
 
-function buildq()
-    return tangible.build{class="login", x=10, y=0, z=0, plane="nowhere", graphic="what"}
-    end