Did a lot of work on the HTTP server side

luprex/core/cpp/http.cpp

@@ -14,17 +14,43 @@
 
 using string_view = std::string_view;
 
+bool is_supported_protocol(string_view protocol) {
+    return (protocol == "HTTP/1.0") || (protocol == "HTTP/1.1");
+}
+
+bool is_supported_method(string_view method) {
+    return ((method == "GET") || (method == "HEAD") || (method == "POST"));
+}
+
 bool words_separated_by_dashes(string_view v) {
     while (true) {
-        if (!sv::ascii_isalpha(sv::zfront(v))) return false;
+        string_view word = sv::read_ascii_identifier(v);
-        v.remove_prefix(1);
+        if (word.empty()) return false;
-        while (sv::ascii_isalnum(sv::zfront(v))) v.remove_prefix(1);
         if (v.empty()) return true;
-        if (sv::zfront(v) != '-') return false;
+        char c = v.front();
+        if (c != '-') return false;
         v.remove_prefix(1);
     }
 }
 
+// This doesn't check whether the mime type is actually
+// registered, obviously.  It only checks that it's in
+// the desired notation.
+bool valid_mime_type(string_view method) {
+    string_view part1 = sv::read_ascii_identifier(method);
+    if (part1.empty()) return false;
+    if (sv::zfront(method) != '/') return false;
+    method.remove_prefix(1);
+    while (true) {
+        string_view word = sv::read_ascii_identifier(method);
+        if (word.empty()) return false;
+        if (method.empty()) return true;
+        char c = method.front();
+        if ((c != '-') && (c != '.') && (c != '+')) return false;
+        method.remove_prefix(1);
+    }
+}
+
 // Technically, this is a true, correct URL encode routine.
 static eng::string url_encode_param(string_view value) {
     eng::ostringstream result;
@@ -142,25 +168,31 @@ public:
     ParsedURL(std::string_view url) {
         clear();
 
-        proto = util::ascii_tolower(sv::read_to_sep(url, ':'));
+        if (!sv::has_prefix(url, "/")) {
-        if (!sv::has_prefix(url, "//")) { clear(); return; }
+            proto = util::ascii_tolower(sv::read_to_sep(url, ':'));
-        url.remove_prefix(2);
+            if (!sv::has_prefix(url, "//")) { clear(); return; }
-        if (!words_separated_by_dashes(proto)) { clear(); return; }
+            url.remove_prefix(2);
-        
+            if (!words_separated_by_dashes(proto)) { clear(); return; }
-        // Extract the host and port as a single string.
+            
-        string_view turl = url;
+            // Extract the host and port as a single string.
-        string_view hostport = sv::read_to_sep(turl, '/');
+            string_view turl = url;
-        url.remove_prefix(hostport.size());
+            string_view hostport = sv::read_to_sep(turl, '/');
+            url.remove_prefix(hostport.size());
 
-        // Split the host and port from each other and parse them.
+            // Split the host and port from each other and parse them.
-        host = util::ascii_tolower(sv::read_to_sep(hostport, ':'));
+            host = util::ascii_tolower(sv::read_to_sep(hostport, ':'));
-        if (host.empty()) { clear(); return; }
+            if (host.empty()) { clear(); return; }
-        if (!hostport.empty()) {
+            if (!hostport.empty()) {
-            int64_t iport = sv::to_int64(hostport);
+                int64_t iport = sv::to_int64(hostport);
-            if ((iport < 1) || (iport > 65535)) {
+                if ((iport < 1) || (iport > 65535)) {
-                clear(); return;
+                    clear(); return;
+                }
+                port = iport;
             }
-            port = iport;
+        } else {
+            // Stick in some defaults for unspecified fields.
+            host = "host";
+            proto = "https";
         }
 
         // Split off the path.
@@ -214,9 +246,8 @@ void HttpClientRequest::set_verify_certificate(bool flag) {
 
 void HttpClientRequest::set_method(const eng::string &s) {
     eng::string method = util::ascii_toupper(s);
-    if ((method != "GET") && (method != "HEAD")) {
+    if (!is_supported_method(method)) {
-        fail(util::ss("HTTP method not implemented: ", method, ".",
+        fail(util::ss("HTTP method not implemented: ", method, "."));
-                    "Currently, only HEAD and GET are implemented."));
         return;
     }
     if ((!method_.empty()) && (method_ != method)) {
@@ -301,6 +332,26 @@ void HttpClientRequest::set_url(string_view url) {
     }
 }
 
+void HttpClientRequest::set_mime_type(const eng::string &mime_type) {
+    if (!valid_mime_type(mime_type)) {
+        fail(util::ss("Not a valid mime type: ", mime_type));
+        return;
+    }
+    if (!mime_type_.empty()) {
+        fail(util::ss("Mime type specified twice: ", mime_type_, " and ", mime_type));
+        return;
+    }
+    mime_type_ = mime_type;
+}
+
+void HttpClientRequest::set_content(const eng::string &content) {
+    if (!content_.empty()) {
+        fail(util::ss("Content specified twice"));
+        return;
+    }
+    content_ = content;
+}
+
 void HttpClientRequest::set_verify_certificate(LuaStack &LS, LuaSlot val) {
     if (!LS.isboolean(val)) {
         fail(util::ss("HTTP verify_certificate must be a boolean"));
@@ -374,6 +425,22 @@ void HttpClientRequest::set_url(LuaStack &LS, LuaSlot val) {
     set_url(LS.ckstring(val));
 }
 
+void HttpClientRequest::set_mime_type(LuaStack &LS, LuaSlot val) {
+    if (!LS.isstring(val)) {
+        fail(util::ss("HTTP mime type must be a string"));
+        return;
+    }
+    set_mime_type(LS.ckstring(val));
+}
+
+void HttpClientRequest::set_content(LuaStack &LS, LuaSlot val) {
+    if (!LS.isstring(val)) {
+        fail(util::ss("HTTP content must be a string"));
+        return;
+    }
+    set_content(LS.ckstring(val));
+}
+
 void HttpClientRequest::set_defaults() {
     if (method_.empty()) {
         method_ = "GET";
@@ -404,6 +471,10 @@ void HttpClientRequest::set_config(LuaStack &LS0, LuaSlot tab) {
             set_url(LS, val);
         } else if (kstr == "verifycertificate") {
             set_verify_certificate(LS, val);
+        } else if (kstr == "mimetype") {
+            set_mime_type(LS, val);
+        } else if (kstr == "content") {
+            set_content(LS, val);
         } else if (kstr == "") {
             fail(util::ss("HTTP config parameter names must be strings."));
         } else {
@@ -428,6 +499,19 @@ eng::string HttpClientRequest::check() const {
     if (path_.empty()) {
         return "HTTP url has not been set";
     }
+    if (method_ == "POST") {
+        if (mime_type_.empty()) {
+            if (content_.empty()) {
+                return "HTTP mime type and content not set for POST request";
+            } else {
+                return "HTTP mime type has not been set for POST request";
+            }
+        }
+    } else {
+        if ((!mime_type_.empty()) || (!content_.empty())) {
+            return "HTTP mime type and content are only for POST requests";
+        }
+    }
     return "";
 }
 
@@ -447,7 +531,7 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
     }
 
     // Choose a linebreak.
-    eng::string linebreak = (debug_string) ? "\n" : "\r\n";
+    eng::string linebreak = "\r\n";
 
     // Send the command.
     sb->write_bytes(method_);
@@ -473,14 +557,30 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
 
     // Add the requester IDs (debug string only)
     if (debug_string && ((request_id_ != 0) || (place_id_ != 0) || (thread_id_ != 0))) {
-        sb->write_bytes("X-Requester-Ids: ");
+        sb->write_bytes("X-requester-ids: ");
         sb->ostream() << "rid=" << request_id_ << "; pid=" << place_id_ << "; tid=" << thread_id_;
         sb->write_bytes(linebreak);
     }
 
-    // Send the extra linebreak.
+    // If it's a post request, send the content length and the content type.
-    if (!debug_string) {
+    if (method_ == "POST") {
+        sb->write_bytes("Content-length: ");
+        sb->ostream() << content_.size();
         sb->write_bytes(linebreak);
+        sb->write_bytes("Content-type: ");
+        sb->write_bytes(mime_type_);
+        if (sv::has_prefix(mime_type_, "text/")) {
+            sb->write_bytes(" ; charset=utf-8");
+        }
+        sb->write_bytes(linebreak);
+    }
+
+    // Send the extra linebreak.
+    sb->write_bytes(linebreak);
+
+    // If it's a post request, send the content.
+    if (method_ == "POST") {
+        sb->write_bytes(content_);
     }
 }
 
@@ -543,63 +643,137 @@ void HttpClientRequestMap::deserialize(StreamBuffer *sb) {
     }
 }
 
-HttpClientResponse::HttpClientResponse() {
+HttpParser::HttpParser() {
-    request_id_ = 0;
+    status_ = 0;
-    status_code_ = 0;
-    response_length_ = 0;
     mime_type_ = "";
     content_length_ = -1;
+    comm_length_ = 0;
 }
 
-eng::string HttpClientResponse::DebugString() const {
+void HttpParser::fail(int code, std::string_view message) {
-    eng::ostringstream oss;
+    status_ = code;
-    oss << "HttpClientResponse:" << std::endl;
-    oss << "  status_code: " << status_code_ << std::endl;
-    oss << "  error: " << error_ << std::endl;
-    oss << "  content_length: " << content_length_ << std::endl;
-    oss << "  transfer_encoding: " << transfer_encoding_ << std::endl;
-    oss << "  location: " << location_ << std::endl;
-    oss << "  mime_type: " << mime_type_ << std::endl;
-    oss << "  charset: " << charset_ << std::endl;
-    oss << "  content: " << content_ << std::endl;
-    oss << "  response_length: " << response_length_ << std::endl;
-    return oss.str();
-}
-
-void HttpClientResponse::fail(int code, string_view message) {
-    status_code_ = code;
     error_ = message;
     mime_type_ = "";
     charset_ = "";
     content_ = "";
 }
 
-void HttpClientResponse::incomplete(bool closed) {
+void HttpParser::syntax(std::string_view detail) {
+    if (is_request_) {
+        fail(400, util::ss("malformed request: ", detail));
+    } else {
+        fail(500, util::ss("malformed response: ", detail));
+    }
+}
+
+void HttpParser::incomplete(bool closed) {
     if (closed) {
-        fail(500, "internal server error: response truncated");
+        syntax("response truncated");
     } else {
         fail(0, "response not yet fully received");
     }
 }
 
-void HttpClientResponse::parse_content_encoding(string_view value) {
+void HttpParser::oversized() {
+    fail(413, util::ss("payload too large: Limit=", MAX_CONTENT_LENGTH));
+}
+
+bool HttpParser::parse_request_line(std::string_view &view, bool closed) {
+    // Extract the request line.
+    //
+    string_view request = sv::trim(sv::read_to_line(view));
+    if (sv::isnull(view)) {
+        incomplete(closed);
+        return false;
+    }
+
+    // Break down the request line.
+    //
+    eng::string method = util::ascii_toupper(sv::read_to_space(request));
+    string_view path = sv::read_to_space(request);
+    eng::string protocol = util::ascii_toupper(sv::read_to_space(request));
+    if ((!request.empty()) || (protocol.empty())) {
+        syntax("invalid request line");
+        return false;
+    }
+    if (!is_supported_method(method)) {
+        fail(405, util::ss("Method Not Allowed: ", method));
+        return false;
+    }
+    if (!is_supported_protocol(protocol)) {
+        syntax(util::ss("unsupported protocol: ", protocol));
+        return false;
+    }
+
+    // Parse the url.
+    //
+    ParsedURL url(path);
+    if (!url.valid) {
+        syntax(util::ss("Invalid URL path: ", path));
+        return false;
+    }
+
+    method_ = method;
+    path_ = url.path;
+    params_ = url.params;
+    return true;
+}
+
+bool HttpParser::parse_status_line(std::string_view &view, bool closed) {
+    // Extract the status line.
+    //
+    string_view status = sv::trim(sv::read_to_line(view));
+    if (sv::isnull(view)) {
+        incomplete(closed);
+        return false;
+    }
+
+    // Break down the status line.
+    //
+    string_view protoversion = sv::read_to_space(status);
+    if (!is_supported_protocol(protoversion)) {
+        syntax(util::ss("unsupported protocol: ", protoversion));
+        return false;
+    }
+    string_view scode = sv::read_to_space(status);
+    int64_t code = sv::to_int64(scode, 0);
+    if ((code < 100) || (code > 599)) {
+        syntax(util::ss("invalid response code: ", scode));
+        return false;
+    }
+    status_ = code;
+
+    // Responses outside the range 200-299 are errors,
+    // and therefore must store a nonempty error message.
+    //
+    if ((code < 200) || (code > 299)) {
+        if (status.empty()) {
+            error_ = util::ss("error code ", code);
+        } else {
+            error_ = status;
+        }
+    }
+    return true;
+}
+
+void HttpParser::parse_content_encoding(string_view value) {
     content_encoding_ = util::ascii_tolower(value);
 }
 
-void HttpClientResponse::parse_content_length(string_view value) {
+void HttpParser::parse_content_length(string_view value) {
     int64_t code = sv::to_int64(value);
     if ((code < 0) || (code > INT_MAX)) {
-        fail(500, util::ss("internal server error: unparseable content-length: ", value));
+        syntax(util::ss("unparseable content-length: ", value));
     }
     content_length_ = code;
 }
 
-void HttpClientResponse::parse_content_type(string_view value) {
+void HttpParser::parse_content_type(string_view value) {
     eng::string ctype = util::ascii_tolower(value);
     string_view ctview(ctype);
     mime_type_ = sv::trim(sv::read_to_sep(ctview, ';'));
     if (mime_type_.empty()) {
-        fail(500, util::ss("internal server error: unparseable content-type: ", value));
+        syntax(util::ss("unparseable content-type: ", value));
         return;
     }
     while (true) {
@@ -614,15 +788,15 @@ void HttpClientResponse::parse_content_type(string_view value) {
     }
 }
 
-void HttpClientResponse::parse_location(string_view value) {
+void HttpParser::parse_location(string_view value) {
     location_ = url_decode(value);
 }
 
-void HttpClientResponse::parse_transfer_encoding(string_view value) {
+void HttpParser::parse_transfer_encoding(string_view value) {
     transfer_encoding_ = util::ascii_tolower(value);
 }
 
-void HttpClientResponse::parse_header(string_view header, string_view value) {
+void HttpParser::parse_header(string_view header, string_view value) {
     if (header == "content-encoding") {
         parse_content_encoding(value);
     } else if (header == "content-length") {
@@ -634,14 +808,38 @@ void HttpClientResponse::parse_header(string_view header, string_view value) {
     } else if (header == "transfer-encoding") {
         parse_transfer_encoding(value);
     } else if (header == "content-range") {
-        fail(406, util::ss("not acceptable: unsupported response header: ", header));
+        fail(416, util::ss("range not satisfiable: unsupported header: ", header));
     }
 }
 
-bool HttpClientResponse::parse_content_basic(std::string_view &view, bool closed) {
+bool HttpParser::parse_headers(std::string_view &view, bool closed) {
+    // Parse the headers.
+    while (true) {
+        string_view header = sv::read_to_line(view);
+        if (sv::isnull(view)) {
+            incomplete(closed);
+            return false;
+        }
+        if (header.empty()) {
+            return true;
+        }
+        eng::string command = util::ascii_tolower(sv::trim(sv::read_to_sep(header, ':')));
+        if (sv::isnull(header)) {
+            syntax(util::ss("no colon in header line: ", command));
+            return false;
+        }
+        if (!words_separated_by_dashes(command)) {
+            syntax(util::ss("invalid header: ", command));
+            return false;
+        }
+        parse_header(command, sv::trim(header));
+    }
+}
+
+bool HttpParser::parse_content_basic(std::string_view &view, bool closed) {
     if (content_length_ >= 0) {
         if (content_length_ > MAX_CONTENT_LENGTH) {
-            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            oversized();
             return false;
         }
         if (int(view.size()) < content_length_) {
@@ -651,7 +849,7 @@ bool HttpClientResponse::parse_content_basic(std::string_view &view, bool closed
         content_ = sv::read_nbytes(view, content_length_);
     } else {
         if (int64_t(view.size()) > MAX_CONTENT_LENGTH) {
-            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            oversized();
             return false;
         }
         if (!closed) {
@@ -663,7 +861,7 @@ bool HttpClientResponse::parse_content_basic(std::string_view &view, bool closed
     return true;
 }
 
-bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool closed) {
+bool HttpParser::parse_content_chunked(std::string_view &view, bool closed) {
     int64_t total_size = 0;
     std::vector<string_view> chunks;
     while (true) {
@@ -674,17 +872,17 @@ bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool clos
         }
         int64_t chunk_size = sv::to_hex64(chunk_header, -1);
         if (chunk_size < 0) {
-            fail(500, "internal server error: unparseable chunk header");
+            syntax("unparseable chunk header");
             return false;
         }
         if (chunk_size > MAX_CONTENT_LENGTH) {
-            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            oversized();
             return false;
         }
         if (chunk_size == 0) break;
         total_size += chunk_size;
         if (total_size > MAX_CONTENT_LENGTH) {
-            fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
+            oversized();
             return false;
         }
         std::string_view chunk = sv::read_nbytes(view, chunk_size);
@@ -694,7 +892,7 @@ bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool clos
         }
         std::string_view newline = sv::read_to_line(view);
         if (!newline.empty()) {
-            fail(500, "internal server error: corrupted chunk encoding");
+            syntax("corrupted chunk encoding");
             return false;
         }
         if (sv::isnull(view)) {
@@ -712,86 +910,29 @@ bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool clos
     return true;
 }
 
-void HttpClientResponse::parse(const StreamBuffer *sb, bool closed) {
+bool HttpParser::parse_content(std::string_view &view, bool closed) {
-    // We're not going to modify the StreamBuffer at all.
+    // Parse the content.
-    // Instead, we work entirely on a view.
-    string_view view = sb->view();
-
-    // Get the status line.
-    string_view status = sv::trim(sv::read_to_line(view));
-    if (sv::isnull(view)) {
-        incomplete(closed);
-        return;
-    }
-
-    // Parse the status line.
-    string_view protoversion = sv::read_to_space(status);
-    if (!sv::has_prefix(protoversion, "HTTP/")) {
-        fail(500, util::ss("internal server error: status line appears corrupt"));
-        return;
-    }
-    string_view scode = sv::read_to_space(status);
-    int64_t code = sv::to_int64(scode, 0);
-    if ((code < 100) || (code > 599)) {
-        fail(500, util::ss("internal server error: invalid response code: ", scode));
-        return;
-    }
-    status_code_ = code;
-
-    // Responses outside the range 200-299 are errors,
-    // and therefore must store an error message.
-    if ((code < 200) || (code > 299)) {
-        if (status.empty()) {
-            error_ = util::ss("error code ", code);
-        } else {
-            error_ = status;
-        }
-    }
-
-    // Parse the headers.
-    while (true) {
-        string_view header = sv::read_to_line(view);
-        if (sv::isnull(view)) {
-            incomplete(closed);
-            return;
-        }
-        if (header.empty()) {
-            break;
-        }
-        eng::string command = util::ascii_tolower(sv::trim(sv::read_to_sep(header, ':')));
-        if (sv::isnull(header)) {
-            fail(500, util::ss("internal server error: no colon in header line: ", command));
-            return;
-        }
-        if (!words_separated_by_dashes(command)) {
-            fail(500, util::ss("internal server error: invalid header: ", command));
-            return;
-        }
-        parse_header(command, sv::trim(header));
-    }
-
-    // Process the content using the transfer encoding.
     if (transfer_encoding_ == "") {
-        if (!parse_content_basic(view, closed)) return;
+        if (!parse_content_basic(view, closed)) {
-    } else if (transfer_encoding_ == "chunked") {
+            return false;
-        if (!parse_content_chunked(view, closed)) return;
-    } else {
-        fail(500, util::ss("unsupported transfer-encoding: ", transfer_encoding_));
-        return;
-    }
-
-    // Calculate the response length.
-    response_length_ = sb->fill() - view.size();
-
-    // If it's not a redirect, disallow 'location'.
-    if ((status_code_ < 300) || (status_code_ > 399)) {
-        if (!location_.empty()) {
-            fail(500, util::ss("internal server error: location specified, but result code not 300-399: ", code));
-            return;
         }
+    } else if (transfer_encoding_ == "chunked") {
+        if (!parse_content_chunked(view, closed)) {
+            return false;
+        }
+    } else {
+        syntax(util::ss("unsupported transfer-encoding: ", transfer_encoding_));
+        return false;
     }
 
-    // If the server didn't specify content-type, make a guess.
+    // Uncompress the content.
+    if ((content_encoding_ == "") || (content_encoding_ == "identity")) {
+    } else {
+        syntax(util::ss("content-encoding not supported: ", content_encoding_));
+        return true;
+    }
+
+    // If the sender didn't specify content-type, make a guess based on the content.
     if (mime_type_.empty()) {
         if (sv::valid_utf8(content_)) {
             mime_type_ = "text/plain";
@@ -802,56 +943,49 @@ void HttpClientResponse::parse(const StreamBuffer *sb, bool closed) {
         }
     }
 
-    // If it's multipart, reject it.
+    // Switch the charset to utf-8, if it's text.
-    if (sv::has_prefix(mime_type_, "multipart/")) {
-        fail(406, "not acceptable: multipart messages not supported");
-        return;
-    }
-
-    // If it's text, demand a reasonable charset.  Otherwise,
-    // ignore the charset.
     if (sv::has_prefix(mime_type_, "text/")) {
-        if (charset_.empty() || (charset_ == "ascii")) {
+        if (charset_.empty() || (charset_ == "ascii") || (charset_ == "utf-8")) {
-            charset_ = "utf-8";
+            // we're already good.
-        }
+        } else {
-        if (charset_ != "utf-8") {
+            // We can't convert charsets yet.
-            fail(406, util::ss("not acceptable: charset not supported: ", charset_));
+            syntax(util::ss("charset not supported: ", charset_));
-            return;
+            return true;
         }
     } else {
+        // Not text.  No need to specify charset.
         charset_.clear();
     }
-
+    return true;
-    // Uncompress the content.
-    if ((content_encoding_ == "") || (content_encoding_ == "identity")) {
-    } else {
-        fail(406, util::ss("not acceptable: content-encoding not supported: ", content_encoding_));
-        return;
-    }
-
-    // If there's an error code, throw out the content.
-    if ((status_code_ < 200) || (status_code_ > 299)) {
-        mime_type_.clear();
-        charset_.clear();
-        content_.clear();
-    }
 }
 
-void HttpClientResponse::store(LuaStack &LS0, LuaSlot tab) const {
+void HttpParser::store_parsed(LuaStack &LS0, LuaSlot tab) const {
-    LuaStack LS(LS0.state());
+    LuaVar ptab;
+    LuaStack LS(LS0.state(), ptab);
 
     LS.newtable(tab);
-    LS.rawset(tab, "responsecode", status_code_);
+    LS.rawset(tab, "status", status_);
     if (!error_.empty()) {
         LS.rawset(tab, "error", error_);
     }
     if (!location_.empty()) {
         LS.rawset(tab, "location", location_);
     }
-    if (!mime_type_.empty()) {
+    if (!mime_type_.empty() || !content_.empty()) {
         LS.rawset(tab, "mimetype", mime_type_);
         LS.rawset(tab, "content", content_);
     }
+    if (!method_.empty()) {
+        LS.rawset(tab, "method", method_);
+    }
+    if (!path_.empty()) {
+        LS.rawset(tab, "path", path_);
+        LS.newtable(ptab);
+        LS.rawset(tab, "params", ptab);
+        for (const auto &pair : params_) {
+            LS.rawset(ptab, pair.first, pair.second);
+        }
+    }
 
     // Debugging fields.  Do not use for lua programming.
     if (content_length_ >= 0) {
@@ -863,17 +997,170 @@ void HttpClientResponse::store(LuaStack &LS0, LuaSlot tab) const {
     if (!charset_.empty()) {
         LS.rawset(tab, "dbg_charset", charset_);
     }
-    if (response_length_ != 0) {
+    if (comm_length_ != 0) {
-        LS.rawset(tab, "dbg_responselength", response_length_);
+        LS.rawset(tab, "dbg_commlength", comm_length_);
     }
 }
 
+void HttpParser::parser_generate_debug_string(std::ostream &oss) const {
+    oss << "  status_code: " << status_ << std::endl;
+    oss << "  error: " << error_ << std::endl;
+    if (content_length_ >= 0) {
+        oss << "  content_length: " << content_length_ << std::endl;
+    }
+    if (!transfer_encoding_.empty()) {
+        oss << "  transfer_encoding: " << transfer_encoding_ << std::endl;
+    }
+    if (!location_.empty()) {
+        oss << "  location: " << location_ << std::endl;
+    }
+    if (!mime_type_.empty()) {
+        oss << "  mime_type: " << mime_type_ << std::endl;
+    }
+    if (!charset_.empty()) {
+        oss << "  charset: " << charset_ << std::endl;
+    }
+    if (!content_.empty()) {
+        oss << "  content: " << content_ << std::endl;
+    }
+    if (!method_.empty()) {
+        oss << "  method: " << method_ << std::endl;
+    }
+    if (!path_.empty()) {
+        oss << "  path: " << path_ << std::endl;
+    }
+    for (const auto &pair : params_) {
+        oss << "  param: " << pair.first << "=" << pair.second << std::endl;
+    }
+    if (comm_length_ > 0) {
+        oss << "  comm_length: " << comm_length_ << std::endl;
+    }
+}
+
+void HttpParser::clear_content_on_error() {
+    if ((status_ < 200) || (status_ > 299)) {
+        mime_type_.clear();
+        charset_.clear();
+        content_.clear();
+    }
+}
+
+HttpClientResponse::HttpClientResponse() {
+    request_id_ = 0;
+    is_request_ = false;
+}
+
+eng::string HttpClientResponse::DebugString() const {
+    eng::ostringstream oss;
+    oss << "HttpClientResponse:" << std::endl;
+    if (request_id_ != 0) {
+        oss << "  request_id: " << request_id_ << std::endl;
+    }
+    parser_generate_debug_string(oss);
+    return oss.str();
+}
+
+void HttpClientResponse::parse(std::string_view view, bool closed) {
+    std::string_view original_view = view;
+
+    // Parse the status line.
+    if (!parse_status_line(view, closed)) {
+        return;
+    }
+
+    // Parse the headers.
+    if (!parse_headers(view, closed)) {
+        return;
+    }
+
+    // Process the content.
+    if (!parse_content(view, closed)) {
+        return;
+    }
+
+    // Calculate the response length.
+    set_comm_length(original_view.size() - view.size());
+
+    // If it's not a redirect, ignore location.
+    if ((status_ < 300) || (status_ > 399)) {
+        location_.clear();
+    }
+
+    // If it's multipart, reject it.
+    if (sv::has_prefix(mime_type_, "multipart/")) {
+        syntax("multipart messages not supported");
+        return;
+    }
+
+    // If there's an error code, throw out the content.
+    clear_content_on_error();
+}
+
+void HttpClientResponse::store(LuaStack &LS, LuaSlot tab) const {
+    store_parsed(LS, tab);
+}
+
 void HttpClientResponse::store_fail(LuaStack &LS, LuaSlot tab, int status_code, std::string_view error) {
     HttpClientResponse response;
     response.fail(status_code, error);
     response.store(LS, tab);
 }
 
+HttpServerRequest::HttpServerRequest() {
+    is_request_ = true;
+}
+
+void HttpServerRequest::parse(std::string_view view, bool closed) {
+    std::string_view original_view = view;
+
+    // Parse the request line.
+    if (!parse_request_line(view, closed)) {
+        return;
+    }
+
+    // Parse the headers.
+    if (!parse_headers(view, closed)) {
+        return;
+    }
+
+    // Process the content, if any.
+    if (method_ == "POST") {
+        if (!parse_content(view, closed)) {
+            return;
+        }
+    }
+
+    // Calculate the comm length.
+    set_comm_length(original_view.size() - view.size());
+
+    // Always ignore location.
+    location_.clear();
+
+    // If it's multipart, reject it.
+    if (sv::has_prefix(mime_type_, "multipart/")) {
+        syntax("multipart messages not supported");
+        return;
+    }
+
+    // If we've made it this far, and there's no
+    // status code, set it to 200 OK.
+    if (status_ == 0) status_ = 200;
+
+    // If there's an error code, throw out the content.
+    clear_content_on_error();
+}
+
+void HttpServerRequest::store(LuaStack &LS, LuaSlot tab) const {
+    store_parsed(LS, tab);
+}
+
+eng::string HttpServerRequest::DebugString() const {
+    eng::ostringstream oss;
+    oss << "HttpServerRequest:" << std::endl;
+    parser_generate_debug_string(oss);
+    return oss.str();
+}
+
 LuaDefine(http_fixurl, "url", "validate URL and repair minor flaws in the URL syntax") {
     LuaArg url;
     LuaRet fixed;
@@ -888,8 +1175,8 @@ LuaDefine(http_fixurl, "url", "validate URL and repair minor flaws in the URL sy
 }
 
 
-LuaDefine(http_request, "request",
+LuaDefine(http_clientrequest, "request",
-    "|Takes an HTTP request in the form of a lua table."
+    "|Takes an HTTP client request in the form of a lua table."
     "|The table may contain these fields:"
     "|"
     "|  method             (ie, 'GET', 'POST', etc)"
@@ -914,7 +1201,7 @@ LuaDefine(http_request, "request",
     "|However, you can talk to a server that has a dummy certificate"
     "|by specifying verifycertificate=false."
     "|"
-    "|This routine, http.request, returns a debug string for the "
+    "|This routine, http.clientrequest, returns a debug string for the "
     "|request.  The debug string looks like the actual http headers"
     "|that would be sent.") {
     LuaArg tab;
@@ -932,15 +1219,15 @@ LuaDefine(http_request, "request",
     return LS.result();
 }
 
-LuaDefine(http_response, "response",
+LuaDefine(http_clientresponse, "response",
-    "|Returns an HTTP response in the form of a lua table."
+    "|Returns an HTTP client response in the form of a lua table."
     "|The table will contain these important fields:"
     "|"
-    "|  responsecode - 3-digit HTTP response code."
+    "|  status    - 3-digit HTTP response code."
-    "|  error        - an error message, or nil if no error."
+    "|  error     - an error message, or nil if no error."
-    "|  content      - on success, the content, as a string."
+    "|  content   - on success, the content, as a string."
-    "|  mimetype     - on success, the mime type of the content."
+    "|  mimetype  - on success, the mime type of the content."
-    "|  location     - for HTTP redirects, the target url."
+    "|  location  - for HTTP redirects, the target url."
     "|"
     "|If the mimetype is a text mimetype, then the content"
     "|is automatically converted to utf-8."
@@ -950,7 +1237,7 @@ LuaDefine(http_response, "response",
     "|  dbg_transferencoding - If there was a Transfer-Encoding header."
     "|  dbg_contentlength    - If there was a Content-length header."
     "|  dbg_charset          - Original character set for text mime types."
-    "|  dbg_responselength   - Total bytes in the response."
+    "|  dbg_commlength       - Total bytes in the communication."
     "|"
     "|None of the dbg fields is needed to understand the response."
     "|For example, consider dbg_charset.  When text content is"
@@ -965,23 +1252,84 @@ LuaDefine(http_response, "response",
     "|   400 (bad request)          - the request was malformed"
     "|   503 (service unavailable)  - dns fail, connect fail, or ssl fail"
     "|   500 (internal server error)- the response contains invalid HTTP"
-    "|   406 (not acceptable)       - the response used a feature we don't support yet"
     "|   413 (payload too large)    - we refuse to download something so big"
     "|   425 (can't resume)         - reloaded a save game with a pending request"
     "|"
-    "|In this case, the error message will be the stock"
+    "|Error messages that are generated locally consist of "
-    "|error message, followed by more information."
+    "|the standard message (eg, 'bad request') followed by more "
+    "|detailed information."
     "|"
-    "|This routine, http.response, generates a response by parsing"
+    "|This routine, http.clientresponse, generates a response by parsing"
     "|an actual HTTP response string.  This is for debugging only.") {
     LuaArg text;
     LuaRet tab;
     LuaStack LS(L, text, tab);
     HttpClientResponse resp;
-    StreamBuffer sb;
+    resp.parse(LS.ckstring(text), true);
-    sb.write_bytes(LS.ckstring(text));
-    resp.parse(&sb, true);
     resp.store(LS, tab);
     return LS.result();
 }
 
+LuaDefine(http_serverrequest, "request",
+    "|Returns an HTTP server request in the form of a lua table."
+    "|The table will contain these important fields:"
+    "|"
+    "|  status    - 3-digit HTTP response code."
+    "|  error     - an error message, or nil if no error."
+    "|  method    - GET, HEAD, or POST"
+    "|  path      - the url-decoded path, eg, '/index.html'"
+    "|  params    - a table of url-decoded URL parameters"
+    "|  content   - the content, as a string (POST only)"
+    "|  mimetype  - the mime type of the content (POST only)"
+    "|  location  - for HTTP redirects, the target url."
+    "|"
+    "|If the mimetype is a text mimetype, then the content"
+    "|is automatically converted to utf-8."
+    "|"
+    "|The table may also contain these debugging-only fields."
+    "|"
+    "|  dbg_transferencoding - If there was a Transfer-Encoding header."
+    "|  dbg_contentlength    - If there was a Content-length header."
+    "|  dbg_charset          - Original character set for text mime types."
+    "|  dbg_commlength       - Total bytes in the communication."
+    "|"
+    "|None of the dbg fields is needed to understand the request."
+    "|For example, consider dbg_charset.  When text content is"
+    "|passed to lua, the content is automatically converted to utf-8."
+    "|So dbg_charset only tells you what the character set used"
+    "|to be, before it was converted to utf-8."
+    "|"
+    "|When the engine is functioning as a webserver, bad requests "
+    "|are never passed to lua.  Therefore, a request that is passed "
+    "|to lua will always contain status=200 and error=nil.  However, "
+    "|when debugging server requests using http.serverrequest, "
+    "|it is possible to see certain other errors:"
+    "|"
+    "|   400 (bad request)          - the request was malformed"
+    "|   503 (service unavailable)  - dns fail, connect fail, or ssl fail"
+    "|   500 (internal server error)- the response contains invalid HTTP"
+    "|   413 (payload too large)    - we refuse to download something so big"
+    "|   425 (can't resume)         - reloaded a save game with a pending request"
+    "|"
+    "|Error messages that are generated locally consist of "
+    "|the standard message (eg, 'bad request') followed by more "
+    "|detailed information."
+    "|"
+    "|This routine, http.serverrequest, generates a request by parsing"
+    "|an actual HTTP request string.  This is for debugging only.") {
+    LuaArg text;
+    LuaRet tab;
+    LuaStack LS(L, text, tab);
+    HttpServerRequest req;
+    req.parse(LS.ckstring(text), true);
+    req.store(LS, tab);
+    return LS.result();
+}
+
+LuaDefine(http_validmime, "(mt)", "") {
+    LuaArg str;
+    LuaRet ok;
+    LuaStack LS(L, str, ok);
+    LS.set(ok, valid_mime_type(LS.ckstring(str)));
+    return LS.result();
+}

luprex/core/cpp/http.hpp

@@ -20,6 +20,7 @@
 #include "luastack.hpp"
 #include "streambuffer.hpp"
 #include "drivenengine.hpp"
+#include <ostream>
 
 using UrlParameters = eng::map<eng::string, eng::string>;
 
@@ -38,7 +39,7 @@ private:
     // True is the default.
     bool verify_certificate_;
 
-    // Method: GET, HEAD, POST, etc.  Must be all-caps.
+    // Method: GET, HEAD, or POST.  Must be all-caps.
     eng::string method_;
 
     // The hostname. Not yet url-encoded.
@@ -53,6 +54,12 @@ private:
     // URL parameters to append to the path.  Not yet url-encoded.
     UrlParameters params_;
 
+    // The mime type of the content, only for POST requests.
+    eng::string mime_type_;
+
+    // The content as a string, only for POST requests.
+    eng::string content_;
+
 private:
     void fail(std::string_view error);
     void send_internal(StreamBuffer *target, bool debug_string) const;
@@ -70,6 +77,8 @@ public:
     const eng::string &host() const { return host_; }
     int port() const { return port_; }
     const eng::string &path() const { return path_; }
+    const eng::string &mime_type() const { return mime_type_; }
+    const eng::string &content() const { return content_; }
     
     // Populate an request-related fields one piece at a time.
     // If you pass an invalid value, or if the field is
@@ -85,6 +94,8 @@ public:
     void set_path(std::string_view path);
     void set_param(const eng::string &key, const eng::string &value);
     void set_url(std::string_view url);
+    void set_mime_type(const eng::string &mime_type);
+    void set_content(const eng::string &content);
  
     void set_verify_certificate(LuaStack &LS, LuaSlot val);
     void set_method(LuaStack &LS, LuaSlot val);
@@ -94,6 +105,8 @@ public:
     void set_param(LuaStack &LS, LuaSlot key, LuaSlot val);
     void set_params(LuaStack &LS, LuaSlot tab);
     void set_url(LuaStack &LS, LuaSlot val);
+    void set_mime_type(LuaStack &LS, LuaSlot val);
+    void set_content(LuaStack &LS, LuaSlot val);
 
     // Set default values for method and port.
     // This must be done after setting regular values.
@@ -129,18 +142,21 @@ public:
     eng::string DebugString();
 };
 
-class HttpClientResponse {
+// HttpParser is used for parsing both requests and responses.
-private:
+// Stores a status code, an error message, headers, and content.
-    // The request ID.
+//
-    int64_t request_id_;
+class HttpParser : public eng::nevernew {
+protected:
+    // True if this is for parsing a request.
+    // This is only used for generating error messages.
+    bool is_request_;
 
-    // The HTTP response status code.
+    // The status code, a 3-digit number.
-    int status_code_;
+    int status_;
 
-    // If the HTTP response contains an error, the
+    // If the communication contains an error, the error can
-    // error message is stored here.  If the HTTP response
+    // be stored here.  In the event of a successful communication,
-    // is a success such as "200 OK" or "201 Created", this
+    // this should always be empty string.
-    // is the empty string, not "OK" or "Created".
     eng::string error_;
 
     // Only if content-length header present, otherwise, -1.
@@ -158,50 +174,144 @@ private:
     // MIME type of the content.
     eng::string mime_type_;
 
-    // Charset of the content.  Hopefully utf-8.
+    // Charset of the content before the content was translated to utf-8.
     eng::string charset_;
 
-    // The content as string.
+    // The method, GET, HEAD, or POST. (only when parsing requests)
+    eng::string method_;
+
+    // The URL path, not url-encoded. (only when parsing requests)
+    eng::string path_;
+
+    // The URL parameters, not url-encoded (only when parsing requests)
+    UrlParameters params_;
+
+    // The content as string.  If it's text, it's been translated to utf-8.
     eng::string content_;
 
-    // The length in bytes of the entire response.
+    // The length in bytes of the entire communication.
-    // May be zero, which means that the response
+    // If the response is complete, but the comm_length_
-    // was so garbled that we couldn't determine the length.
+    // is zero, it means we couldn't find the end of 
-    int response_length_;
+    // the request.
-    
+    int comm_length_;
-private:
+
+protected:
     // Store a message indicating that we haven't received enough
     // bytes yet.  If the connection is closed and we still haven't
     // received enough bytes, that's a fatal error.
+    //
     void incomplete(bool closed);
 
-    // Parse a response header.  Most headers are ignored.
+    // Store a message indicating that we couldn't parse because
-    // If the header contains an error, the error is stored.
+    // something was syntactically malformed, ie, not valid HTTP.
-    void parse_header(std::string_view header, std::string_view value);
+    //
+    void syntax(std::string_view detail);
+
+    // Store a message indicating that the content was too large
+    // and we refused to download it.
+    //
+    void oversized();
+
+    // If the status is not in the range 200-299 (OK),
+    // then discard the content.
+    void clear_content_on_error();
+
+    // Parse a response status line, such as "HTTP/1.1 200 OK"
+    // returns false if we couldn't parse the whole line.
+    //
+    bool parse_status_line(std::string_view &view, bool closed);
+
+    // Parse a request line, such as "GET /index.html HTTP/1.1"
+    // returns false if we couldn't parse the whole line.
+    //
+    bool parse_request_line(std::string_view &view, bool closed);
 
     // Parse specific headers.
-    // For several headers, all we do is verify that they aren't
+    //
-    // invoking unsupported features.
     void parse_content_encoding(std::string_view value);
     void parse_content_length(std::string_view value);
     void parse_content_type(std::string_view value);
     void parse_location(std::string_view value);
     void parse_transfer_encoding(std::string_view value);
 
-    // parse the body
+    // Parse a single response header.  Most headers are ignored.
+    // If the header contains an error, the error is stored.
+    //
+    void parse_header(std::string_view header, std::string_view value);
+
+    // Parse all of the headers, including the blank line after
+    // the headers.  Return true if the view is at the end of the headers.
+    //
+    bool parse_headers(std::string_view &view, bool closed);
+    
+    // parse the content, for different transfer encodings.
+    // Returns true if the view is at the end of the content.
+    //
     bool parse_content_basic(std::string_view &view, bool closed);
     bool parse_content_chunked(std::string_view &view, bool closed);
+
+    // Parse the content.  Return true if the view is at the end of the
+    // content.
+    //
+    // - Uses the appropriate transfer-encoding specified in the headers.
+    // - Decompresses the content using the appropriate content-encoding.
+    // - Guesses a mimetype and charset if one was not specified in headers.
+    // - If it's text, converts the charset to utf-8.
+    //
+    bool parse_content(std::string_view &view, bool closed);
+
+    // Get or set the communication length.
+    //
+    int comm_length() const { return comm_length_; }
+    void set_comm_length(int len) { comm_length_ = len; }
+
+    // Construct a blank parser.
+    //
+    HttpParser();
+
+    // Store the parsed fields into a lua table.
+    //
+    void store_parsed(LuaStack &LS, LuaSlot tab) const;
+
+    // Emit all the parser fields as a debug string.
+    //
+    void parser_generate_debug_string(std::ostream &oss) const;
+
 public:
+    // The parser will not try to parse content longer than this.
+    //
     const int64_t MAX_CONTENT_LENGTH = 1000000;
 
-    // Construct a blank response.
+    // Store a status code and an error message, and clear the content.
-    HttpClientResponse();
-
-    // Store a result code and an error message, and clear the content.
     // This is generally used when the client detects an error,
     // such as a DNS lookup fail, a connection failed, an SSL negotiation
     // failed, or the like.
-    void fail(int status_code, std::string_view error);
+    //
+    void fail(int status, std::string_view error);
+
+    // Return true if the communication was complete.
+    //
+    bool complete() const { return status_ != 0; }
+};
+
+class HttpClientResponse : public HttpParser {
+private:
+    // Most of the data is stored in the HttpParser.
+    // That includes the status code, error, headers, and content.
+    
+    // The request ID.
+    int64_t request_id_;
+
+    // The length in bytes of the entire communication.
+    // If the response is complete, but the comm_length_
+    // is zero, it means we couldn't find the end of 
+    // the request.
+    //
+    int comm_length_;
+
+public:
+    // Construct a blank response.
+    HttpClientResponse();
 
     // Parse the HTTP response.  The closed flag is to be set to true if the
     // remote has closed the connection.
@@ -209,28 +319,62 @@ public:
     // If the request is incomplete, generates a status code of zero. In that
     // case, loading more data from the server might improve the situation.
     //
-    // Note that the response is not ever removed from the StreamBuffer, which
+    // If successful, indicates how much of the text was consumed by
-    // is always unmodified. If you want to remove the response from the
+    // setting comm_length_.
-    // StreamBuffer, see response_length.
     //
-    void parse(const StreamBuffer *sb, bool closed);
+    void parse(std::string_view text, bool closed);
-
-    // Return true if the response is complete.
-    bool complete() const { return status_code_ != 0; }
 
     // Get or Set the request ID.
     // This class does nothing with the request ID, it just stores it.
+    //
     int64_t request_id() const { return request_id_; }
     void set_request_id(int64_t v) { request_id_ = v; }
     
     // Convert the HTTP response to a lua table.
+    //
     void store(LuaStack &LS, LuaSlot tab) const;
 
     // Convert to a debug string.
+    //
     eng::string DebugString() const;
 
     // Synthesize an error response and store it in lua.
-    static void store_fail(LuaStack &LS, LuaSlot tab, int status_code, std::string_view error);
+    //
+    static void store_fail(LuaStack &LS, LuaSlot tab, int status, std::string_view error);
+};
+
+class HttpServerRequest : public HttpParser {
+private:
+    // The length in bytes of the entire communication.
+    // If the request is complete, but the comm_length_
+    // is zero, it means we couldn't find the end of 
+    // the request.
+    //
+    int comm_length_;
+
+public:
+    // Construct a blank request.
+    //
+    HttpServerRequest();
+
+    // Parse the HTTP request.  The closed flag is to be set to true if the
+    // remote has closed the connection.
+    //
+    // If the request is incomplete, generates a status code of zero. In that
+    // case, loading more data from the server might improve the situation.
+    //
+    // If successful, indicates how much of the text was consumed by
+    // setting comm_length.
+    //
+    void parse(std::string_view text, bool closed);
+    
+    // Convert the HTTP request to a lua table.
+    //
+    void store(LuaStack &LS, LuaSlot tab) const;
+
+    // Convert to a debug string.
+    //
+    eng::string DebugString() const;
 };
 
 class HttpClientRequestMap : public eng::map<int64_t, HttpClientRequest> {

luprex/core/cpp/lpxserver.cpp

@@ -225,7 +225,7 @@ public:
                     response.fail(503, util::ss("Service Unavailable: ", channel.error()));
                 } else {
                     htchan.parsed_bytes_ = channel.in()->fill();
-                    response.parse(channel.in(), channel.closed());
+                    response.parse(channel.in()->view(), channel.closed());
                 }
                 if (response.complete()) {
                     response.set_request_id(pair.first);

luprex/core/cpp/util.cpp

@@ -218,6 +218,19 @@ string_view read_nbytes(string_view &source, int nbytes) {
     return result;
 }
 
+string_view read_ascii_identifier(string_view &source) {
+    size_t len = 0;
+    if ((len < source.size()) && (sv::ascii_isalpha(source[len]))) {
+        len += 1;
+        while ((len < source.size()) && (sv::ascii_isalnum(source[len]))) {
+            len += 1;
+        }
+    }
+    string_view result = source.substr(0, len);
+    source.remove_prefix(len);
+    return result;
+}
+
 bool valid_utf8(string_view s)
 {
     const unsigned char *bytes = (const unsigned char *)s.data();

luprex/core/cpp/util.hpp

@@ -126,6 +126,12 @@ string_view read_to_space(string_view &source);
 //
 string_view read_nbytes(string_view &source, int nbytes);
 
+// Read an ascii identifier from a string_view
+//
+// If there's no valid identifier, returns empty string.
+//
+string_view read_ascii_identifier(string_view &source);
+
 // Return true if the string is valid utf-8.
 bool valid_utf8(string_view s);
 

luprex/core/cpp/world-accessor.cpp

@@ -525,7 +525,7 @@ LuaDefine(doc, "function",
 
 LuaDefine(http_get, "request",
     "|Make an HTTP GET request.  Returns an HTTP response."
-    "|See doc(http.request) and doc(http.response).") {
+    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
     World *w = World::fetch_global_pointer(L);
     w->guard_blockable(L, "http.get");