HTTP now automatically encodes jsonvalue content

luprex/core/cpp/http.cpp

@@ -9,6 +9,7 @@
 #include "wrap-string.hpp"
 #include "util.hpp"
 #include "luastack.hpp"
+#include "json.hpp"
 
 #include <cstdint>
 
@@ -319,6 +320,38 @@ static void send_error_response(bool http11, int code, eng::string extrainfo, St
     }
 }
 
+// HTTP 1.0: An entity body is included with a request message only when the request method calls for one.
+// The presence of an entity body in a request is signaled by the inclusion of a Content-Length header field
+// in the request message headers. HTTP/1.0 requests containing an entity body must include a valid
+// Content-Length header field.
+//
+// HTTP 1.1: The presence of a message-body in a request is signaled by the inclusion of a Content-Length or
+// Transfer-Encoding header field in the request's message-headers
+//
+static bool request_contains_content(int content_length, std::string_view transfer_encoding) {
+    return (content_length >= 0) || (!transfer_encoding.empty());
+}
+
+// HTTP 1.0: For response messages, whether or not an entity body is included with a message is dependent on
+// both the request method and the response code. All responses to the HEAD request method must not include a
+// body, even though the presence of entity header fields may lead one to believe they do. All 1xx (informational),
+// 204 (no content), and 304 (not modified) responses must not include a body. All other responses must include an
+// entity body or a Content-Length header field defined with a value of zero (0).
+//
+// HTTP 1.1: For response messages, whether or not a message-body is included with a message is dependent on both
+// the request method and the response status code (section 6.1.1). All responses to the HEAD request method MUST
+// NOT include a message-body, even though the presence of entity- header fields might lead one to believe they
+// do. All 1xx (informational), 204 (no content), and 304 (not modified) responses MUST NOT include a message-body.
+// All other responses do include a message-body, although it MAY be of zero length.
+//
+static bool response_contains_content(std::string_view method, int status) {
+    if (method == "HEAD") return false;
+    if ((status >= 100) && (status <= 199)) return false;
+    if (status == 204) return false;
+    if (status == 304) return false;
+    return true;
+}
+
 // In a properly-formed url, the hostname and path are url encoded.
 // This parser expects an encoded URL.
 struct ParsedURL {
@@ -451,8 +484,8 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
         sb->write_bytes("\r\n");
     }
 
-    // If it's a post request, send the content length and the content type.
+    // Send the content length and the content type.
-    if (method_ == "POST") {
+    if (content_assigned_) {
         send_content_length_header(true, content_.size(), sb);
         send_content_type_header(true, mime_type_, sb);
     }
@@ -460,8 +493,8 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
     // Send the extra linebreak.
     sb->write_bytes("\r\n");
 
-    // If it's a post request, send the content.
+    // Send the content.
-    if (method_ == "POST") {
+    if (content_assigned_) {
         sb->write_bytes(content_);
     }
 }
@@ -681,6 +714,17 @@ void HttpClientRequest::set_content(LuaStack &LS, LuaSlot val) {
     set_content(LS.ckstring(val));
 }
 
+void HttpClientRequest::set_jsonvalue(LuaStack &LS, LuaSlot val) {
+    eng::string out;
+    eng::string err = json::encode(LS, val, out, false, HttpParser::MAX_CONTENT_LENGTH);
+    if (!err.empty()) {
+        check_fail(util::ss("json encode failure: ", err));
+        return;
+    }
+    set_content(out);
+    set_mime_type("application/json");
+}
+
 void HttpClientRequest::set_defaults() {
     if (method_.empty()) {
         method_ = "GET";
@@ -715,6 +759,20 @@ void HttpClientRequest::set_config(LuaStack &LS0, LuaSlot tab) {
             set_mime_type(LS, val);
         } else if (kstr == "content") {
             set_content(LS, val);
+        } else if (kstr == "html") {
+            set_content(LS, val);
+            set_mime_type("text/html");
+        } else if (kstr == "text") {
+            set_content(LS, val);
+            set_mime_type("text/plain");
+        } else if (kstr == "json") {
+            set_content(LS, val);
+            set_mime_type("application/json");
+        } else if (kstr == "bytes") {
+            set_content(LS, val);
+            set_mime_type("application/octet-stream");
+        } else if (kstr == "jsonvalue") {
+            set_jsonvalue(LS, val);
         } else if (kstr == "") {
             check_fail(util::ss("configuration parameter names must be strings."));
         } else {
@@ -755,13 +813,6 @@ eng::string HttpClientRequest::check() const {
         if (mime_type_.empty()) {
             return "mime type not set for POST request";
         }
-    } else {
-        if (content_assigned_) {
-            return "content can only be specified for POST requests";
-        }
-        if (!mime_type_.empty()) {
-            return "mime type can only be specified for POST requests";
-        }
     }
     return "";
 }
@@ -822,20 +873,23 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons
         return;
     }
 
+    // Determine if we're going to be sending content.
+    bool contains_content = response_contains_content("GET", status_);
+
     // The status message is the human-readable status code.
     eng::string statusmsg = status_code_to_string(status_);
 
     // Annotate error messages.
     const eng::string *content = &content_;
-    if (errstatus() && (mime_type_ == "text/plain") &&
+    if (errstatus() && contains_content &&
-        !content_.empty() && !contains_newline(content_)) {
+        (mime_type_ == "text/plain") && !contains_newline(content_)) {
         if (sv::has_prefix(content_, statusmsg)) {
             statusmsg = content_;
-        } else {
+        } else if (!content_.empty()) {
             statusmsg += ": ";
             statusmsg += content_;
-            content = &statusmsg;
         }
+        content = &statusmsg;
     }
 
     // Send the status line.
@@ -848,7 +902,7 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons
     }
 
     // Send the headers that make sense if we're sending content.
-    if (content_assigned_) {
+    if (contains_content) {
         if (!errstatus()) {
             send_cache_control_header(http11_, max_age_, sb);
         }
@@ -860,7 +914,7 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons
     sb->write_bytes("\r\n");
 
     // Send the content.
-    if (content_assigned_) {
+    if (contains_content) {
         sb->write_bytes(*content);
     }
 }
@@ -960,6 +1014,17 @@ void HttpServerResponse::set_content(LuaStack &LS, LuaSlot val) {
     set_content(LS.ckstring(val));
 }
 
+void HttpServerResponse::set_jsonvalue(LuaStack &LS, LuaSlot val) {
+    eng::string out;
+    eng::string err = json::encode(LS, val, out, false, HttpParser::MAX_CONTENT_LENGTH);
+    if (!err.empty()) {
+        check_fail(util::ss("json encode failure: ", err));
+        return;
+    }
+    set_content(out);
+    set_mime_type("application/json");
+}
+
 void HttpServerResponse::set_config(LuaStack &LS0, LuaSlot tab) {
     LuaVar key, val;
     LuaStack LS(LS0.state(), key, val);
@@ -987,6 +1052,8 @@ void HttpServerResponse::set_config(LuaStack &LS0, LuaSlot tab) {
         } else if (kstr == "bytes") {
             set_content(LS, val);
             set_mime_type("application/octet-stream");
+        } else if (kstr == "jsonvalue") {
+            set_jsonvalue(LS, val);
         } else if (kstr == "") {
             check_fail(util::ss("response configuration parameters must be strings."));
         } else {
@@ -1010,13 +1077,13 @@ void HttpServerResponse::set_defaults() {
         status_ = 200;
     }
 
-    // If you're sending an error message along with
+    // If the response status indicates that we're sending
-    // an error status code, then assume the error
+    // content, and there's no content, generate blank content.
-    // message is text/plain.
+    if (response_contains_content("GET", status_) &&
-    if ((status_ >= 400) && (status_ <= 599)) {
+        (!content_assigned_) && (mime_type_.empty())) {
-        if (content_assigned_ && (mime_type_.empty())) {
+        content_assigned_ = true;
-            mime_type_ = "text/plain";
+        content_ = "";
-        }
+        mime_type_ = "text/plain";
     }
 }
 
@@ -1030,15 +1097,16 @@ eng::string HttpServerResponse::check() const {
         return "status code not specified";
     }
 
-    // If you specify content, you have to specify mime
+    // If you assigned a mime type and didn't specify
-    // type, and vice versa.  Also, we need both for a valid response.
+    // content, that's bad.
-    if ((status_ == 200) || (content_assigned_) || (!mime_type_.empty())) {
+    if ((!content_assigned_) && (!mime_type_.empty())) {
-        if (!content_assigned_) {
+        return "mime type specified without content";
-            return "content not specified";
+    }
-        }
+
-        if (mime_type_.empty()) {
+    // If you assigned content, but didn't assign
-            return "mime type not specified";
+    // a mime type, that's bad.
-        }
+    if (content_assigned_ && (mime_type_.empty())) {
+        return "content specified without mime type";
     }
     return "";
 }
@@ -1337,6 +1405,17 @@ bool HttpParser::parse_content_chunked(std::string_view &view, bool closed) {
 }
 
 bool HttpParser::parse_content(std::string_view &view, bool closed) {
+    // If there's no body, just return true.
+    if (is_request_) {
+        if (!request_contains_content(content_length_, transfer_encoding_)) {
+            return true;
+        }
+    } else {
+        if (!response_contains_content(method_, status_)) {
+            return true;
+        }
+    }
+
     // Parse the content.
     if (transfer_encoding_ == "") {
         if (!parse_content_basic(view, closed)) {
@@ -1386,8 +1465,8 @@ bool HttpParser::parse_content(std::string_view &view, bool closed) {
 }
 
 void HttpParser::store(LuaStack &LS0, LuaSlot tab) const {
-    LuaVar ptab;
+    LuaVar ptab, djson;
-    LuaStack LS(LS0.state(), ptab);
+    LuaStack LS(LS0.state(), ptab, djson);
 
     LS.newtable(tab);
     if (!is_request_) {
@@ -1402,6 +1481,10 @@ void HttpParser::store(LuaStack &LS0, LuaSlot tab) const {
     if (!mime_type_.empty() || !content_.empty()) {
         LS.rawset(tab, "mimetype", mime_type_);
         LS.rawset(tab, "content", content_);
+        if (mime_type_ == "application/json") {
+            json::decode(LS, djson, content_);
+            LS.rawset(tab, "jsonvalue", djson);
+        }
     }
     // We currently don't store the method, because
     // our server only supports GET.  Even if we
@@ -1475,9 +1558,10 @@ eng::string HttpParser::debug_string() const {
     return oss.str();
 }
 
-void HttpParser::parse_response(std::string_view view, bool closed) {
+void HttpParser::parse_response(std::string_view view, bool closed, std::string_view method) {
     std::string_view original_view = view;
     is_request_ = false;
+    method_ = method;
 
     // Parse the status line.
     if (!parse_status_line(view, closed)) {
@@ -1524,10 +1608,8 @@ void HttpParser::parse_request(std::string_view view, bool closed) {
     }
 
     // Process the content, if any.
-    if (method_ == "POST") {
+    if (!parse_content(view, closed)) {
-        if (!parse_content(view, closed)) {
+        return;
-            return;
-        }
     }
 
     // Calculate the comm length.
@@ -1594,8 +1676,22 @@ LuaDefine(http_clientrequest, "request",
     "|  path               (ie, '/index.html')"
     "|  params             (a table of url parameters)"
     "|  verifycertificate  (default: true)"
+    "|  content            (the body to send to the server, if any)"
+    "|  mimetype           (mime type of the body, if any)"
+    "|"
+    "|The table can also contain this field, which sets"
+    "|host, port, path, and params in a single directive."
+    "|"
     "|  url                (ie, 'https://host:port/path.html?a=b&c=d')"
     "|"
+    "|The table can also contain these fields, which set"
+    "|both the content and the mimetype in a single directive:"
+    "|"
+    "|  html      - content as a string (text/html)"
+    "|  text      - content as a string (text/plain)"
+    "|  json      - content as a string (application/json)"
+    "|  bytes     - content as a string (application/octet-stream)"
+    "|"
     "|You can specify url components separately (host, port, path,"
     "|and params), or you can specify the entire url as a unit. "
     "|If you specify components, they must not be url-encoded. "
@@ -1674,7 +1770,7 @@ LuaDefine(http_clientresponse, "response",
     LuaRet tab;
     LuaStack LS(L, text, tab);
     HttpParser parser;
-    parser.parse_response(LS.ckstring(text), true);
+    parser.parse_response(LS.ckstring(text), true, "GET");
     parser.store(LS, tab);
     return LS.result();
 }

luprex/core/cpp/http.hpp

@@ -97,6 +97,7 @@ public:
     void set_url(LuaStack &LS, LuaSlot val);
     void set_mime_type(LuaStack &LS, LuaSlot val);
     void set_content(LuaStack &LS, LuaSlot val);
+    void set_jsonvalue(LuaStack &LS, LuaSlot val);
 
     // Set default values for method and port.
     // This must be done after setting regular values.
@@ -126,6 +127,10 @@ public:
     //
     eng::string check() const;
 
+    // Get the method.
+    //
+    eng::string method() const { return method_; }
+
     // Put the request into the stream, assuming HTTP/1.1
     //
     void send(StreamBuffer *target) const { send_internal(target, false); }
@@ -186,6 +191,7 @@ public:
     //
     HttpServerResponse();
 
+    void set_method(const eng::string &method);
     void set_status(int status);
     void set_max_age(int max_age);
     void set_mime_type(const eng::string &mime_type);
@@ -195,6 +201,7 @@ public:
     void set_max_age(LuaStack &LS, LuaSlot val);
     void set_mime_type(LuaStack &LS, LuaSlot val);
     void set_content(LuaStack &LS, LuaSlot val);
+    void set_jsonvalue(LuaStack &LS, LuaSlot val);
 
     // Set default values.
     //
@@ -275,7 +282,7 @@ protected:
     // The protocol: true for HTTP/1.1, false for HTTP/1.0
     bool http11_;
 
-    // The method: always "GET". (only when parsing requests)
+    // The method.  In a response, this is assigned before parsing.
     eng::string method_;
 
     // The URL path, not url-encoded. (only when parsing requests)
@@ -383,7 +390,7 @@ public:
 
     // The parser will not try to parse content longer than this.
     //
-    const int64_t MAX_CONTENT_LENGTH = 1000000;
+    static constexpr int64_t MAX_CONTENT_LENGTH = 1000000;
 
     // Parse a request or a response.
     //
@@ -391,7 +398,7 @@ public:
     // construct a new parser.
     //
     void parse_request(std::string_view view, bool closed);
-    void parse_response(std::string_view view, bool closed);
+    void parse_response(std::string_view view, bool closed, std::string_view method);
 
     // Store a status code and an error message, and clear the content.
     // This is generally used when the client detects an error,
@@ -428,6 +435,7 @@ using HttpParserVec = eng::vector<HttpParser>;
 class HttpChannel {
 public:
     SharedChannel channel_;
+    eng::string method_;
     int64_t parsed_bytes_;
 
     bool marked_for_deletion() const { return channel_ == nullptr; }

luprex/core/cpp/json.cpp

@@ -13,6 +13,7 @@
 
 LuaTokenConstant(json_null, "null", "");
 LuaTokenConstant(json_object, "object", "");
+LuaTokenConstant(json_error, "error", "");
 
 static void indent(eng::ostringstream &oss, int level) {
     if (level < NOINDENT_LEVEL) {
@@ -543,23 +544,26 @@ bool decode(LuaStack &LS, LuaSlot out, std::string_view v) {
     lua_State *L = LS.state();
 
     // Try to read a single value from the view.
+    int top = lua_gettop(L);
     bool ok = decode_value(L, v);
     lua_replace(L, out.index());
-    if (!ok) return false;
+    lua_settop(L, top);
-
+    if (!ok) {
-    // There should be nothing left of the input text.
+        LS.set(out, LuaToken("error"));
-    if (v.size() > 0) {
-        lua_pushnil(L);
-        lua_replace(L, out.index());
         return false;
     }
 
-    // Special case: if the top-level result is jsonnull,
+    // Special case: if the top level value is 'null', change
-    // then change it to nil.
+    // it to 'nil.'
     if (LS.istoken(out)) {
         LS.set(out, LuaNil);
     }
 
+    // There should be nothing left of the input text.
+    if (v.size() > 0) {
+        LS.set(out, LuaToken("error"));
+        return false;
+    }
     return true;
 }
 

luprex/core/cpp/json.hpp

@@ -24,7 +24,8 @@ namespace json {
     // See doc(http.jsondecode) for a lot more information.
     //
     // The only error condition is syntactically invalid json.
-    // In that case, we return false.
+    // In that case, we return false and set 'out' to the
+    // token 'error'.
     //
     bool decode(LuaStack &LS, LuaSlot out, std::string_view in);
 }

luprex/core/cpp/lpxserver.cpp

@@ -219,6 +219,7 @@ public:
             HttpChannel &channel = http_client_channels_[request.request_id()];
             if (channel.channel_ == nullptr) {
                 channel.channel_ = new_outgoing_channel(request.target());
+                channel.method_ = request.method();
                 channel.parsed_bytes_ = 0;
                 request.send(channel.channel_->out());
             }
@@ -235,7 +236,7 @@ public:
                     response.fail(503, util::ss("Service Unavailable: ", channel.error()));
                 } else {
                     htchan.parsed_bytes_ = channel.in()->fill();
-                    response.parse_response(channel.in()->view(), channel.closed());
+                    response.parse_response(channel.in()->view(), channel.closed(), htchan.method_);
                 }
                 if (response.complete()) {
                     response.set_request_id(pair.first);

luprex/core/cpp/util.hpp

@@ -344,12 +344,15 @@ inline eng::string ss(const ARGS & ... args) {
     return oss.str();
 }
 
-// This is a better way to do std::setfill, std::hex, std::setprecision
+// A better API than std::setfill, std::hex, std::setw, std::setprecision
 //
 // Usage examples:
 //   std::cout << util::hex.width(5).fill('0').val(123)
 //   std::cout << util::dec.fill('$').precision(val(123)
 //
+// The reason that other API is bad is that it can leave std::cout
+// in an unpredictable state.  This API always leaves the stream clean.
+//
 template <class VALUE>
 class FormattedNumber {
 public:

luprex/core/cpp/world-accessor.cpp

@@ -654,9 +654,7 @@ LuaDefine(doc, "function",
     return LS.result();
 }
 
-LuaDefine(http_get, "request",
+int lfn_http_request(lua_State *L, const char *method) {
-    "|Make an HTTP GET request.  Returns an HTTP response."
-    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
     World *w = World::fetch_global_pointer(L);
     w->guard_blockable(L, "http.get");
 
@@ -667,7 +665,7 @@ LuaDefine(http_get, "request",
     
     // Parse the request and make sure it's valid.
     // If not, immediately pass a '400 bad request' back to lua.
-    req.set_method("GET");
+    req.set_method(method);
     req.set_config(LS, request);
     req.set_defaults();
     eng::string error = req.check();
@@ -686,4 +684,22 @@ LuaDefine(http_get, "request",
 
     // Block.
     return lua_yield(L, 0);
-}
+}
+
+LuaDefine(http_get, "request",
+    "|Make an HTTP GET request.  Returns an HTTP response."
+    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
+    return lfn_http_request(L, "GET");
+}
+
+LuaDefine(http_head, "request",
+    "|Make an HTTP HEAD request.  Returns an HTTP response."
+    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
+    return lfn_http_request(L, "HEAD");
+}
+
+LuaDefine(http_post, "request",
+    "|Make an HTTP POST request.  Returns an HTTP response."
+    "|See doc(http.clientrequest) and doc(http.clientresponse).") {
+    return lfn_http_request(L, "POST");
+}

luprex/core/cpp/world-core.cpp

@@ -301,6 +301,7 @@ eng::string World::probe_lua(int64_t actor_id, const eng::string &lua) {
         for (int i = top + 1; i <= lua_gettop(L); i++) {
             LuaSpecial root(i);
             pprint(LS, root, true, ostream);
+            // TODO: this endl is unnecessary if we just printed a newline.
             (*ostream) << std::endl;
         }
     } else {

luprex/core/cpp/world.hpp

@@ -557,7 +557,7 @@ private:
     friend int lfn_math_randomstate(lua_State *L);
     friend int lfn_wait(lua_State *L);
     friend int lfn_nopredict(lua_State *L);
-    friend int lfn_http_get(lua_State *L);
+    friend int lfn_http_request(lua_State *L, const char *method);
 };
 
 using UniqueWorld = std::unique_ptr<World>;