talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HTTP now automatically encodes jsonvalue content

Browse Source
This commit is contained in:
jyelon
2022-06-07 01:54:08 -04:00
parent 779d9e20b8
commit 79b84a588a
9 changed files with 191 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
luprex/core/cpp/http.cpp
View File

@@ -9,6 +9,7 @@
#include "wrap-string.hpp"
#include "util.hpp"
#include "luastack.hpp"
#include "json.hpp"
#include <cstdint>
@@ -319,6 +320,38 @@ static void send_error_response(bool http11, int code, eng::string extrainfo, St
}
}
// HTTP 1.0: An entity body is included with a request message only when the request method calls for one.
// The presence of an entity body in a request is signaled by the inclusion of a Content-Length header field
// in the request message headers. HTTP/1.0 requests containing an entity body must include a valid
// Content-Length header field.
//
// HTTP 1.1: The presence of a message-body in a request is signaled by the inclusion of a Content-Length or
// Transfer-Encoding header field in the request's message-headers
//
static bool request_contains_content(int content_length, std::string_view transfer_encoding) {
return (content_length >= 0) || (!transfer_encoding.empty());
}
// HTTP 1.0: For response messages, whether or not an entity body is included with a message is dependent on
// both the request method and the response code. All responses to the HEAD request method must not include a
// body, even though the presence of entity header fields may lead one to believe they do. All 1xx (informational),
// 204 (no content), and 304 (not modified) responses must not include a body. All other responses must include an
// entity body or a Content-Length header field defined with a value of zero (0).
//
// HTTP 1.1: For response messages, whether or not a message-body is included with a message is dependent on both
// the request method and the response status code (section 6.1.1). All responses to the HEAD request method MUST
// NOT include a message-body, even though the presence of entity- header fields might lead one to believe they
// do. All 1xx (informational), 204 (no content), and 304 (not modified) responses MUST NOT include a message-body.
// All other responses do include a message-body, although it MAY be of zero length.
//
static bool response_contains_content(std::string_view method, int status) {
if (method == "HEAD") return false;
if ((status >= 100) && (status <= 199)) return false;
if (status == 204) return false;
if (status == 304) return false;
return true;
}
// In a properly-formed url, the hostname and path are url encoded.
// This parser expects an encoded URL.
struct ParsedURL {
@@ -451,8 +484,8 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
sb->write_bytes("\r\n");
}
// If it's a post request, send the content length and the content type.
if (method_ == "POST") {
// Send the content length and the content type.
if (content_assigned_) {
send_content_length_header(true, content_.size(), sb);
send_content_type_header(true, mime_type_, sb);
}
@@ -460,8 +493,8 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
// Send the extra linebreak.
sb->write_bytes("\r\n");
// If it's a post request, send the content.
if (method_ == "POST") {
// Send the content.
if (content_assigned_) {
sb->write_bytes(content_);
}
}
@@ -681,6 +714,17 @@ void HttpClientRequest::set_content(LuaStack &LS, LuaSlot val) {
set_content(LS.ckstring(val));
}
void HttpClientRequest::set_jsonvalue(LuaStack &LS, LuaSlot val) {
eng::string out;
eng::string err = json::encode(LS, val, out, false, HttpParser::MAX_CONTENT_LENGTH);
if (!err.empty()) {
check_fail(util::ss("json encode failure: ", err));
return;
}
set_content(out);
set_mime_type("application/json");
}
void HttpClientRequest::set_defaults() {
if (method_.empty()) {
method_ = "GET";
@@ -715,6 +759,20 @@ void HttpClientRequest::set_config(LuaStack &LS0, LuaSlot tab) {
set_mime_type(LS, val);
} else if (kstr == "content") {
set_content(LS, val);
} else if (kstr == "html") {
set_content(LS, val);
set_mime_type("text/html");
} else if (kstr == "text") {
set_content(LS, val);
set_mime_type("text/plain");
} else if (kstr == "json") {
set_content(LS, val);
set_mime_type("application/json");
} else if (kstr == "bytes") {
set_content(LS, val);
set_mime_type("application/octet-stream");
} else if (kstr == "jsonvalue") {
set_jsonvalue(LS, val);
} else if (kstr == "") {
check_fail(util::ss("configuration parameter names must be strings."));
} else {
@@ -755,13 +813,6 @@ eng::string HttpClientRequest::check() const {
if (mime_type_.empty()) {
return "mime type not set for POST request";
}
} else {
if (content_assigned_) {
return "content can only be specified for POST requests";
}
if (!mime_type_.empty()) {
return "mime type can only be specified for POST requests";
}
}
return "";
}
@@ -822,20 +873,23 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons
return;
}
// Determine if we're going to be sending content.
bool contains_content = response_contains_content("GET", status_);
// The status message is the human-readable status code.
eng::string statusmsg = status_code_to_string(status_);
// Annotate error messages.
const eng::string *content = &content_;
if (errstatus() && (mime_type_ == "text/plain") &&
!content_.empty() && !contains_newline(content_)) {
if (errstatus() && contains_content &&
(mime_type_ == "text/plain") && !contains_newline(content_)) {
if (sv::has_prefix(content_, statusmsg)) {
statusmsg = content_;
} else {
} else if (!content_.empty()) {
statusmsg += ": ";
statusmsg += content_;
content = &statusmsg;
}
content = &statusmsg;
}
// Send the status line.
@@ -848,7 +902,7 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons
}
// Send the headers that make sense if we're sending content.
if (content_assigned_) {
if (contains_content) {
if (!errstatus()) {
send_cache_control_header(http11_, max_age_, sb);
}
@@ -860,7 +914,7 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons
sb->write_bytes("\r\n");
// Send the content.
if (content_assigned_) {
if (contains_content) {
sb->write_bytes(*content);
}
}
@@ -960,6 +1014,17 @@ void HttpServerResponse::set_content(LuaStack &LS, LuaSlot val) {
set_content(LS.ckstring(val));
}
void HttpServerResponse::set_jsonvalue(LuaStack &LS, LuaSlot val) {
eng::string out;
eng::string err = json::encode(LS, val, out, false, HttpParser::MAX_CONTENT_LENGTH);
if (!err.empty()) {
check_fail(util::ss("json encode failure: ", err));
return;
}
set_content(out);
set_mime_type("application/json");
}
void HttpServerResponse::set_config(LuaStack &LS0, LuaSlot tab) {
LuaVar key, val;
LuaStack LS(LS0.state(), key, val);
@@ -987,6 +1052,8 @@ void HttpServerResponse::set_config(LuaStack &LS0, LuaSlot tab) {
} else if (kstr == "bytes") {
set_content(LS, val);
set_mime_type("application/octet-stream");
} else if (kstr == "jsonvalue") {
set_jsonvalue(LS, val);
} else if (kstr == "") {
check_fail(util::ss("response configuration parameters must be strings."));
} else {
@@ -1010,14 +1077,14 @@ void HttpServerResponse::set_defaults() {
status_ = 200;
}
// If you're sending an error message along with
// an error status code, then assume the error
// message is text/plain.
if ((status_ >= 400) && (status_ <= 599)) {
if (content_assigned_ && (mime_type_.empty())) {
// If the response status indicates that we're sending
// content, and there's no content, generate blank content.
if (response_contains_content("GET", status_) &&
(!content_assigned_) && (mime_type_.empty())) {
content_assigned_ = true;
content_ = "";
mime_type_ = "text/plain";
}
}
}
eng::string HttpServerResponse::check() const {
@@ -1030,15 +1097,16 @@ eng::string HttpServerResponse::check() const {
return "status code not specified";
}
// If you specify content, you have to specify mime
// type, and vice versa. Also, we need both for a valid response.
if ((status_ == 200) || (content_assigned_) || (!mime_type_.empty())) {
if (!content_assigned_) {
return "content not specified";
}
if (mime_type_.empty()) {
return "mime type not specified";
// If you assigned a mime type and didn't specify
// content, that's bad.
if ((!content_assigned_) && (!mime_type_.empty())) {
return "mime type specified without content";
}
// If you assigned content, but didn't assign
// a mime type, that's bad.
if (content_assigned_ && (mime_type_.empty())) {
return "content specified without mime type";
}
return "";
}
@@ -1337,6 +1405,17 @@ bool HttpParser::parse_content_chunked(std::string_view &view, bool closed) {
}
bool HttpParser::parse_content(std::string_view &view, bool closed) {
// If there's no body, just return true.
if (is_request_) {
if (!request_contains_content(content_length_, transfer_encoding_)) {
return true;
}
} else {
if (!response_contains_content(method_, status_)) {
return true;
}
}
// Parse the content.
if (transfer_encoding_ == "") {
if (!parse_content_basic(view, closed)) {
@@ -1386,8 +1465,8 @@ bool HttpParser::parse_content(std::string_view &view, bool closed) {
}
void HttpParser::store(LuaStack &LS0, LuaSlot tab) const {
LuaVar ptab;
LuaStack LS(LS0.state(), ptab);
LuaVar ptab, djson;
LuaStack LS(LS0.state(), ptab, djson);
LS.newtable(tab);
if (!is_request_) {
@@ -1402,6 +1481,10 @@ void HttpParser::store(LuaStack &LS0, LuaSlot tab) const {
if (!mime_type_.empty() || !content_.empty()) {
LS.rawset(tab, "mimetype", mime_type_);
LS.rawset(tab, "content", content_);
if (mime_type_ == "application/json") {
json::decode(LS, djson, content_);
LS.rawset(tab, "jsonvalue", djson);
}
}
// We currently don't store the method, because
// our server only supports GET. Even if we
@@ -1475,9 +1558,10 @@ eng::string HttpParser::debug_string() const {
return oss.str();
}
void HttpParser::parse_response(std::string_view view, bool closed) {
void HttpParser::parse_response(std::string_view view, bool closed, std::string_view method) {
std::string_view original_view = view;
is_request_ = false;
method_ = method;
// Parse the status line.
if (!parse_status_line(view, closed)) {
@@ -1524,11 +1608,9 @@ void HttpParser::parse_request(std::string_view view, bool closed) {
}
// Process the content, if any.
if (method_ == "POST") {
if (!parse_content(view, closed)) {
return;
}
}
// Calculate the comm length.
comm_length_ = original_view.size() - view.size();
@@ -1594,8 +1676,22 @@ LuaDefine(http_clientrequest, "request",
"| path (ie, '/index.html')"
"| params (a table of url parameters)"
"| verifycertificate (default: true)"
"| content (the body to send to the server, if any)"
"| mimetype (mime type of the body, if any)"
"|"
"|The table can also contain this field, which sets"
"|host, port, path, and params in a single directive."
"|"
"| url (ie, 'https://host:port/path.html?a=b&c=d')"
"|"
"|The table can also contain these fields, which set"
"|both the content and the mimetype in a single directive:"
"|"
"| html - content as a string (text/html)"
"| text - content as a string (text/plain)"
"| json - content as a string (application/json)"
"| bytes - content as a string (application/octet-stream)"
"|"
"|You can specify url components separately (host, port, path,"
"|and params), or you can specify the entire url as a unit. "
"|If you specify components, they must not be url-encoded. "
@@ -1674,7 +1770,7 @@ LuaDefine(http_clientresponse, "response",
LuaRet tab;
LuaStack LS(L, text, tab);
HttpParser parser;
parser.parse_response(LS.ckstring(text), true);
parser.parse_response(LS.ckstring(text), true, "GET");
parser.store(LS, tab);
return LS.result();
}

14
luprex/core/cpp/http.hpp
View File

@@ -97,6 +97,7 @@ public:
void set_url(LuaStack &LS, LuaSlot val);
void set_mime_type(LuaStack &LS, LuaSlot val);
void set_content(LuaStack &LS, LuaSlot val);
void set_jsonvalue(LuaStack &LS, LuaSlot val);
// Set default values for method and port.
// This must be done after setting regular values.
@@ -126,6 +127,10 @@ public:
//
eng::string check() const;
// Get the method.
//
eng::string method() const { return method_; }
// Put the request into the stream, assuming HTTP/1.1
//
void send(StreamBuffer *target) const { send_internal(target, false); }
@@ -186,6 +191,7 @@ public:
//
HttpServerResponse();
void set_method(const eng::string &method);
void set_status(int status);
void set_max_age(int max_age);
void set_mime_type(const eng::string &mime_type);
@@ -195,6 +201,7 @@ public:
void set_max_age(LuaStack &LS, LuaSlot val);
void set_mime_type(LuaStack &LS, LuaSlot val);
void set_content(LuaStack &LS, LuaSlot val);
void set_jsonvalue(LuaStack &LS, LuaSlot val);
// Set default values.
//
@@ -275,7 +282,7 @@ protected:
// The protocol: true for HTTP/1.1, false for HTTP/1.0
bool http11_;
// The method: always "GET". (only when parsing requests)
// The method. In a response, this is assigned before parsing.
eng::string method_;
// The URL path, not url-encoded. (only when parsing requests)
@@ -383,7 +390,7 @@ public:
// The parser will not try to parse content longer than this.
//
const int64_t MAX_CONTENT_LENGTH = 1000000;
static constexpr int64_t MAX_CONTENT_LENGTH = 1000000;
// Parse a request or a response.
//
@@ -391,7 +398,7 @@ public:
// construct a new parser.
//
void parse_request(std::string_view view, bool closed);
void parse_response(std::string_view view, bool closed);
void parse_response(std::string_view view, bool closed, std::string_view method);
// Store a status code and an error message, and clear the content.
// This is generally used when the client detects an error,
@@ -428,6 +435,7 @@ using HttpParserVec = eng::vector<HttpParser>;
class HttpChannel {
public:
SharedChannel channel_;
eng::string method_;
int64_t parsed_bytes_;
bool marked_for_deletion() const { return channel_ == nullptr; }

20
luprex/core/cpp/json.cpp
View File

@@ -13,6 +13,7 @@
LuaTokenConstant(json_null, "null", "");
LuaTokenConstant(json_object, "object", "");
LuaTokenConstant(json_error, "error", "");
static void indent(eng::ostringstream &oss, int level) {
if (level < NOINDENT_LEVEL) {
@@ -543,23 +544,26 @@ bool decode(LuaStack &LS, LuaSlot out, std::string_view v) {
lua_State *L = LS.state();
// Try to read a single value from the view.
int top = lua_gettop(L);
bool ok = decode_value(L, v);
lua_replace(L, out.index());
if (!ok) return false;
// There should be nothing left of the input text.
if (v.size() > 0) {
lua_pushnil(L);
lua_replace(L, out.index());
lua_settop(L, top);
if (!ok) {
LS.set(out, LuaToken("error"));
return false;
}
// Special case: if the top-level result is jsonnull,
// then change it to nil.
// Special case: if the top level value is 'null', change
// it to 'nil.'
if (LS.istoken(out)) {
LS.set(out, LuaNil);
}
// There should be nothing left of the input text.
if (v.size() > 0) {
LS.set(out, LuaToken("error"));
return false;
}
return true;
}

3
luprex/core/cpp/json.hpp
View File

@@ -24,7 +24,8 @@ namespace json {
// See doc(http.jsondecode) for a lot more information.
//
// The only error condition is syntactically invalid json.
// In that case, we return false.
// In that case, we return false and set 'out' to the
// token 'error'.
//
bool decode(LuaStack &LS, LuaSlot out, std::string_view in);
}

3
luprex/core/cpp/lpxserver.cpp
View File

@@ -219,6 +219,7 @@ public:
HttpChannel &channel = http_client_channels_[request.request_id()];
if (channel.channel_ == nullptr) {
channel.channel_ = new_outgoing_channel(request.target());
channel.method_ = request.method();
channel.parsed_bytes_ = 0;
request.send(channel.channel_->out());
}
@@ -235,7 +236,7 @@ public:
response.fail(503, util::ss("Service Unavailable: ", channel.error()));
} else {
htchan.parsed_bytes_ = channel.in()->fill();
response.parse_response(channel.in()->view(), channel.closed());
response.parse_response(channel.in()->view(), channel.closed(), htchan.method_);
}
if (response.complete()) {
response.set_request_id(pair.first);

5
luprex/core/cpp/util.hpp
View File

@@ -344,12 +344,15 @@ inline eng::string ss(const ARGS & ... args) {
return oss.str();
}
// This is a better way to do std::setfill, std::hex, std::setprecision
// A better API than std::setfill, std::hex, std::setw, std::setprecision
//
// Usage examples:
// std::cout << util::hex.width(5).fill('0').val(123)
// std::cout << util::dec.fill('$').precision(val(123)
//
// The reason that other API is bad is that it can leave std::cout
// in an unpredictable state. This API always leaves the stream clean.
//
template <class VALUE>
class FormattedNumber {
public:

24
luprex/core/cpp/world-accessor.cpp
View File

@@ -654,9 +654,7 @@ LuaDefine(doc, "function",
return LS.result();
}
LuaDefine(http_get, "request",
"|Make an HTTP GET request. Returns an HTTP response."
"|See doc(http.clientrequest) and doc(http.clientresponse).") {
int lfn_http_request(lua_State *L, const char *method) {
World *w = World::fetch_global_pointer(L);
w->guard_blockable(L, "http.get");
@@ -667,7 +665,7 @@ LuaDefine(http_get, "request",
// Parse the request and make sure it's valid.
// If not, immediately pass a '400 bad request' back to lua.
req.set_method("GET");
req.set_method(method);
req.set_config(LS, request);
req.set_defaults();
eng::string error = req.check();
@@ -687,3 +685,21 @@ LuaDefine(http_get, "request",
// Block.
return lua_yield(L, 0);
}
LuaDefine(http_get, "request",
"|Make an HTTP GET request. Returns an HTTP response."
"|See doc(http.clientrequest) and doc(http.clientresponse).") {
return lfn_http_request(L, "GET");
}
LuaDefine(http_head, "request",
"|Make an HTTP HEAD request. Returns an HTTP response."
"|See doc(http.clientrequest) and doc(http.clientresponse).") {
return lfn_http_request(L, "HEAD");
}
LuaDefine(http_post, "request",
"|Make an HTTP POST request. Returns an HTTP response."
"|See doc(http.clientrequest) and doc(http.clientresponse).") {
return lfn_http_request(L, "POST");
}

1
luprex/core/cpp/world-core.cpp
View File

@@ -301,6 +301,7 @@ eng::string World::probe_lua(int64_t actor_id, const eng::string &lua) {
for (int i = top + 1; i <= lua_gettop(L); i++) {
LuaSpecial root(i);
pprint(LS, root, true, ostream);
// TODO: this endl is unnecessary if we just printed a newline.
(*ostream) << std::endl;
}
} else {

2
luprex/core/cpp/world.hpp
View File

@@ -557,7 +557,7 @@ private:
friend int lfn_math_randomstate(lua_State *L);
friend int lfn_wait(lua_State *L);
friend int lfn_nopredict(lua_State *L);
friend int lfn_http_get(lua_State *L);
friend int lfn_http_request(lua_State *L, const char *method);
};
using UniqueWorld = std::unique_ptr<World>;