talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Machinery for blocking http requests in place, except actual networking

Browse Source
This commit is contained in:
jyelon
2022-05-03 00:36:11 -04:00
parent d28c588468
commit c41d0522df
9 changed files with 381 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
luprex/core/cpp/http.cpp
View File

@@ -189,6 +189,9 @@ public:
HttpOutRequest::HttpOutRequest() {
verify_certificate_ = true;
port_ = 0;
request_id_ = 0;
place_id_ = 0;
thread_id_ = 0;
}
void HttpOutRequest::fail(string_view s) {
@@ -395,8 +398,6 @@ void HttpOutRequest::set_config(LuaStack &LS0, LuaSlot tab) {
set_port(LS, val);
} else if (kstr == "path") {
set_path(LS, val);
} else if (kstr == "encodedpath") {
set_path(LS, val);
} else if (kstr == "params") {
set_params(LS, val);
} else if (kstr == "url") {
@@ -430,6 +431,7 @@ eng::string HttpOutRequest::check() const {
return "";
}
void HttpOutRequest::send_internal(StreamBuffer *sb, bool debug_string) const {
// If there's an error in the request, handle it. In debug string mode,
// we just put the error into the output. In production mode, we assert
@@ -469,18 +471,78 @@ void HttpOutRequest::send_internal(StreamBuffer *sb, bool debug_string) const {
sb->write_bytes("User-agent: Mozilla 5.0 (luprex)");
sb->write_bytes(linebreak);
// Add the requester IDs (debug string only)
if (debug_string && ((request_id_ != 0) || (place_id_ != 0) || (thread_id_ != 0))) {
sb->write_bytes("X-Requester-Ids: ");
sb->ostream() << "rid=" << request_id_ << "; pid=" << place_id_ << "; tid=" << thread_id_;
sb->write_bytes(linebreak);
}
// Send the extra linebreak.
if (!debug_string) {
sb->write_bytes(linebreak);
}
}
void HttpOutRequest::serialize(StreamBuffer *sb) const {
sb->write_int64(request_id_);
sb->write_int64(place_id_);
sb->write_int64(thread_id_);
sb->write_string(error_);
sb->write_bool(verify_certificate_);
sb->write_string(method_);
sb->write_string(host_);
sb->write_int32(port_);
sb->write_string(path_);
sb->write_int32(params_.size());
for (const auto &pair : params_) {
sb->write_string(pair.first);
sb->write_string(pair.second);
}
}
void HttpOutRequest::deserialize(StreamBuffer *sb) {
request_id_ = sb->read_int64();
place_id_ = sb->read_int64();
thread_id_ = sb->read_int64();
error_ = sb->read_string();
verify_certificate_ = sb->read_bool();
method_ = sb->read_string();
host_ = sb->read_string();
port_ = sb->read_int32();
path_ = sb->read_string();
int32_t nparams = sb->read_int32();
params_.clear();
for (int i = 0; i < nparams; i++) {
eng::string k = sb->read_string();
eng::string v = sb->read_string();
params_[k] = v;
}
}
eng::string HttpOutRequest::DebugString() {
StreamBuffer sb;
send_internal(&sb, true);
return eng::string(sb.view());
}
void HttpOutRequestMap::serialize(StreamBuffer *sb) const {
sb->write_int32(size());
for (const auto &pair : *this) {
pair.second.serialize(sb);
}
}
void HttpOutRequestMap::deserialize(StreamBuffer *sb) {
int32_t count = sb->read_int32();
clear();
HttpOutRequest req;
for (int i = 0; i < count; i++) {
req.deserialize(sb);
(*this)[req.request_id()] = req;
}
}
HttpInResponse::HttpInResponse() {
status_code_ = 0;
response_length_ = 0;
@@ -513,7 +575,7 @@ void HttpInResponse::fail(int code, string_view message) {
void HttpInResponse::incomplete(bool closed) {
if (closed) {
fail(500, "response truncated");
fail(500, "internal server error: response truncated");
} else {
fail(0, "response not yet fully received");
}
@@ -526,7 +588,7 @@ void HttpInResponse::parse_content_encoding(string_view value) {
void HttpInResponse::parse_content_length(string_view value) {
int64_t code = sv::to_int64(value);
if ((code < 0) || (code > INT_MAX)) {
fail(500, util::ss("unparseable content-length: ", value));
fail(500, util::ss("internal server error: unparseable content-length: ", value));
}
content_length_ = code;
}
@@ -536,7 +598,7 @@ void HttpInResponse::parse_content_type(string_view value) {
string_view ctview(ctype);
mime_type_ = sv::trim(sv::read_to_sep(ctview, ';'));
if (mime_type_.empty()) {
fail(500, util::ss("unparseable content-type: ", value));
fail(500, util::ss("internal server error: unparseable content-type: ", value));
return;
}
while (true) {
@@ -571,14 +633,14 @@ void HttpInResponse::parse_header(string_view header, string_view value) {
} else if (header == "transfer-encoding") {
parse_transfer_encoding(value);
} else if (header == "content-range") {
fail(500, util::ss("unsupported response header: ", header));
fail(406, util::ss("not acceptable: unsupported response header: ", header));
}
}
bool HttpInResponse::parse_content_basic(std::string_view &view, bool closed) {
if (content_length_ >= 0) {
if (content_length_ > MAX_CONTENT_LENGTH) {
fail(500, "content too long");
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
return false;
}
if (int(view.size()) < content_length_) {
@@ -588,7 +650,7 @@ bool HttpInResponse::parse_content_basic(std::string_view &view, bool closed) {
content_ = sv::read_nbytes(view, content_length_);
} else {
if (int64_t(view.size()) > MAX_CONTENT_LENGTH) {
fail(500, "content too long");
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
return false;
}
if (!closed) {
@@ -611,17 +673,17 @@ bool HttpInResponse::parse_content_chunked(std::string_view &view, bool closed)
}
int64_t chunk_size = sv::to_hex64(chunk_header, -1);
if (chunk_size < 0) {
fail(500, "unparseable chunk header");
fail(500, "internal server error: unparseable chunk header");
return false;
}
if (chunk_size > MAX_CONTENT_LENGTH) {
fail(500, "content too long");
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
return false;
}
if (chunk_size == 0) break;
total_size += chunk_size;
if (total_size > MAX_CONTENT_LENGTH) {
fail(500, "content too long");
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
return false;
}
std::string_view chunk = sv::read_nbytes(view, chunk_size);
@@ -631,7 +693,7 @@ bool HttpInResponse::parse_content_chunked(std::string_view &view, bool closed)
}
std::string_view newline = sv::read_to_line(view);
if (!newline.empty()) {
fail(500, "corrupted chunk encoding");
fail(500, "internal server error: corrupted chunk encoding");
return false;
}
if (sv::isnull(view)) {
@@ -665,7 +727,7 @@ void HttpInResponse::parse(const StreamBuffer *sb, bool closed) {
string_view scode = sv::read_to_space(status);
int64_t code = sv::to_int64(scode, 0);
if ((code < 100) || (code > 599)) {
fail(500, util::ss("protocol error: invalid response code: ", scode));
fail(500, util::ss("internal server error: invalid response code: ", scode));
}
status_code_ = code;
@@ -690,11 +752,11 @@ void HttpInResponse::parse(const StreamBuffer *sb, bool closed) {
}
eng::string command = util::ascii_tolower(sv::trim(sv::read_to_sep(header, ':')));
if (sv::isnull(header)) {
fail(500, util::ss("protocol error: no colon in header line: ", command));
fail(500, util::ss("internal server error: no colon in header line: ", command));
return;
}
if (!words_separated_by_dashes(command)) {
fail(500, util::ss("protocol error: invalid header: ", command));
fail(500, util::ss("internal server error: invalid header: ", command));
return;
}
parse_header(command, sv::trim(header));
@@ -716,7 +778,7 @@ void HttpInResponse::parse(const StreamBuffer *sb, bool closed) {
// If it's not a redirect, disallow 'location'.
if ((status_code_ < 300) || (status_code_ > 399)) {
if (!location_.empty()) {
fail(500, util::ss("redirect specified, but result code not 300-399: ", code));
fail(500, util::ss("internal server error: location specified, but result code not 300-399: ", code));
return;
}
}
@@ -734,18 +796,18 @@ void HttpInResponse::parse(const StreamBuffer *sb, bool closed) {
// If it's multipart, reject it.
if (sv::has_prefix(mime_type_, "multipart/")) {
fail(500, "multipart messages not implemented");
fail(406, "not acceptable: multipart messages not supported");
return;
}
// If it's text, demand a reasonable charset. Otherwise,
// ignore the charset.
if (sv::has_prefix(mime_type_, "text/")) {
if (charset_.empty()) {
if (charset_.empty() || (charset_ == "ascii")) {
charset_ = "utf-8";
}
if (charset_ != "utf-8") {
fail(500, util::ss("charset not supported: ", charset_));
fail(406, util::ss("not acceptable: charset not supported: ", charset_));
return;
}
} else {
@@ -755,7 +817,7 @@ void HttpInResponse::parse(const StreamBuffer *sb, bool closed) {
// Uncompress the content.
if ((content_encoding_ == "") || (content_encoding_ == "identity")) {
} else {
fail(500, util::ss("content-encoding not supported: ", content_encoding_));
fail(406, util::ss("not acceptable: content-encoding not supported: ", content_encoding_));
return;
}
@@ -767,7 +829,7 @@ void HttpInResponse::parse(const StreamBuffer *sb, bool closed) {
}
}
void HttpInResponse::store(LuaStack &LS0, LuaSlot tab) {
void HttpInResponse::store(LuaStack &LS0, LuaSlot tab) const {
LuaStack LS(LS0.state());
LS.newtable(tab);
@@ -784,10 +846,24 @@ void HttpInResponse::store(LuaStack &LS0, LuaSlot tab) {
}
// Debugging fields. Do not use for lua programming.
LS.rawset(tab, "dbg-content-length", content_length_);
LS.rawset(tab, "dbg-transfer-encoding", transfer_encoding_);
LS.rawset(tab, "dbg-charset", charset_);
LS.rawset(tab, "dbg-response-length", response_length_);
if (content_length_ >= 0) {
LS.rawset(tab, "dbg_contentlength", content_length_);
}
if (!transfer_encoding_.empty()) {
LS.rawset(tab, "dbg_transferencoding", transfer_encoding_);
}
if (!charset_.empty()) {
LS.rawset(tab, "dbg_charset", charset_);
}
if (response_length_ != 0) {
LS.rawset(tab, "dbg_responselength", response_length_);
}
}
void HttpInResponse::store_fail(LuaStack &LS, LuaSlot tab, int status_code, std::string_view error) {
HttpInResponse response;
response.fail(status_code, error);
response.store(LS, tab);
}
LuaDefine(http_fixurl, "url", "validate URL and repair minor flaws in the URL syntax") {
@@ -804,36 +880,35 @@ LuaDefine(http_fixurl, "url", "validate URL and repair minor flaws in the URL sy
}
LuaDefine(http_request, "reqtab",
"|Given an HTTP request in the form of a table, returns the same "
"|request as a string, to assist with debugging."
LuaDefine(http_request, "request",
"|Takes an HTTP request in the form of a lua table."
"|The table may contain these fields:"
"|"
"|The table can contain:"
"|"
"| method (ie, GET, HEAD, POST, etc)"
"| method (ie, 'GET', 'POST', etc)"
"| host (ie, 'google.com')"
"| port (default: 443)"
"| url (ie, '/index.html')"
"| path (ie, '/index.html')"
"| params (a table of url parameters)"
"| verifycertificate (default: true)"
"| url (ie, 'https://host:port/path.html?a=b&c=d')"
"|"
"|The url can start with 'https://', or with '/'. If it starts"
"|with 'https://', then the URL includes the host and port, which"
"|then must not be specified separately."
"|You can specify url components separately (host, port, path,"
"|and params), or you can specify the entire url as a unit. "
"|If you specify components, they must not be url-encoded. "
"|If you specify the url as a whole, it must already be url-encoded."
"|"
"|Note that plain HTTP is not supported - we only allow HTTPS."
"|You can omit the port, in which case it defaults to the"
"|standard https port. You can omit verifycertificate, in which"
"|case it defaults to true. You can omit the method if the"
"|method is implied by the function you called (eg, 'http.get')."
"|"
"|Note that unencrypted http is not supported - we only allow https."
"|However, you can talk to a server that has a dummy certificate"
"|by specifying verifycertificate=false."
"|"
"|This module will automatically url encode everything for you."
"|Therefore, you shouldn't url encode anything, otherwise,"
"|you'll end up double-encoding."
"|"
"|You cannot include url parameters as part of the url. If you try,"
"|then your ?, &, and = characters will get url encoded, which will"
"|cause them to not function. To use url parameters, you must"
"|use the separate params table."
"|") {
"|This routine, http.request, returns a debug string for the "
"|request. The debug string looks like the actual http headers"
"|that would be sent.") {
LuaArg tab;
LuaRet str;
LuaStack LS(L, tab, str);
@@ -849,7 +924,48 @@ LuaDefine(http_request, "reqtab",
return LS.result();
}
LuaDefine(http_response, "text", "") {
LuaDefine(http_response, "response",
"|Returns an HTTP response in the form of a lua table."
"|The table will contain these important fields:"
"|"
"| responsecode - 3-digit HTTP response code."
"| error - an error message, or nil if no error."
"| content - on success, the content, as a string."
"| mimetype - on success, the mime type of the content."
"| location - for HTTP redirects, the target url."
"|"
"|If the mimetype is a text mimetype, then the content"
"|is automatically converted to utf-8."
"|"
"|The table may also contain these debugging-only fields."
"|"
"| dbg_transferencoding - If there was a Transfer-Encoding header."
"| dbg_contentlength - If there was a Content-length header."
"| dbg_charset - Original character set for text mime types."
"| dbg_responselength - Total bytes in the response."
"|"
"|None of the dbg fields is needed to understand the response."
"|For example, consider dbg_charset. When text content is"
"|passed to lua, the content is automatically converted to utf-8."
"|So dbg_charset only tells you what the character set used"
"|to be, before it was converted to utf-8."
"|"
"|If an http routine generates an error, that error will be"
"|expressed as a status code. These locally-generated status"
"|codes can be:"
"|"
"| 400 (bad request) - the request was malformed"
"| 503 (service unavailable) - dns fail, connect fail, or ssl fail"
"| 500 (internal server error)- the response contains invalid HTTP"
"| 406 (not acceptable) - the response used a feature we don't support yet"
"| 413 (payload too large) - we refuse to download something so big"
"| 425 (can't resume) - reloaded a save game with a pending request"
"|"
"|In this case, the error message will be the stock"
"|error message, followed by more information."
"|"
"|This routine, http.response, generates a response by parsing"
"|an actual HTTP response string. This is for debugging only.") {
LuaArg text;
LuaRet tab;
LuaStack LS(L, text, tab);