talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Did a lot of work on the HTTP server side

Browse Source
This commit is contained in:
jyelon
2022-05-12 13:48:42 -04:00
parent 297ca3d56e
commit 9cf04a7741
6 changed files with 747 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

740
luprex/core/cpp/http.cpp
View File

@@ -14,17 +14,43 @@
using string_view = std::string_view;
bool is_supported_protocol(string_view protocol) {
return (protocol == "HTTP/1.0") || (protocol == "HTTP/1.1");
}
bool is_supported_method(string_view method) {
return ((method == "GET") || (method == "HEAD") || (method == "POST"));
}
bool words_separated_by_dashes(string_view v) {
while (true) {
if (!sv::ascii_isalpha(sv::zfront(v))) return false;
v.remove_prefix(1);
while (sv::ascii_isalnum(sv::zfront(v))) v.remove_prefix(1);
string_view word = sv::read_ascii_identifier(v);
if (word.empty()) return false;
if (v.empty()) return true;
if (sv::zfront(v) != '-') return false;
char c = v.front();
if (c != '-') return false;
v.remove_prefix(1);
}
}
// This doesn't check whether the mime type is actually
// registered, obviously. It only checks that it's in
// the desired notation.
bool valid_mime_type(string_view method) {
string_view part1 = sv::read_ascii_identifier(method);
if (part1.empty()) return false;
if (sv::zfront(method) != '/') return false;
method.remove_prefix(1);
while (true) {
string_view word = sv::read_ascii_identifier(method);
if (word.empty()) return false;
if (method.empty()) return true;
char c = method.front();
if ((c != '-') && (c != '.') && (c != '+')) return false;
method.remove_prefix(1);
}
}
// Technically, this is a true, correct URL encode routine.
static eng::string url_encode_param(string_view value) {
eng::ostringstream result;
@@ -142,25 +168,31 @@ public:
ParsedURL(std::string_view url) {
clear();
proto = util::ascii_tolower(sv::read_to_sep(url, ':'));
if (!sv::has_prefix(url, "//")) { clear(); return; }
url.remove_prefix(2);
if (!words_separated_by_dashes(proto)) { clear(); return; }
// Extract the host and port as a single string.
string_view turl = url;
string_view hostport = sv::read_to_sep(turl, '/');
url.remove_prefix(hostport.size());
if (!sv::has_prefix(url, "/")) {
proto = util::ascii_tolower(sv::read_to_sep(url, ':'));
if (!sv::has_prefix(url, "//")) { clear(); return; }
url.remove_prefix(2);
if (!words_separated_by_dashes(proto)) { clear(); return; }
// Extract the host and port as a single string.
string_view turl = url;
string_view hostport = sv::read_to_sep(turl, '/');
url.remove_prefix(hostport.size());
// Split the host and port from each other and parse them.
host = util::ascii_tolower(sv::read_to_sep(hostport, ':'));
if (host.empty()) { clear(); return; }
if (!hostport.empty()) {
int64_t iport = sv::to_int64(hostport);
if ((iport < 1) || (iport > 65535)) {
clear(); return;
// Split the host and port from each other and parse them.
host = util::ascii_tolower(sv::read_to_sep(hostport, ':'));
if (host.empty()) { clear(); return; }
if (!hostport.empty()) {
int64_t iport = sv::to_int64(hostport);
if ((iport < 1) || (iport > 65535)) {
clear(); return;
}
port = iport;
}
port = iport;
} else {
// Stick in some defaults for unspecified fields.
host = "host";
proto = "https";
}
// Split off the path.
@@ -214,9 +246,8 @@ void HttpClientRequest::set_verify_certificate(bool flag) {
void HttpClientRequest::set_method(const eng::string &s) {
eng::string method = util::ascii_toupper(s);
if ((method != "GET") && (method != "HEAD")) {
fail(util::ss("HTTP method not implemented: ", method, ".",
"Currently, only HEAD and GET are implemented."));
if (!is_supported_method(method)) {
fail(util::ss("HTTP method not implemented: ", method, "."));
return;
}
if ((!method_.empty()) && (method_ != method)) {
@@ -301,6 +332,26 @@ void HttpClientRequest::set_url(string_view url) {
}
}
void HttpClientRequest::set_mime_type(const eng::string &mime_type) {
if (!valid_mime_type(mime_type)) {
fail(util::ss("Not a valid mime type: ", mime_type));
return;
}
if (!mime_type_.empty()) {
fail(util::ss("Mime type specified twice: ", mime_type_, " and ", mime_type));
return;
}
mime_type_ = mime_type;
}
void HttpClientRequest::set_content(const eng::string &content) {
if (!content_.empty()) {
fail(util::ss("Content specified twice"));
return;
}
content_ = content;
}
void HttpClientRequest::set_verify_certificate(LuaStack &LS, LuaSlot val) {
if (!LS.isboolean(val)) {
fail(util::ss("HTTP verify_certificate must be a boolean"));
@@ -374,6 +425,22 @@ void HttpClientRequest::set_url(LuaStack &LS, LuaSlot val) {
set_url(LS.ckstring(val));
}
void HttpClientRequest::set_mime_type(LuaStack &LS, LuaSlot val) {
if (!LS.isstring(val)) {
fail(util::ss("HTTP mime type must be a string"));
return;
}
set_mime_type(LS.ckstring(val));
}
void HttpClientRequest::set_content(LuaStack &LS, LuaSlot val) {
if (!LS.isstring(val)) {
fail(util::ss("HTTP content must be a string"));
return;
}
set_content(LS.ckstring(val));
}
void HttpClientRequest::set_defaults() {
if (method_.empty()) {
method_ = "GET";
@@ -404,6 +471,10 @@ void HttpClientRequest::set_config(LuaStack &LS0, LuaSlot tab) {
set_url(LS, val);
} else if (kstr == "verifycertificate") {
set_verify_certificate(LS, val);
} else if (kstr == "mimetype") {
set_mime_type(LS, val);
} else if (kstr == "content") {
set_content(LS, val);
} else if (kstr == "") {
fail(util::ss("HTTP config parameter names must be strings."));
} else {
@@ -428,6 +499,19 @@ eng::string HttpClientRequest::check() const {
if (path_.empty()) {
return "HTTP url has not been set";
}
if (method_ == "POST") {
if (mime_type_.empty()) {
if (content_.empty()) {
return "HTTP mime type and content not set for POST request";
} else {
return "HTTP mime type has not been set for POST request";
}
}
} else {
if ((!mime_type_.empty()) || (!content_.empty())) {
return "HTTP mime type and content are only for POST requests";
}
}
return "";
}
@@ -447,7 +531,7 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
}
// Choose a linebreak.
eng::string linebreak = (debug_string) ? "\n" : "\r\n";
eng::string linebreak = "\r\n";
// Send the command.
sb->write_bytes(method_);
@@ -473,14 +557,30 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const
// Add the requester IDs (debug string only)
if (debug_string && ((request_id_ != 0) || (place_id_ != 0) || (thread_id_ != 0))) {
sb->write_bytes("X-Requester-Ids: ");
sb->write_bytes("X-requester-ids: ");
sb->ostream() << "rid=" << request_id_ << "; pid=" << place_id_ << "; tid=" << thread_id_;
sb->write_bytes(linebreak);
}
// Send the extra linebreak.
if (!debug_string) {
// If it's a post request, send the content length and the content type.
if (method_ == "POST") {
sb->write_bytes("Content-length: ");
sb->ostream() << content_.size();
sb->write_bytes(linebreak);
sb->write_bytes("Content-type: ");
sb->write_bytes(mime_type_);
if (sv::has_prefix(mime_type_, "text/")) {
sb->write_bytes(" ; charset=utf-8");
}
sb->write_bytes(linebreak);
}
// Send the extra linebreak.
sb->write_bytes(linebreak);
// If it's a post request, send the content.
if (method_ == "POST") {
sb->write_bytes(content_);
}
}
@@ -543,63 +643,137 @@ void HttpClientRequestMap::deserialize(StreamBuffer *sb) {
}
}
HttpClientResponse::HttpClientResponse() {
request_id_ = 0;
status_code_ = 0;
response_length_ = 0;
HttpParser::HttpParser() {
status_ = 0;
mime_type_ = "";
content_length_ = -1;
comm_length_ = 0;
}
eng::string HttpClientResponse::DebugString() const {
eng::ostringstream oss;
oss << "HttpClientResponse:" << std::endl;
oss << " status_code: " << status_code_ << std::endl;
oss << " error: " << error_ << std::endl;
oss << " content_length: " << content_length_ << std::endl;
oss << " transfer_encoding: " << transfer_encoding_ << std::endl;
oss << " location: " << location_ << std::endl;
oss << " mime_type: " << mime_type_ << std::endl;
oss << " charset: " << charset_ << std::endl;
oss << " content: " << content_ << std::endl;
oss << " response_length: " << response_length_ << std::endl;
return oss.str();
}
void HttpClientResponse::fail(int code, string_view message) {
status_code_ = code;
void HttpParser::fail(int code, std::string_view message) {
status_ = code;
error_ = message;
mime_type_ = "";
charset_ = "";
content_ = "";
}
void HttpClientResponse::incomplete(bool closed) {
void HttpParser::syntax(std::string_view detail) {
if (is_request_) {
fail(400, util::ss("malformed request: ", detail));
} else {
fail(500, util::ss("malformed response: ", detail));
}
}
void HttpParser::incomplete(bool closed) {
if (closed) {
fail(500, "internal server error: response truncated");
syntax("response truncated");
} else {
fail(0, "response not yet fully received");
}
}
void HttpClientResponse::parse_content_encoding(string_view value) {
void HttpParser::oversized() {
fail(413, util::ss("payload too large: Limit=", MAX_CONTENT_LENGTH));
}
bool HttpParser::parse_request_line(std::string_view &view, bool closed) {
// Extract the request line.
//
string_view request = sv::trim(sv::read_to_line(view));
if (sv::isnull(view)) {
incomplete(closed);
return false;
}
// Break down the request line.
//
eng::string method = util::ascii_toupper(sv::read_to_space(request));
string_view path = sv::read_to_space(request);
eng::string protocol = util::ascii_toupper(sv::read_to_space(request));
if ((!request.empty()) || (protocol.empty())) {
syntax("invalid request line");
return false;
}
if (!is_supported_method(method)) {
fail(405, util::ss("Method Not Allowed: ", method));
return false;
}
if (!is_supported_protocol(protocol)) {
syntax(util::ss("unsupported protocol: ", protocol));
return false;
}
// Parse the url.
//
ParsedURL url(path);
if (!url.valid) {
syntax(util::ss("Invalid URL path: ", path));
return false;
}
method_ = method;
path_ = url.path;
params_ = url.params;
return true;
}
bool HttpParser::parse_status_line(std::string_view &view, bool closed) {
// Extract the status line.
//
string_view status = sv::trim(sv::read_to_line(view));
if (sv::isnull(view)) {
incomplete(closed);
return false;
}
// Break down the status line.
//
string_view protoversion = sv::read_to_space(status);
if (!is_supported_protocol(protoversion)) {
syntax(util::ss("unsupported protocol: ", protoversion));
return false;
}
string_view scode = sv::read_to_space(status);
int64_t code = sv::to_int64(scode, 0);
if ((code < 100) || (code > 599)) {
syntax(util::ss("invalid response code: ", scode));
return false;
}
status_ = code;
// Responses outside the range 200-299 are errors,
// and therefore must store a nonempty error message.
//
if ((code < 200) || (code > 299)) {
if (status.empty()) {
error_ = util::ss("error code ", code);
} else {
error_ = status;
}
}
return true;
}
void HttpParser::parse_content_encoding(string_view value) {
content_encoding_ = util::ascii_tolower(value);
}
void HttpClientResponse::parse_content_length(string_view value) {
void HttpParser::parse_content_length(string_view value) {
int64_t code = sv::to_int64(value);
if ((code < 0) || (code > INT_MAX)) {
fail(500, util::ss("internal server error: unparseable content-length: ", value));
syntax(util::ss("unparseable content-length: ", value));
}
content_length_ = code;
}
void HttpClientResponse::parse_content_type(string_view value) {
void HttpParser::parse_content_type(string_view value) {
eng::string ctype = util::ascii_tolower(value);
string_view ctview(ctype);
mime_type_ = sv::trim(sv::read_to_sep(ctview, ';'));
if (mime_type_.empty()) {
fail(500, util::ss("internal server error: unparseable content-type: ", value));
syntax(util::ss("unparseable content-type: ", value));
return;
}
while (true) {
@@ -614,15 +788,15 @@ void HttpClientResponse::parse_content_type(string_view value) {
}
}
void HttpClientResponse::parse_location(string_view value) {
void HttpParser::parse_location(string_view value) {
location_ = url_decode(value);
}
void HttpClientResponse::parse_transfer_encoding(string_view value) {
void HttpParser::parse_transfer_encoding(string_view value) {
transfer_encoding_ = util::ascii_tolower(value);
}
void HttpClientResponse::parse_header(string_view header, string_view value) {
void HttpParser::parse_header(string_view header, string_view value) {
if (header == "content-encoding") {
parse_content_encoding(value);
} else if (header == "content-length") {
@@ -634,14 +808,38 @@ void HttpClientResponse::parse_header(string_view header, string_view value) {
} else if (header == "transfer-encoding") {
parse_transfer_encoding(value);
} else if (header == "content-range") {
fail(406, util::ss("not acceptable: unsupported response header: ", header));
fail(416, util::ss("range not satisfiable: unsupported header: ", header));
}
}
bool HttpClientResponse::parse_content_basic(std::string_view &view, bool closed) {
bool HttpParser::parse_headers(std::string_view &view, bool closed) {
// Parse the headers.
while (true) {
string_view header = sv::read_to_line(view);
if (sv::isnull(view)) {
incomplete(closed);
return false;
}
if (header.empty()) {
return true;
}
eng::string command = util::ascii_tolower(sv::trim(sv::read_to_sep(header, ':')));
if (sv::isnull(header)) {
syntax(util::ss("no colon in header line: ", command));
return false;
}
if (!words_separated_by_dashes(command)) {
syntax(util::ss("invalid header: ", command));
return false;
}
parse_header(command, sv::trim(header));
}
}
bool HttpParser::parse_content_basic(std::string_view &view, bool closed) {
if (content_length_ >= 0) {
if (content_length_ > MAX_CONTENT_LENGTH) {
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
oversized();
return false;
}
if (int(view.size()) < content_length_) {
@@ -651,7 +849,7 @@ bool HttpClientResponse::parse_content_basic(std::string_view &view, bool closed
content_ = sv::read_nbytes(view, content_length_);
} else {
if (int64_t(view.size()) > MAX_CONTENT_LENGTH) {
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
oversized();
return false;
}
if (!closed) {
@@ -663,7 +861,7 @@ bool HttpClientResponse::parse_content_basic(std::string_view &view, bool closed
return true;
}
bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool closed) {
bool HttpParser::parse_content_chunked(std::string_view &view, bool closed) {
int64_t total_size = 0;
std::vector<string_view> chunks;
while (true) {
@@ -674,17 +872,17 @@ bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool clos
}
int64_t chunk_size = sv::to_hex64(chunk_header, -1);
if (chunk_size < 0) {
fail(500, "internal server error: unparseable chunk header");
syntax("unparseable chunk header");
return false;
}
if (chunk_size > MAX_CONTENT_LENGTH) {
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
oversized();
return false;
}
if (chunk_size == 0) break;
total_size += chunk_size;
if (total_size > MAX_CONTENT_LENGTH) {
fail(413, util::ss("payload too large: luprex limit=", MAX_CONTENT_LENGTH));
oversized();
return false;
}
std::string_view chunk = sv::read_nbytes(view, chunk_size);
@@ -694,7 +892,7 @@ bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool clos
}
std::string_view newline = sv::read_to_line(view);
if (!newline.empty()) {
fail(500, "internal server error: corrupted chunk encoding");
syntax("corrupted chunk encoding");
return false;
}
if (sv::isnull(view)) {
@@ -712,86 +910,29 @@ bool HttpClientResponse::parse_content_chunked(std::string_view &view, bool clos
return true;
}
void HttpClientResponse::parse(const StreamBuffer *sb, bool closed) {
// We're not going to modify the StreamBuffer at all.
// Instead, we work entirely on a view.
string_view view = sb->view();
// Get the status line.
string_view status = sv::trim(sv::read_to_line(view));
if (sv::isnull(view)) {
incomplete(closed);
return;
}
// Parse the status line.
string_view protoversion = sv::read_to_space(status);
if (!sv::has_prefix(protoversion, "HTTP/")) {
fail(500, util::ss("internal server error: status line appears corrupt"));
return;
}
string_view scode = sv::read_to_space(status);
int64_t code = sv::to_int64(scode, 0);
if ((code < 100) || (code > 599)) {
fail(500, util::ss("internal server error: invalid response code: ", scode));
return;
}
status_code_ = code;
// Responses outside the range 200-299 are errors,
// and therefore must store an error message.
if ((code < 200) || (code > 299)) {
if (status.empty()) {
error_ = util::ss("error code ", code);
} else {
error_ = status;
}
}
// Parse the headers.
while (true) {
string_view header = sv::read_to_line(view);
if (sv::isnull(view)) {
incomplete(closed);
return;
}
if (header.empty()) {
break;
}
eng::string command = util::ascii_tolower(sv::trim(sv::read_to_sep(header, ':')));
if (sv::isnull(header)) {
fail(500, util::ss("internal server error: no colon in header line: ", command));
return;
}
if (!words_separated_by_dashes(command)) {
fail(500, util::ss("internal server error: invalid header: ", command));
return;
}
parse_header(command, sv::trim(header));
}
// Process the content using the transfer encoding.
bool HttpParser::parse_content(std::string_view &view, bool closed) {
// Parse the content.
if (transfer_encoding_ == "") {
if (!parse_content_basic(view, closed)) return;
} else if (transfer_encoding_ == "chunked") {
if (!parse_content_chunked(view, closed)) return;
} else {
fail(500, util::ss("unsupported transfer-encoding: ", transfer_encoding_));
return;
}
// Calculate the response length.
response_length_ = sb->fill() - view.size();
// If it's not a redirect, disallow 'location'.
if ((status_code_ < 300) || (status_code_ > 399)) {
if (!location_.empty()) {
fail(500, util::ss("internal server error: location specified, but result code not 300-399: ", code));
return;
if (!parse_content_basic(view, closed)) {
return false;
}
} else if (transfer_encoding_ == "chunked") {
if (!parse_content_chunked(view, closed)) {
return false;
}
} else {
syntax(util::ss("unsupported transfer-encoding: ", transfer_encoding_));
return false;
}
// If the server didn't specify content-type, make a guess.
// Uncompress the content.
if ((content_encoding_ == "") || (content_encoding_ == "identity")) {
} else {
syntax(util::ss("content-encoding not supported: ", content_encoding_));
return true;
}
// If the sender didn't specify content-type, make a guess based on the content.
if (mime_type_.empty()) {
if (sv::valid_utf8(content_)) {
mime_type_ = "text/plain";
@@ -802,56 +943,49 @@ void HttpClientResponse::parse(const StreamBuffer *sb, bool closed) {
}
}
// If it's multipart, reject it.
if (sv::has_prefix(mime_type_, "multipart/")) {
fail(406, "not acceptable: multipart messages not supported");
return;
}
// If it's text, demand a reasonable charset. Otherwise,
// ignore the charset.
// Switch the charset to utf-8, if it's text.
if (sv::has_prefix(mime_type_, "text/")) {
if (charset_.empty() || (charset_ == "ascii")) {
charset_ = "utf-8";
}
if (charset_ != "utf-8") {
fail(406, util::ss("not acceptable: charset not supported: ", charset_));
return;
if (charset_.empty() || (charset_ == "ascii") || (charset_ == "utf-8")) {
// we're already good.
} else {
// We can't convert charsets yet.
syntax(util::ss("charset not supported: ", charset_));
return true;
}
} else {
// Not text. No need to specify charset.
charset_.clear();
}
// Uncompress the content.
if ((content_encoding_ == "") || (content_encoding_ == "identity")) {
} else {
fail(406, util::ss("not acceptable: content-encoding not supported: ", content_encoding_));
return;
}
// If there's an error code, throw out the content.
if ((status_code_ < 200) || (status_code_ > 299)) {
mime_type_.clear();
charset_.clear();
content_.clear();
}
return true;
}
void HttpClientResponse::store(LuaStack &LS0, LuaSlot tab) const {
LuaStack LS(LS0.state());
void HttpParser::store_parsed(LuaStack &LS0, LuaSlot tab) const {
LuaVar ptab;
LuaStack LS(LS0.state(), ptab);
LS.newtable(tab);
LS.rawset(tab, "responsecode", status_code_);
LS.rawset(tab, "status", status_);
if (!error_.empty()) {
LS.rawset(tab, "error", error_);
}
if (!location_.empty()) {
LS.rawset(tab, "location", location_);
}
if (!mime_type_.empty()) {
if (!mime_type_.empty() || !content_.empty()) {
LS.rawset(tab, "mimetype", mime_type_);
LS.rawset(tab, "content", content_);
}
if (!method_.empty()) {
LS.rawset(tab, "method", method_);
}
if (!path_.empty()) {
LS.rawset(tab, "path", path_);
LS.newtable(ptab);
LS.rawset(tab, "params", ptab);
for (const auto &pair : params_) {
LS.rawset(ptab, pair.first, pair.second);
}
}
// Debugging fields. Do not use for lua programming.
if (content_length_ >= 0) {
@@ -863,17 +997,170 @@ void HttpClientResponse::store(LuaStack &LS0, LuaSlot tab) const {
if (!charset_.empty()) {
LS.rawset(tab, "dbg_charset", charset_);
}
if (response_length_ != 0) {
LS.rawset(tab, "dbg_responselength", response_length_);
if (comm_length_ != 0) {
LS.rawset(tab, "dbg_commlength", comm_length_);
}
}
void HttpParser::parser_generate_debug_string(std::ostream &oss) const {
oss << " status_code: " << status_ << std::endl;
oss << " error: " << error_ << std::endl;
if (content_length_ >= 0) {
oss << " content_length: " << content_length_ << std::endl;
}
if (!transfer_encoding_.empty()) {
oss << " transfer_encoding: " << transfer_encoding_ << std::endl;
}
if (!location_.empty()) {
oss << " location: " << location_ << std::endl;
}
if (!mime_type_.empty()) {
oss << " mime_type: " << mime_type_ << std::endl;
}
if (!charset_.empty()) {
oss << " charset: " << charset_ << std::endl;
}
if (!content_.empty()) {
oss << " content: " << content_ << std::endl;
}
if (!method_.empty()) {
oss << " method: " << method_ << std::endl;
}
if (!path_.empty()) {
oss << " path: " << path_ << std::endl;
}
for (const auto &pair : params_) {
oss << " param: " << pair.first << "=" << pair.second << std::endl;
}
if (comm_length_ > 0) {
oss << " comm_length: " << comm_length_ << std::endl;
}
}
void HttpParser::clear_content_on_error() {
if ((status_ < 200) || (status_ > 299)) {
mime_type_.clear();
charset_.clear();
content_.clear();
}
}
HttpClientResponse::HttpClientResponse() {
request_id_ = 0;
is_request_ = false;
}
eng::string HttpClientResponse::DebugString() const {
eng::ostringstream oss;
oss << "HttpClientResponse:" << std::endl;
if (request_id_ != 0) {
oss << " request_id: " << request_id_ << std::endl;
}
parser_generate_debug_string(oss);
return oss.str();
}
void HttpClientResponse::parse(std::string_view view, bool closed) {
std::string_view original_view = view;
// Parse the status line.
if (!parse_status_line(view, closed)) {
return;
}
// Parse the headers.
if (!parse_headers(view, closed)) {
return;
}
// Process the content.
if (!parse_content(view, closed)) {
return;
}
// Calculate the response length.
set_comm_length(original_view.size() - view.size());
// If it's not a redirect, ignore location.
if ((status_ < 300) || (status_ > 399)) {
location_.clear();
}
// If it's multipart, reject it.
if (sv::has_prefix(mime_type_, "multipart/")) {
syntax("multipart messages not supported");
return;
}
// If there's an error code, throw out the content.
clear_content_on_error();
}
void HttpClientResponse::store(LuaStack &LS, LuaSlot tab) const {
store_parsed(LS, tab);
}
void HttpClientResponse::store_fail(LuaStack &LS, LuaSlot tab, int status_code, std::string_view error) {
HttpClientResponse response;
response.fail(status_code, error);
response.store(LS, tab);
}
HttpServerRequest::HttpServerRequest() {
is_request_ = true;
}
void HttpServerRequest::parse(std::string_view view, bool closed) {
std::string_view original_view = view;
// Parse the request line.
if (!parse_request_line(view, closed)) {
return;
}
// Parse the headers.
if (!parse_headers(view, closed)) {
return;
}
// Process the content, if any.
if (method_ == "POST") {
if (!parse_content(view, closed)) {
return;
}
}
// Calculate the comm length.
set_comm_length(original_view.size() - view.size());
// Always ignore location.
location_.clear();
// If it's multipart, reject it.
if (sv::has_prefix(mime_type_, "multipart/")) {
syntax("multipart messages not supported");
return;
}
// If we've made it this far, and there's no
// status code, set it to 200 OK.
if (status_ == 0) status_ = 200;
// If there's an error code, throw out the content.
clear_content_on_error();
}
void HttpServerRequest::store(LuaStack &LS, LuaSlot tab) const {
store_parsed(LS, tab);
}
eng::string HttpServerRequest::DebugString() const {
eng::ostringstream oss;
oss << "HttpServerRequest:" << std::endl;
parser_generate_debug_string(oss);
return oss.str();
}
LuaDefine(http_fixurl, "url", "validate URL and repair minor flaws in the URL syntax") {
LuaArg url;
LuaRet fixed;
@@ -888,8 +1175,8 @@ LuaDefine(http_fixurl, "url", "validate URL and repair minor flaws in the URL sy
}
LuaDefine(http_request, "request",
"|Takes an HTTP request in the form of a lua table."
LuaDefine(http_clientrequest, "request",
"|Takes an HTTP client request in the form of a lua table."
"|The table may contain these fields:"
"|"
"| method (ie, 'GET', 'POST', etc)"
@@ -914,7 +1201,7 @@ LuaDefine(http_request, "request",
"|However, you can talk to a server that has a dummy certificate"
"|by specifying verifycertificate=false."
"|"
"|This routine, http.request, returns a debug string for the "
"|This routine, http.clientrequest, returns a debug string for the "
"|request. The debug string looks like the actual http headers"
"|that would be sent.") {
LuaArg tab;
@@ -932,15 +1219,15 @@ LuaDefine(http_request, "request",
return LS.result();
}
LuaDefine(http_response, "response",
"|Returns an HTTP response in the form of a lua table."
LuaDefine(http_clientresponse, "response",
"|Returns an HTTP client response in the form of a lua table."
"|The table will contain these important fields:"
"|"
"| responsecode - 3-digit HTTP response code."
"| error - an error message, or nil if no error."
"| content - on success, the content, as a string."
"| mimetype - on success, the mime type of the content."
"| location - for HTTP redirects, the target url."
"| status - 3-digit HTTP response code."
"| error - an error message, or nil if no error."
"| content - on success, the content, as a string."
"| mimetype - on success, the mime type of the content."
"| location - for HTTP redirects, the target url."
"|"
"|If the mimetype is a text mimetype, then the content"
"|is automatically converted to utf-8."
@@ -950,7 +1237,7 @@ LuaDefine(http_response, "response",
"| dbg_transferencoding - If there was a Transfer-Encoding header."
"| dbg_contentlength - If there was a Content-length header."
"| dbg_charset - Original character set for text mime types."
"| dbg_responselength - Total bytes in the response."
"| dbg_commlength - Total bytes in the communication."
"|"
"|None of the dbg fields is needed to understand the response."
"|For example, consider dbg_charset. When text content is"
@@ -965,23 +1252,84 @@ LuaDefine(http_response, "response",
"| 400 (bad request) - the request was malformed"
"| 503 (service unavailable) - dns fail, connect fail, or ssl fail"
"| 500 (internal server error)- the response contains invalid HTTP"
"| 406 (not acceptable) - the response used a feature we don't support yet"
"| 413 (payload too large) - we refuse to download something so big"
"| 425 (can't resume) - reloaded a save game with a pending request"
"|"
"|In this case, the error message will be the stock"
"|error message, followed by more information."
"|Error messages that are generated locally consist of "
"|the standard message (eg, 'bad request') followed by more "
"|detailed information."
"|"
"|This routine, http.response, generates a response by parsing"
"|This routine, http.clientresponse, generates a response by parsing"
"|an actual HTTP response string. This is for debugging only.") {
LuaArg text;
LuaRet tab;
LuaStack LS(L, text, tab);
HttpClientResponse resp;
StreamBuffer sb;
sb.write_bytes(LS.ckstring(text));
resp.parse(&sb, true);
resp.parse(LS.ckstring(text), true);
resp.store(LS, tab);
return LS.result();
}
LuaDefine(http_serverrequest, "request",
"|Returns an HTTP server request in the form of a lua table."
"|The table will contain these important fields:"
"|"
"| status - 3-digit HTTP response code."
"| error - an error message, or nil if no error."
"| method - GET, HEAD, or POST"
"| path - the url-decoded path, eg, '/index.html'"
"| params - a table of url-decoded URL parameters"
"| content - the content, as a string (POST only)"
"| mimetype - the mime type of the content (POST only)"
"| location - for HTTP redirects, the target url."
"|"
"|If the mimetype is a text mimetype, then the content"
"|is automatically converted to utf-8."
"|"
"|The table may also contain these debugging-only fields."
"|"
"| dbg_transferencoding - If there was a Transfer-Encoding header."
"| dbg_contentlength - If there was a Content-length header."
"| dbg_charset - Original character set for text mime types."
"| dbg_commlength - Total bytes in the communication."
"|"
"|None of the dbg fields is needed to understand the request."
"|For example, consider dbg_charset. When text content is"
"|passed to lua, the content is automatically converted to utf-8."
"|So dbg_charset only tells you what the character set used"
"|to be, before it was converted to utf-8."
"|"
"|When the engine is functioning as a webserver, bad requests "
"|are never passed to lua. Therefore, a request that is passed "
"|to lua will always contain status=200 and error=nil. However, "
"|when debugging server requests using http.serverrequest, "
"|it is possible to see certain other errors:"
"|"
"| 400 (bad request) - the request was malformed"
"| 503 (service unavailable) - dns fail, connect fail, or ssl fail"
"| 500 (internal server error)- the response contains invalid HTTP"
"| 413 (payload too large) - we refuse to download something so big"
"| 425 (can't resume) - reloaded a save game with a pending request"
"|"
"|Error messages that are generated locally consist of "
"|the standard message (eg, 'bad request') followed by more "
"|detailed information."
"|"
"|This routine, http.serverrequest, generates a request by parsing"
"|an actual HTTP request string. This is for debugging only.") {
LuaArg text;
LuaRet tab;
LuaStack LS(L, text, tab);
HttpServerRequest req;
req.parse(LS.ckstring(text), true);
req.store(LS, tab);
return LS.result();
}
LuaDefine(http_validmime, "(mt)", "") {
LuaArg str;
LuaRet ok;
LuaStack LS(L, str, ok);
LS.set(ok, valid_mime_type(LS.ckstring(str)));
return LS.result();
}

220
luprex/core/cpp/http.hpp
View File

@@ -20,6 +20,7 @@
#include "luastack.hpp"
#include "streambuffer.hpp"
#include "drivenengine.hpp"
#include <ostream>
using UrlParameters = eng::map<eng::string, eng::string>;
@@ -38,7 +39,7 @@ private:
// True is the default.
bool verify_certificate_;
// Method: GET, HEAD, POST, etc. Must be all-caps.
// Method: GET, HEAD, or POST. Must be all-caps.
eng::string method_;
// The hostname. Not yet url-encoded.
@@ -53,6 +54,12 @@ private:
// URL parameters to append to the path. Not yet url-encoded.
UrlParameters params_;
// The mime type of the content, only for POST requests.
eng::string mime_type_;
// The content as a string, only for POST requests.
eng::string content_;
private:
void fail(std::string_view error);
void send_internal(StreamBuffer *target, bool debug_string) const;
@@ -70,6 +77,8 @@ public:
const eng::string &host() const { return host_; }
int port() const { return port_; }
const eng::string &path() const { return path_; }
const eng::string &mime_type() const { return mime_type_; }
const eng::string &content() const { return content_; }
// Populate an request-related fields one piece at a time.
// If you pass an invalid value, or if the field is
@@ -85,6 +94,8 @@ public:
void set_path(std::string_view path);
void set_param(const eng::string &key, const eng::string &value);
void set_url(std::string_view url);
void set_mime_type(const eng::string &mime_type);
void set_content(const eng::string &content);
void set_verify_certificate(LuaStack &LS, LuaSlot val);
void set_method(LuaStack &LS, LuaSlot val);
@@ -94,6 +105,8 @@ public:
void set_param(LuaStack &LS, LuaSlot key, LuaSlot val);
void set_params(LuaStack &LS, LuaSlot tab);
void set_url(LuaStack &LS, LuaSlot val);
void set_mime_type(LuaStack &LS, LuaSlot val);
void set_content(LuaStack &LS, LuaSlot val);
// Set default values for method and port.
// This must be done after setting regular values.
@@ -129,18 +142,21 @@ public:
eng::string DebugString();
};
class HttpClientResponse {
private:
// The request ID.
int64_t request_id_;
// HttpParser is used for parsing both requests and responses.
// Stores a status code, an error message, headers, and content.
//
class HttpParser : public eng::nevernew {
protected:
// True if this is for parsing a request.
// This is only used for generating error messages.
bool is_request_;
// The HTTP response status code.
int status_code_;
// The status code, a 3-digit number.
int status_;
// If the HTTP response contains an error, the
// error message is stored here. If the HTTP response
// is a success such as "200 OK" or "201 Created", this
// is the empty string, not "OK" or "Created".
// If the communication contains an error, the error can
// be stored here. In the event of a successful communication,
// this should always be empty string.
eng::string error_;
// Only if content-length header present, otherwise, -1.
@@ -158,50 +174,144 @@ private:
// MIME type of the content.
eng::string mime_type_;
// Charset of the content. Hopefully utf-8.
// Charset of the content before the content was translated to utf-8.
eng::string charset_;
// The content as string.
// The method, GET, HEAD, or POST. (only when parsing requests)
eng::string method_;
// The URL path, not url-encoded. (only when parsing requests)
eng::string path_;
// The URL parameters, not url-encoded (only when parsing requests)
UrlParameters params_;
// The content as string. If it's text, it's been translated to utf-8.
eng::string content_;
// The length in bytes of the entire response.
// May be zero, which means that the response
// was so garbled that we couldn't determine the length.
int response_length_;
private:
// The length in bytes of the entire communication.
// If the response is complete, but the comm_length_
// is zero, it means we couldn't find the end of
// the request.
int comm_length_;
protected:
// Store a message indicating that we haven't received enough
// bytes yet. If the connection is closed and we still haven't
// received enough bytes, that's a fatal error.
//
void incomplete(bool closed);
// Parse a response header. Most headers are ignored.
// If the header contains an error, the error is stored.
void parse_header(std::string_view header, std::string_view value);
// Store a message indicating that we couldn't parse because
// something was syntactically malformed, ie, not valid HTTP.
//
void syntax(std::string_view detail);
// Store a message indicating that the content was too large
// and we refused to download it.
//
void oversized();
// If the status is not in the range 200-299 (OK),
// then discard the content.
void clear_content_on_error();
// Parse a response status line, such as "HTTP/1.1 200 OK"
// returns false if we couldn't parse the whole line.
//
bool parse_status_line(std::string_view &view, bool closed);
// Parse a request line, such as "GET /index.html HTTP/1.1"
// returns false if we couldn't parse the whole line.
//
bool parse_request_line(std::string_view &view, bool closed);
// Parse specific headers.
// For several headers, all we do is verify that they aren't
// invoking unsupported features.
//
void parse_content_encoding(std::string_view value);
void parse_content_length(std::string_view value);
void parse_content_type(std::string_view value);
void parse_location(std::string_view value);
void parse_transfer_encoding(std::string_view value);
// parse the body
// Parse a single response header. Most headers are ignored.
// If the header contains an error, the error is stored.
//
void parse_header(std::string_view header, std::string_view value);
// Parse all of the headers, including the blank line after
// the headers. Return true if the view is at the end of the headers.
//
bool parse_headers(std::string_view &view, bool closed);
// parse the content, for different transfer encodings.
// Returns true if the view is at the end of the content.
//
bool parse_content_basic(std::string_view &view, bool closed);
bool parse_content_chunked(std::string_view &view, bool closed);
// Parse the content. Return true if the view is at the end of the
// content.
//
// - Uses the appropriate transfer-encoding specified in the headers.
// - Decompresses the content using the appropriate content-encoding.
// - Guesses a mimetype and charset if one was not specified in headers.
// - If it's text, converts the charset to utf-8.
//
bool parse_content(std::string_view &view, bool closed);
// Get or set the communication length.
//
int comm_length() const { return comm_length_; }
void set_comm_length(int len) { comm_length_ = len; }
// Construct a blank parser.
//
HttpParser();
// Store the parsed fields into a lua table.
//
void store_parsed(LuaStack &LS, LuaSlot tab) const;
// Emit all the parser fields as a debug string.
//
void parser_generate_debug_string(std::ostream &oss) const;
public:
// The parser will not try to parse content longer than this.
//
const int64_t MAX_CONTENT_LENGTH = 1000000;
// Construct a blank response.
HttpClientResponse();
// Store a result code and an error message, and clear the content.
// Store a status code and an error message, and clear the content.
// This is generally used when the client detects an error,
// such as a DNS lookup fail, a connection failed, an SSL negotiation
// failed, or the like.
void fail(int status_code, std::string_view error);
//
void fail(int status, std::string_view error);
// Return true if the communication was complete.
//
bool complete() const { return status_ != 0; }
};
class HttpClientResponse : public HttpParser {
private:
// Most of the data is stored in the HttpParser.
// That includes the status code, error, headers, and content.
// The request ID.
int64_t request_id_;
// The length in bytes of the entire communication.
// If the response is complete, but the comm_length_
// is zero, it means we couldn't find the end of
// the request.
//
int comm_length_;
public:
// Construct a blank response.
HttpClientResponse();
// Parse the HTTP response. The closed flag is to be set to true if the
// remote has closed the connection.
@@ -209,28 +319,62 @@ public:
// If the request is incomplete, generates a status code of zero. In that
// case, loading more data from the server might improve the situation.
//
// Note that the response is not ever removed from the StreamBuffer, which
// is always unmodified. If you want to remove the response from the
// StreamBuffer, see response_length.
// If successful, indicates how much of the text was consumed by
// setting comm_length_.
//
void parse(const StreamBuffer *sb, bool closed);
// Return true if the response is complete.
bool complete() const { return status_code_ != 0; }
void parse(std::string_view text, bool closed);
// Get or Set the request ID.
// This class does nothing with the request ID, it just stores it.
//
int64_t request_id() const { return request_id_; }
void set_request_id(int64_t v) { request_id_ = v; }
// Convert the HTTP response to a lua table.
//
void store(LuaStack &LS, LuaSlot tab) const;
// Convert to a debug string.
//
eng::string DebugString() const;
// Synthesize an error response and store it in lua.
static void store_fail(LuaStack &LS, LuaSlot tab, int status_code, std::string_view error);
//
static void store_fail(LuaStack &LS, LuaSlot tab, int status, std::string_view error);
};
class HttpServerRequest : public HttpParser {
private:
// The length in bytes of the entire communication.
// If the request is complete, but the comm_length_
// is zero, it means we couldn't find the end of
// the request.
//
int comm_length_;
public:
// Construct a blank request.
//
HttpServerRequest();
// Parse the HTTP request. The closed flag is to be set to true if the
// remote has closed the connection.
//
// If the request is incomplete, generates a status code of zero. In that
// case, loading more data from the server might improve the situation.
//
// If successful, indicates how much of the text was consumed by
// setting comm_length.
//
void parse(std::string_view text, bool closed);
// Convert the HTTP request to a lua table.
//
void store(LuaStack &LS, LuaSlot tab) const;
// Convert to a debug string.
//
eng::string DebugString() const;
};
class HttpClientRequestMap : public eng::map<int64_t, HttpClientRequest> {

2
luprex/core/cpp/lpxserver.cpp
View File

@@ -225,7 +225,7 @@ public:
response.fail(503, util::ss("Service Unavailable: ", channel.error()));
} else {
htchan.parsed_bytes_ = channel.in()->fill();
response.parse(channel.in(), channel.closed());
response.parse(channel.in()->view(), channel.closed());
}
if (response.complete()) {
response.set_request_id(pair.first);

13
luprex/core/cpp/util.cpp
View File

@@ -218,6 +218,19 @@ string_view read_nbytes(string_view &source, int nbytes) {
return result;
}
string_view read_ascii_identifier(string_view &source) {
size_t len = 0;
if ((len < source.size()) && (sv::ascii_isalpha(source[len]))) {
len += 1;
while ((len < source.size()) && (sv::ascii_isalnum(source[len]))) {
len += 1;
}
}
string_view result = source.substr(0, len);
source.remove_prefix(len);
return result;
}
bool valid_utf8(string_view s)
{
const unsigned char *bytes = (const unsigned char *)s.data();

6
luprex/core/cpp/util.hpp
View File

@@ -126,6 +126,12 @@ string_view read_to_space(string_view &source);
//
string_view read_nbytes(string_view &source, int nbytes);
// Read an ascii identifier from a string_view
//
// If there's no valid identifier, returns empty string.
//
string_view read_ascii_identifier(string_view &source);
// Return true if the string is valid utf-8.
bool valid_utf8(string_view s);

2
luprex/core/cpp/world-accessor.cpp
View File

@@ -525,7 +525,7 @@ LuaDefine(doc, "function",
LuaDefine(http_get, "request",
"|Make an HTTP GET request. Returns an HTTP response."
"|See doc(http.request) and doc(http.response).") {
"|See doc(http.clientrequest) and doc(http.clientresponse).") {
World *w = World::fetch_global_pointer(L);
w->guard_blockable(L, "http.get");