QOL improvements to docsearch: alphabetize, filter out unittests

Integration.code-workspace.tpl.json

@@ -77,7 +77,16 @@
                 "presentation": {
                     "clear": true
                 },
-                "problemMatcher": "$msCompile",
+                "problemMatcher": [
+                    {
+                    "base": "$gcc",
+                    "fileLocation": ["relative", "${workspaceFolder}/luprex"]
+                    },
+                    {
+                    "base": "$gcc",
+                    "fileLocation": ["relative", "${workspaceFolder}"]
+                    }
+                ],
                 "type": "shell"
             }
         }

luprex/cpp/core/animqueue.cpp

@@ -323,7 +323,7 @@ void AnimState::parse(std::string_view config) {
     while (true) {
         config = sv::ltrim(config);
         if (config.empty()) break;
-        eng::string name(sv::read_ascii_identifier(config));
+        eng::string name(sv::read_simple_identifier(config));
         assert(!name.empty());
         AnimValue &value = map_[name];
         bool has_equal = sv::has_prefix(config, "=");

luprex/cpp/core/http.cpp

@@ -143,7 +143,7 @@ bool contains_newline(std::string_view s) {
 
 bool words_separated_by_dashes(string_view v) {
     while (true) {
-        string_view word = sv::read_ascii_identifier(v);
+        string_view word = sv::read_simple_identifier(v);
         if (word.empty()) return false;
         if (v.empty()) return true;
         char c = v.front();
@@ -156,12 +156,12 @@ bool words_separated_by_dashes(string_view v) {
 // registered, obviously.  It only checks that it's in
 // the desired notation.
 bool valid_mime_type(string_view method) {
-    string_view part1 = sv::read_ascii_identifier(method);
+    string_view part1 = sv::read_simple_identifier(method);
     if (part1.empty()) return false;
     if (sv::zfront(method) != '/') return false;
     method.remove_prefix(1);
     while (true) {
-        string_view word = sv::read_ascii_identifier(method);
+        string_view word = sv::read_simple_identifier(method);
         if (word.empty()) return false;
         if (method.empty()) return true;
         char c = method.front();

luprex/cpp/core/json.cpp

@@ -295,7 +295,7 @@ static bool encode_value(lua_State *L, eng::ostringstream &oss, int level, int m
 static bool decode_value(lua_State *L, std::string_view &v);
 
 static bool decode_id(lua_State *L, std::string_view &v) {
-    std::string_view id = sv::read_ascii_identifier(v);
+    std::string_view id = sv::read_simple_identifier(v);
     if (id == "null") lua_pushlightuserdata(L, LuaToken("null").voidvalue());
     else if (id == "true") lua_pushboolean(L, 1);
     else if (id == "false") lua_pushboolean(L, 0);

luprex/cpp/core/source.cpp

@@ -577,24 +577,26 @@ void SourceDB::register_lua_builtins() {
 }
 
 bool SourceDB::search_docs(const eng::string &substring, std::ostream &ostream) {
-    bool found_anything = false;
+    eng::map<eng::string, eng::string> results;
 
     // Search the built-in functions.
     for (const LuaFunctionReg *reg = LuaFunctionReg::All; reg != nullptr; reg=reg->next()) {
         eng::string proto = get_reg_prototype(reg);
         if (reg->get_sandbox()) continue;
+        if (sv::has_prefix(reg->get_name(), "unittests_")) continue;
         if (sv::contains_substring_utf8(proto, substring) ||
             sv::contains_substring_utf8(reg->get_docs(), substring)) {
-            ostream << proto;
+            std::ostringstream oss;
+            oss << proto;
             util::StringVec docs = util::split_docstring(reg->get_docs());
             for (const eng::string &line : docs) {
                 if (sv::contains_substring_utf8(line, substring)) {
-                    ostream << " -- " << sv::trim(line);
+                    oss << " -- " << sv::trim(line);
                     break;
                 }
             }
-            ostream << std::endl;
+            oss << std::endl;
-            found_anything = true;
+            results[get_reg_fullname(reg->get_name())] = oss.str();
         }
     }
 
@@ -606,17 +608,20 @@ bool SourceDB::search_docs(const eng::string &substring, std::ostream &ostream)
         util::StringVec lines = util::split_lines(code);
         int comment_lines = 0;
         for (int i = 0; i < int(lines.size()); i++) {
-            if (sv::is_lua_function_prototype(lines[i])) {
+            std::string_view fullname = sv::lua_function_proto_name(lines[i]);
+            if (sv::has_prefix(fullname, "unittests.")) continue;
+            if (!fullname.empty()) {
                 if (lines_contain_substring(lines, i - comment_lines, i+1, substring)) {
-                    ostream << lines[i];
+                    eng::ostringstream oss;
+                    oss << lines[i];
                     for (int j = i - comment_lines; j < i; j++) {
                         if (sv::contains_substring_utf8(lines[j], substring)) {
-                            ostream << " " << sv::trim(lines[j]);
+                            oss << " " << sv::trim(lines[j]);
                             break;
                         }
                     }
-                    ostream << std::endl;
+                    oss << std::endl;
-                    found_anything = true;
+                    results[eng::string(fullname)] = oss.str();
                 }
                 comment_lines = 0;
             } else if (sv::is_lua_comment(lines[i])) {
@@ -627,7 +632,10 @@ bool SourceDB::search_docs(const eng::string &substring, std::ostream &ostream)
         }
     }
 
-    return found_anything;
+    for (const auto &pair : results) {
+        ostream << pair.second;
+    }
+    return !results.empty();
 }
 
 bool SourceDB::function_docs(const LuaCoreStack &LS, LuaSlot fn, std::ostream &ostream) {

luprex/cpp/core/util.cpp

@@ -181,10 +181,10 @@ int common_prefix_length(string_view a, string_view b) {
 bool is_lua_id(string_view str) {
     if (str.size() == 0) return false;
     char c=str[0];
-    if ((!ascii_isalpha(c)) && (c!='_')) return false;
+    if (!ascii_isualpha(c)) return false;
     for (int i = 1; i < int(str.size()); i++) {
         char c = str[i];
-        if ((!ascii_isalnum(c)) && (c!='_')) return false;
+        if (!ascii_isualnum(c)) return false;
     }
     return true;
 }
@@ -200,15 +200,6 @@ bool is_lua_comment(string_view s) {
     return s.substr(start, 2) == "--";
 }
 
-bool is_lua_function_prototype(string_view s) {
-    int start = 0;
-    while ((start < int(s.size())) && ((s[start]==' ') || (s[start]=='\t'))) start++;
-    s.remove_prefix(start);
-    if (!has_prefix(s, "function")) return false;
-    s.remove_prefix(8);
-    return ((!s.empty()) && (ascii_isspace(s[0])));
-}
-
 bool is_whitespace(string_view s) {
     for (int i = 0; i < int(s.size()); i++) {
         if (!ascii_isspace(s[i])) {
@@ -218,6 +209,46 @@ bool is_whitespace(string_view s) {
     return true;
 }
 
+string_view lua_function_proto_name(string_view s) {
+    // Skip any leading whitespace.
+    read_space(s);
+
+    // Skip over the word 'function'.  If it's not there, fail.
+    if (!read_prefix(s, "function")) return string_view();
+
+    // Skip whitespace.  If there isn't any, fail.
+    string_view w = read_space(s);
+    if (w.empty()) return string_view();
+
+    // Read the function name
+    string_view work = s;
+    if (read_lua_identifier(work).empty()) return string_view();
+    if (read_prefix(work, ".")) {
+        if (read_lua_identifier(work).empty()) return string_view();
+    }
+    size_t namelen = s.size() - work.size();
+    string_view fullname = s.substr(0, namelen);
+    s.remove_prefix(namelen);
+
+    // Skip whitespace
+    read_space(s);
+
+    // Make sure there's an open parentheses.
+    if (!read_prefix(s, "(")) return string_view();
+
+    // This is where we stop parsing.
+    return fullname;
+}
+
+string_view read_space(string_view &source)
+{
+    size_t pos = 0;
+    while ((pos < source.size()) && ascii_isspace(source[pos])) pos++;
+    string_view result = source.substr(0, pos);
+    source.remove_prefix(pos);
+    return result;
+}
+
 string_view read_to_sep(string_view &source, char sep) {
     size_t pos = source.find(sep);
     string_view result;
@@ -282,7 +313,7 @@ string_view read_nbytes(string_view &source, int nbytes) {
     return result;
 }
 
-string_view read_ascii_identifier(string_view &source) {
+string_view read_simple_identifier(string_view &source) {
     size_t len = 0;
     if ((len < source.size()) && (sv::ascii_isalpha(source[len]))) {
         len += 1;
@@ -295,6 +326,19 @@ string_view read_ascii_identifier(string_view &source) {
     return result;
 }
 
+string_view read_lua_identifier(string_view &source) {
+    size_t len = 0;
+    if ((len < source.size()) && (sv::ascii_isualpha(source[len]))) {
+        len += 1;
+        while ((len < source.size()) && (sv::ascii_isualnum(source[len]))) {
+            len += 1;
+        }
+    }
+    string_view result = source.substr(0, len);
+    source.remove_prefix(len);
+    return result;
+}
+
 std::string_view read_number(string_view &source, bool plus, bool minus, bool dec, bool exp) {
     const char *p = source.data();
     const char *l = p + source.size();

luprex/cpp/core/util.hpp

@@ -49,7 +49,9 @@ inline bool ascii_isupper(char c) { return (c >= 'A') && (c <= 'Z'); }
 inline bool ascii_islower(char c) { return (c >= 'a') && (c <= 'z'); }
 inline bool ascii_isdigit(char c) { return (c >= '0') && (c <= '9'); }
 inline bool ascii_isalpha(char c) { return ascii_isupper(c) || ascii_islower(c); }
+inline bool ascii_isualpha(char c) { return ascii_isalpha(c) || (c == '_'); }
 inline bool ascii_isalnum(char c) { return ascii_isalpha(c) || ascii_isdigit(c); }
+inline bool ascii_isualnum(char c) { return ascii_isalpha(c) || ascii_isdigit(c) || (c == '_'); }
 inline bool ascii_isspace(char c) { return (c==' ')||(c=='\t')||(c=='\r')||(c=='\n')||(c=='\f')||(c=='\v'); }
 
 // Check for the null string_view
@@ -110,18 +112,24 @@ bool is_lua_classname(string_view s);
 // Return true if the line of code is a lua comment.
 bool is_lua_comment(string_view s);
 
-// Return true if the line is a lua function prototype.
-bool is_lua_function_prototype(string_view s);
-
 // Return true if the line is entirely whitespace.
 bool is_whitespace(string_view s);
 
+// Get the function name from a lua function prototype.
+// Returns empty string if the prototype is malformed or
+// is not a lua function prototype at all.
+string_view lua_function_proto_name(string_view s);
+
 // Return the first character, but if the view is empty,
 // return zero.
 inline char zfront(string_view &s) {
     return s.empty() ? char(0) : s.front();
 }
 
+// Read whitespace from a string_view.
+//
+string_view read_space(string_view &source);
+
 // Read from a string_view until separator is reached.
 //
 // If the separator appears in the source, returns everything
@@ -166,11 +174,19 @@ string_view read_to_space(string_view &source);
 //
 string_view read_nbytes(string_view &source, int nbytes);
 
-// Read an ascii identifier from a string_view
+// Read an identifier from a string_view
 //
 // If there's no valid identifier, returns empty string.
+// Underscores are not allowed in the identifier.
 //
-string_view read_ascii_identifier(string_view &source);
+string_view read_simple_identifier(string_view &source);
+
+// Read an identifier from a string_view
+//
+// If there's no valid identifier, returns empty string.
+// Lua identifiers are allowed to have underscores.
+//
+string_view read_lua_identifier(string_view &source);
 
 // Read a number from a string view
 //