Lots of work on unicode support

luprex/cpp/drv/drvutil.cpp

@@ -77,6 +77,131 @@ void split_target(std::string_view target, std::string &cert, std::string &host,
     port = std::string(split[2]);
 }
 
+bool is_single_wchar_t(char32_t c) {
+    if ((c >= 0xD800) && (c <= 0xDFFF)) return false;
+    if ((c >= 0) && (c <= 0xFFFF)) return true;
+    return false;
+}
+
+static int buffer_codepoint_utf8(char32_t scp, char *buffer) {
+    uint32_t cp = (uint32_t)scp;
+    unsigned char *c = (unsigned char *)buffer;
+    if (cp < 0) {
+        return 0;
+    }
+    else if (cp <= 0x7F) {
+        c[0] = cp;
+        return 1;
+    }
+    else if (cp <= 0x7FF) {
+        c[0] = (cp>>6)+192;
+        c[1] = (cp&63)+128;
+        return 2;
+    }
+    else if (cp <= 0xFFFF) {
+        if ((cp >= 0xD800) && (cp <= 0xDFFF)) {
+            return 0;
+        }
+        c[0] = (cp>>12)+224;
+        c[1] = ((cp>>6)&63)+128;
+        c[2] = (cp&63)+128;
+        return 3;
+    }
+    else if (cp <= 0x10FFFF) {
+        c[0] = (cp>>18)+240;
+        c[1] = ((cp>>12)&63)+128;
+        c[2] = ((cp>>6)&63)+128;
+        c[3] = (cp&63)+128;
+        return 4;
+    } else {
+        return 0;
+    }
+}
+
+static int32_t read_codepoint_utf8(std::string_view &source) {
+    size_t size = source.size();
+    if (size == 0) return -1;
+
+    const unsigned char *bytes = (const unsigned char *)source.data();
+    int codepoint;
+    size_t seqlen;
+    if ((bytes[0] & 0x80) == 0x00) {
+        // U+0000 to U+007F
+        codepoint = (bytes[0] & 0x7F);
+        seqlen = 1;
+    } else if ((bytes[0] & 0xE0) == 0xC0) {
+        // U+0080 to U+07FF
+        codepoint = (bytes[0] & 0x1F);
+        seqlen = 2;
+    } else if ((bytes[0] & 0xF0) == 0xE0) {
+        // U+0800 to U+FFFF
+        codepoint = (bytes[0] & 0x0F);
+        seqlen = 3;
+    } else if ((bytes[0] & 0xF8) == 0xF0) {
+        // U+10000 to U+10FFFF
+        codepoint = (bytes[0] & 0x07);
+        seqlen = 4;
+    } else {
+        // Bad character. return invalid CP.
+        return -2;
+    }
+
+    if (seqlen > size) {
+        return -1;
+    }
+
+    for (size_t i = 1; i < seqlen; ++i) {
+        if ((bytes[i] & 0xC0) != 0x80) {
+            // Bad character. return invalid CP.
+            return -2;
+        }
+        codepoint = (codepoint << 6) | (bytes[i] & 0x3F);
+    }
+
+    if ((codepoint > 0x10FFFF) ||
+        ((codepoint >= 0xD800) && (codepoint <= 0xDFFF)) ||
+        ((codepoint <= 0x007F) && (seqlen != 1)) ||
+        ((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) ||
+        ((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) ||
+        ((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) {
+        // Bad character. return invalid CP.
+        return -2;
+    }
+
+    source.remove_prefix(seqlen);
+    return codepoint;
+}
+
+std::string to_utf8(const std::u32string &s) {
+    std::string result(s.size() * 4, 0);
+    char *buffer = &result[0];
+    int len = 0;
+    for (char32_t c : s) {
+        int clen = buffer_codepoint_utf8(c, buffer + len);
+        len += clen;
+    }
+    return result.substr(0, len);
+}
+
+std::u32string from_utf8(std::string_view s, int *consumed) {
+    std::string_view rest = s;
+    std::u32string result(s.size(), 0);
+    int len = 0;
+    while (true) {
+        int32_t c = read_codepoint_utf8(rest);
+        if (c == -1) {
+            break; // EOF reached;
+        } else if (c < 0) {
+            rest.remove_prefix(1);
+        } else {
+            result[len++] = (char32_t)c;
+        }
+    }
+    if (consumed != nullptr) {
+        *consumed = s.size() - rest.size();
+    }
+    return result.substr(0, len);
+}
 
 static std::vector<std::string> parse_control_lst(std::string_view ctrl) {
     std::vector<std::string> result;