talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More refactors to prepare for doc-search, including moving unicode support into ext.

Browse Source
This commit is contained in:
jyelon
2026-01-14 12:30:44 -05:00
parent 4e374294b6
commit 850b4aa43b
10 changed files with 334 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
luprex/cpp/drv/drvutil.cpp
View File

@@ -7,6 +7,7 @@
#include <fstream>
#include <iostream>
#include <filesystem>
#include "../../ext/unicode-stuff.hpp"
namespace drvutil {
@@ -74,189 +75,12 @@ bool is_single_wchar_t(char32_t c) {
return false;
}
static int buffer_codepoint_utf8(char32_t scp, char *buffer) {
uint32_t cp = (uint32_t)scp;
unsigned char *c = (unsigned char *)buffer;
if (cp < 0) {
return 0;
}
else if (cp <= 0x7F) {
c[0] = cp;
return 1;
}
else if (cp <= 0x7FF) {
c[0] = (cp>>6)+192;
c[1] = (cp&63)+128;
return 2;
}
else if (cp <= 0xFFFF) {
if ((cp >= 0xD800) && (cp <= 0xDFFF)) {
return 0;
}
c[0] = (cp>>12)+224;
c[1] = ((cp>>6)&63)+128;
c[2] = (cp&63)+128;
return 3;
}
else if (cp <= 0x10FFFF) {
c[0] = (cp>>18)+240;
c[1] = ((cp>>12)&63)+128;
c[2] = ((cp>>6)&63)+128;
c[3] = (cp&63)+128;
return 4;
} else {
return 0;
}
}
using UC = UnicodeStuff<std::string, std::u16string, std::u32string>;
static int32_t read_codepoint_utf16(std::u16string_view &source) {
if (source.empty()) return -1;
int32_t word0 = ((const uint16_t *)source.data())[0];
source.remove_prefix(1);
if (word0 < 0xD800) {
return word0;
} else if (word0 < 0xDC00) {
if (source.empty()) {
return -2;
}
int32_t word1 = ((const uint16_t *)source.data())[0];
if ((word1 < 0xDC00)||(word1 > 0xDFFF)) {
return -2;
}
int32_t part1 = word0 & 0x3FF;
int32_t part2 = word1 & 0x3FF;
int32_t result = ((part1 << 10) | part2) + 0x10000;
source.remove_prefix(1);
return result;
} else if (word0 < 0xE000) {
return -2;
} else {
return word0;
}
}
static int32_t read_codepoint_utf8(std::string_view &source) {
size_t size = source.size();
if (size == 0) return -1;
const unsigned char *bytes = (const unsigned char *)source.data();
int codepoint;
size_t seqlen;
if ((bytes[0] & 0x80) == 0x00) {
// U+0000 to U+007F
codepoint = (bytes[0] & 0x7F);
seqlen = 1;
} else if ((bytes[0] & 0xE0) == 0xC0) {
// U+0080 to U+07FF
codepoint = (bytes[0] & 0x1F);
seqlen = 2;
} else if ((bytes[0] & 0xF0) == 0xE0) {
// U+0800 to U+FFFF
codepoint = (bytes[0] & 0x0F);
seqlen = 3;
} else if ((bytes[0] & 0xF8) == 0xF0) {
// U+10000 to U+10FFFF
codepoint = (bytes[0] & 0x07);
seqlen = 4;
} else {
// Bad character. return invalid CP.
return -2;
}
if (seqlen > size) {
return -1;
}
for (size_t i = 1; i < seqlen; ++i) {
if ((bytes[i] & 0xC0) != 0x80) {
// Bad character. return invalid CP.
return -2;
}
codepoint = (codepoint << 6) | (bytes[i] & 0x3F);
}
if ((codepoint > 0x10FFFF) ||
((codepoint >= 0xD800) && (codepoint <= 0xDFFF)) ||
((codepoint <= 0x007F) && (seqlen != 1)) ||
((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) ||
((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) ||
((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) {
// Bad character. return invalid CP.
return -2;
}
source.remove_prefix(seqlen);
return codepoint;
}
std::string utf32_to_utf8(const std::u32string &s) {
std::string result(s.size() * 4, 0);
char *buffer = &result[0];
int len = 0;
for (char32_t c : s) {
int clen = buffer_codepoint_utf8(c, buffer + len);
len += clen;
}
return result.substr(0, len);
}
std::u32string utf8_to_utf32(std::string_view s, int *consumed) {
std::string_view rest = s;
std::u32string result(s.size(), 0);
int len = 0;
while (true) {
int32_t c = read_codepoint_utf8(rest);
if (c == -1) {
break; // EOF reached;
} else if (c < 0) {
rest.remove_prefix(1);
} else {
result[len++] = (char32_t)c;
}
}
if (consumed != nullptr) {
*consumed = s.size() - rest.size();
}
return result.substr(0, len);
}
std::u16string utf8_to_ucs2(std::string_view s, int *consumed) {
std::string_view rest = s;
std::u16string result(s.size(), 0);
int len = 0;
while (true) {
int32_t c = read_codepoint_utf8(rest);
if (c == -1) {
break; // EOF reached;
} else if (c < 0) {
rest.remove_prefix(1);
} else if ((c >= 0xD800) && (c <= 0xDFFF)) {
result[len++] = 0x2610;
} else if (c > 0xFFFF) {
result[len++] = 0x2610;
} else {
result[len++] = (char16_t)c;
}
}
if (consumed != nullptr) {
*consumed = s.size() - rest.size();
}
return result.substr(0, len);
}
std::string utf16_to_utf8(std::u16string_view s) {
std::string result(s.size() * 4, 0);
int len = 0;
while (true) {
int codepoint = read_codepoint_utf16(s);
if (codepoint == -1) break;
if (codepoint < 0) continue;
len += buffer_codepoint_utf8(codepoint, &result[len]);
}
return result.substr(0, len);
}
std::string utf32_to_utf8(const std::u32string &s) { return UC::utf32_to_utf8(s); }
std::u32string utf8_to_utf32(std::string_view s, int *consumed) { return UC::utf8_to_utf32(s, consumed); }
std::u16string utf8_to_ucs2(std::string_view s, int *consumed) { return UC::utf8_to_ucs2(s, consumed); }
std::string utf16_to_utf8(std::u16string_view s) { return UC::utf16_to_utf8(s); }
static std::vector<std::string> parse_control_lst(std::string_view ctrl) {
std::vector<std::string> result;