gbdev/rgbds
1
0
Fork 0
mirror of https://github.com/gbdev/rgbds.git synced 2026-08-20 05:44:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Encapuslate UTF-8 decoder state in a struct

Browse Source
This commit is contained in:
Rangi
2026-07-03 19:35:23 -04:00
parent e3c594c250
commit 0996a2f5ed
5 changed files with 36 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/extern/utf8decoder.hpp
Vendored
+6 -1
View File
@@ -8,6 +8,11 @@
#define UTF8_ACCEPT 0
#define UTF8_REJECT 12
uint32_t decode(uint32_t *state, uint32_t *codep, uint8_t byte);
struct Utf8Decoder {
uint32_t state = UTF8_ACCEPT;
uint32_t codepoint = 0;
uint32_t update(uint8_t byte);
};
#endif // RGBDS_EXTERN_UTF8DECODER_HPP
src/asm/actions.cpp
+16 -19
View File
@@ -279,18 +279,17 @@ static void errorInvalidUTF8Byte(uint8_t byte, char const *functionName) {
size_t act_StringLen(std::string const &str, bool printErrors) {
size_t len = 0;
uint32_t state = UTF8_ACCEPT;
uint32_t codepoint = 0;
Utf8Decoder decoder;
for (char c : str) {
uint8_t byte = static_cast<uint8_t>(c);
switch (decode(&state, &codepoint, byte)) {
switch (decoder.update(byte)) {
case UTF8_REJECT:
if (printErrors) {
errorInvalidUTF8Byte(byte, "STRLEN");
}
state = UTF8_ACCEPT;
decoder.state = UTF8_ACCEPT;
// fallthrough
case UTF8_ACCEPT:
++len;
@@ -299,7 +298,7 @@ size_t act_StringLen(std::string const &str, bool printErrors) {
}
// Check for partial code point.
if (state != UTF8_ACCEPT) {
if (decoder.state != UTF8_ACCEPT) {
if (printErrors) {
error("STRLEN: Incomplete UTF-8 character");
}
@@ -317,16 +316,15 @@ std::string
size_t strLen = str.length();
size_t index = 0;
uint32_t state = UTF8_ACCEPT;
uint32_t codepoint = 0;
Utf8Decoder decoder;
uint32_t curIdx = 0;
// Advance to starting index in source string.
while (index < strLen && curIdx < start) {
switch (decode(&state, &codepoint, str[index])) {
switch (decoder.update(str[index])) {
case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSLICE");
state = UTF8_ACCEPT;
decoder.state = UTF8_ACCEPT;
// fallthrough
case UTF8_ACCEPT:
++curIdx;
@@ -349,10 +347,10 @@ std::string
// Advance to ending index in source string.
while (index < strLen && curIdx < stop) {
switch (decode(&state, &codepoint, str[index])) {
switch (decoder.update(str[index])) {
case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSLICE");
state = UTF8_ACCEPT;
decoder.state = UTF8_ACCEPT;
// fallthrough
case UTF8_ACCEPT:
++curIdx;
@@ -362,7 +360,7 @@ std::string
}
// Check for partial code point.
if (state != UTF8_ACCEPT) {
if (decoder.state != UTF8_ACCEPT) {
error("STRSLICE: Incomplete UTF-8 character");
++curIdx;
}
@@ -387,16 +385,15 @@ std::string act_StringSub(std::string const &str, int32_t negPos, std::optional<
size_t strLen = str.length();
size_t index = 0;
uint32_t state = UTF8_ACCEPT;
uint32_t codepoint = 0;
Utf8Decoder decoder;
uint32_t curPos = 1;
// Advance to starting position in source string.
while (index < strLen && curPos < pos) {
switch (decode(&state, &codepoint, str[index])) {
switch (decoder.update(str[index])) {
case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSUB");
state = UTF8_ACCEPT;
decoder.state = UTF8_ACCEPT;
// fallthrough
case UTF8_ACCEPT:
++curPos;
@@ -418,10 +415,10 @@ std::string act_StringSub(std::string const &str, int32_t negPos, std::optional<
// Compute the result length in bytes.
while (index < strLen && curLen < len) {
switch (decode(&state, &codepoint, str[index])) {
switch (decoder.update(str[index])) {
case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSUB");
state = UTF8_ACCEPT;
decoder.state = UTF8_ACCEPT;
// fallthrough
case UTF8_ACCEPT:
++curLen;
@@ -431,7 +428,7 @@ std::string act_StringSub(std::string const &str, int32_t negPos, std::optional<
}
// Check for partial code point.
if (state != UTF8_ACCEPT) {
if (decoder.state != UTF8_ACCEPT) {
error("STRSUB: Incomplete UTF-8 character");
++curLen;
}
src/asm/charmap.cpp
+3 -4
View File
@@ -273,15 +273,14 @@ size_t charmap_ConvertNext(std::string_view &input, std::vector<int32_t> *output
} else if (inputIdx < input.length()) { // No match found, but there is some input left
size_t codepointLen = 0;
// This will write the codepoint's value to `output`, little-endian
for (uint32_t state = UTF8_ACCEPT, codepoint = 0;
inputIdx + codepointLen < input.length();) {
if (decode(&state, &codepoint, input[inputIdx + codepointLen]) == UTF8_REJECT) {
for (Utf8Decoder decoder; inputIdx + codepointLen < input.length();) {
if (decoder.update(input[inputIdx + codepointLen]) == UTF8_REJECT) {
error("Input string is not valid UTF-8");
codepointLen = 1;
break;
}
++codepointLen;
if (state == UTF8_ACCEPT) {
if (decoder.state == UTF8_ACCEPT) {
break;
}
}
src/extern/utf8decoder.cpp
Vendored
+4 -4
View File
@@ -42,9 +42,9 @@ static uint8_t const utf8d[] = {
};
// clang-format on
uint32_t decode(uint32_t *state, uint32_t *codep, uint8_t byte) {
uint32_t Utf8Decoder::update(uint8_t byte) {
uint8_t type = utf8d[byte];
*codep = *state != UTF8_ACCEPT ? (byte & 0b111111) | (*codep << 6) : (0xff >> type) & byte;
*state = utf8d[0x100 + *state + type];
return *state;
codepoint = state != UTF8_ACCEPT ? (byte & 0b111111) | (codepoint << 6) : (0xff >> type) & byte;
state = utf8d[0x100 + state + type];
return state;
}
src/link/output.cpp
+7 -6
View File
@@ -275,23 +275,24 @@ static void writeSymName(std::string const &name, FILE *file) {
// Output illegal characters using Unicode escapes ('\u' or '\U')
// Decode the UTF-8 codepoint; or at least attempt to
uint32_t state = UTF8_ACCEPT, codepoint;
Utf8Decoder decoder;
do {
decode(&state, &codepoint, *ptr);
if (state != UTF8_REJECT) {
if (decoder.update(*ptr) != UTF8_REJECT) {
++ptr;
continue;
}
// This sequence was invalid; emit a U+FFFD, and recover
codepoint = 0xFFFD;
decoder.codepoint = 0xFFFD;
// Skip continuation bytes
// A NUL byte does not qualify, so we're good
while ((*ptr & 0xC0) == 0x80) {
++ptr;
}
break;
} while (state != UTF8_ACCEPT);
fprintf(file, codepoint <= 0xFFFF ? "\\u%04" PRIx32 : "\\U%08" PRIx32, codepoint);
} while (decoder.state != UTF8_ACCEPT);
fprintf(
file, decoder.codepoint <= 0xFFFF ? "\\u%04" PRIx32 : "\\U%08" PRIx32, decoder.codepoint
);
}
}