gbdev/rgbds
1
0
Fork 0
mirror of https://github.com/gbdev/rgbds.git synced 2026-09-03 04:18:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Encapuslate UTF-8 decoder state in a struct

Browse Source
This commit is contained in:
Rangi
2026-07-03 19:35:23 -04:00
parent e3c594c250
commit 0996a2f5ed
5 changed files with 36 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/extern/utf8decoder.hpp
Vendored
+6 -1
View File
@@ -8,6 +8,11 @@
#define UTF8_ACCEPT 0 #define UTF8_ACCEPT 0
#define UTF8_REJECT 12 #define UTF8_REJECT 12
uint32_t decode(uint32_t *state, uint32_t *codep, uint8_t byte); struct Utf8Decoder {
uint32_t state = UTF8_ACCEPT;
uint32_t codepoint = 0;
uint32_t update(uint8_t byte);
};
#endif // RGBDS_EXTERN_UTF8DECODER_HPP #endif // RGBDS_EXTERN_UTF8DECODER_HPP
src/asm/actions.cpp
+16 -19
View File
@@ -279,18 +279,17 @@ static void errorInvalidUTF8Byte(uint8_t byte, char const *functionName) {
size_t act_StringLen(std::string const &str, bool printErrors) { size_t act_StringLen(std::string const &str, bool printErrors) {
size_t len = 0; size_t len = 0;
uint32_t state = UTF8_ACCEPT; Utf8Decoder decoder;
uint32_t codepoint = 0;
for (char c : str) { for (char c : str) {
uint8_t byte = static_cast<uint8_t>(c); uint8_t byte = static_cast<uint8_t>(c);
switch (decode(&state, &codepoint, byte)) { switch (decoder.update(byte)) {
case UTF8_REJECT: case UTF8_REJECT:
if (printErrors) { if (printErrors) {
errorInvalidUTF8Byte(byte, "STRLEN"); errorInvalidUTF8Byte(byte, "STRLEN");
} }
state = UTF8_ACCEPT; decoder.state = UTF8_ACCEPT;
// fallthrough // fallthrough
case UTF8_ACCEPT: case UTF8_ACCEPT:
++len; ++len;
@@ -299,7 +298,7 @@ size_t act_StringLen(std::string const &str, bool printErrors) {
} }
// Check for partial code point. // Check for partial code point.
if (state != UTF8_ACCEPT) { if (decoder.state != UTF8_ACCEPT) {
if (printErrors) { if (printErrors) {
error("STRLEN: Incomplete UTF-8 character"); error("STRLEN: Incomplete UTF-8 character");
} }
@@ -317,16 +316,15 @@ std::string
size_t strLen = str.length(); size_t strLen = str.length();
size_t index = 0; size_t index = 0;
uint32_t state = UTF8_ACCEPT; Utf8Decoder decoder;
uint32_t codepoint = 0;
uint32_t curIdx = 0; uint32_t curIdx = 0;
// Advance to starting index in source string. // Advance to starting index in source string.
while (index < strLen && curIdx < start) { while (index < strLen && curIdx < start) {
switch (decode(&state, &codepoint, str[index])) { switch (decoder.update(str[index])) {
case UTF8_REJECT: case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSLICE"); errorInvalidUTF8Byte(str[index], "STRSLICE");
state = UTF8_ACCEPT; decoder.state = UTF8_ACCEPT;
// fallthrough // fallthrough
case UTF8_ACCEPT: case UTF8_ACCEPT:
++curIdx; ++curIdx;
@@ -349,10 +347,10 @@ std::string
// Advance to ending index in source string. // Advance to ending index in source string.
while (index < strLen && curIdx < stop) { while (index < strLen && curIdx < stop) {
switch (decode(&state, &codepoint, str[index])) { switch (decoder.update(str[index])) {
case UTF8_REJECT: case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSLICE"); errorInvalidUTF8Byte(str[index], "STRSLICE");
state = UTF8_ACCEPT; decoder.state = UTF8_ACCEPT;
// fallthrough // fallthrough
case UTF8_ACCEPT: case UTF8_ACCEPT:
++curIdx; ++curIdx;
@@ -362,7 +360,7 @@ std::string
} }
// Check for partial code point. // Check for partial code point.
if (state != UTF8_ACCEPT) { if (decoder.state != UTF8_ACCEPT) {
error("STRSLICE: Incomplete UTF-8 character"); error("STRSLICE: Incomplete UTF-8 character");
++curIdx; ++curIdx;
} }
@@ -387,16 +385,15 @@ std::string act_StringSub(std::string const &str, int32_t negPos, std::optional<
size_t strLen = str.length(); size_t strLen = str.length();
size_t index = 0; size_t index = 0;
uint32_t state = UTF8_ACCEPT; Utf8Decoder decoder;
uint32_t codepoint = 0;
uint32_t curPos = 1; uint32_t curPos = 1;
// Advance to starting position in source string. // Advance to starting position in source string.
while (index < strLen && curPos < pos) { while (index < strLen && curPos < pos) {
switch (decode(&state, &codepoint, str[index])) { switch (decoder.update(str[index])) {
case UTF8_REJECT: case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSUB"); errorInvalidUTF8Byte(str[index], "STRSUB");
state = UTF8_ACCEPT; decoder.state = UTF8_ACCEPT;
// fallthrough // fallthrough
case UTF8_ACCEPT: case UTF8_ACCEPT:
++curPos; ++curPos;
@@ -418,10 +415,10 @@ std::string act_StringSub(std::string const &str, int32_t negPos, std::optional<
// Compute the result length in bytes. // Compute the result length in bytes.
while (index < strLen && curLen < len) { while (index < strLen && curLen < len) {
switch (decode(&state, &codepoint, str[index])) { switch (decoder.update(str[index])) {
case UTF8_REJECT: case UTF8_REJECT:
errorInvalidUTF8Byte(str[index], "STRSUB"); errorInvalidUTF8Byte(str[index], "STRSUB");
state = UTF8_ACCEPT; decoder.state = UTF8_ACCEPT;
// fallthrough // fallthrough
case UTF8_ACCEPT: case UTF8_ACCEPT:
++curLen; ++curLen;
@@ -431,7 +428,7 @@ std::string act_StringSub(std::string const &str, int32_t negPos, std::optional<
} }
// Check for partial code point. // Check for partial code point.
if (state != UTF8_ACCEPT) { if (decoder.state != UTF8_ACCEPT) {
error("STRSUB: Incomplete UTF-8 character"); error("STRSUB: Incomplete UTF-8 character");
++curLen; ++curLen;
} }
src/asm/charmap.cpp
+3 -4
View File
@@ -273,15 +273,14 @@ size_t charmap_ConvertNext(std::string_view &input, std::vector<int32_t> *output
} else if (inputIdx < input.length()) { // No match found, but there is some input left } else if (inputIdx < input.length()) { // No match found, but there is some input left
size_t codepointLen = 0; size_t codepointLen = 0;
// This will write the codepoint's value to `output`, little-endian // This will write the codepoint's value to `output`, little-endian
for (uint32_t state = UTF8_ACCEPT, codepoint = 0; for (Utf8Decoder decoder; inputIdx + codepointLen < input.length();) {
inputIdx + codepointLen < input.length();) { if (decoder.update(input[inputIdx + codepointLen]) == UTF8_REJECT) {
if (decode(&state, &codepoint, input[inputIdx + codepointLen]) == UTF8_REJECT) {
error("Input string is not valid UTF-8"); error("Input string is not valid UTF-8");
codepointLen = 1; codepointLen = 1;
break; break;
} }
++codepointLen; ++codepointLen;
if (state == UTF8_ACCEPT) { if (decoder.state == UTF8_ACCEPT) {
break; break;
} }
} }
src/extern/utf8decoder.cpp
Vendored
+4 -4
View File
@@ -42,9 +42,9 @@ static uint8_t const utf8d[] = {
}; };
// clang-format on // clang-format on
uint32_t decode(uint32_t *state, uint32_t *codep, uint8_t byte) { uint32_t Utf8Decoder::update(uint8_t byte) {
uint8_t type = utf8d[byte]; uint8_t type = utf8d[byte];
*codep = *state != UTF8_ACCEPT ? (byte & 0b111111) | (*codep << 6) : (0xff >> type) & byte; codepoint = state != UTF8_ACCEPT ? (byte & 0b111111) | (codepoint << 6) : (0xff >> type) & byte;
*state = utf8d[0x100 + *state + type]; state = utf8d[0x100 + state + type];
return *state; return state;
} }
src/link/output.cpp
+7 -6
View File
@@ -275,23 +275,24 @@ static void writeSymName(std::string const &name, FILE *file) {
// Output illegal characters using Unicode escapes ('\u' or '\U') // Output illegal characters using Unicode escapes ('\u' or '\U')
// Decode the UTF-8 codepoint; or at least attempt to // Decode the UTF-8 codepoint; or at least attempt to
uint32_t state = UTF8_ACCEPT, codepoint; Utf8Decoder decoder;
do { do {
decode(&state, &codepoint, *ptr); if (decoder.update(*ptr) != UTF8_REJECT) {
if (state != UTF8_REJECT) {
++ptr; ++ptr;
continue; continue;
} }
// This sequence was invalid; emit a U+FFFD, and recover // This sequence was invalid; emit a U+FFFD, and recover
codepoint = 0xFFFD; decoder.codepoint = 0xFFFD;
// Skip continuation bytes // Skip continuation bytes
// A NUL byte does not qualify, so we're good // A NUL byte does not qualify, so we're good
while ((*ptr & 0xC0) == 0x80) { while ((*ptr & 0xC0) == 0x80) {
++ptr; ++ptr;
} }
break; break;
} while (state != UTF8_ACCEPT); } while (decoder.state != UTF8_ACCEPT);
fprintf(file, codepoint <= 0xFFFF ? "\\u%04" PRIx32 : "\\U%08" PRIx32, codepoint); fprintf(
file, decoder.codepoint <= 0xFFFF ? "\\u%04" PRIx32 : "\\U%08" PRIx32, decoder.codepoint
);
} }
} }