fix: handle UTF-8 BOM-prefixed input (#919)

Fixes https://github.com/Laupetin/OpenAssetTools/issues/771
This commit is contained in:
mo
2026-07-20 21:08:59 +02:00
committed by GitHub
parent fadcd74182
commit 3743537f9e
3 changed files with 57 additions and 0 deletions
@@ -0,0 +1,29 @@
#include "Parsing/Impl/ParserSingleInputStream.h"
#include <catch2/catch_test_macros.hpp>
#include <sstream>
namespace test::parsing::impl::parser_single_input_stream
{
TEST_CASE("ParserSingleInputStream: Strips UTF-8 BOM at start of input", "[parsing][parsingstream]")
{
std::istringstream input("\xEF\xBB\xBF"
"first line\nsecond line");
ParserSingleInputStream stream(input, "input");
REQUIRE(stream.NextLine().m_line == "first line");
REQUIRE(stream.NextLine().m_line == "second line");
}
TEST_CASE("ParserSingleInputStream: Preserves UTF-8 BOM outside start of input", "[parsing][parsingstream]")
{
std::istringstream input("first line\n\xEF\xBB\xBF"
"second line");
ParserSingleInputStream stream(input, "input");
REQUIRE(stream.NextLine().m_line == "first line");
REQUIRE(stream.NextLine().m_line
== "\xEF\xBB\xBF"
"second line");
}
} // namespace test::parsing::impl::parser_single_input_stream
@@ -808,6 +808,24 @@ namespace test::parsing::simple::expression
namespace it
{
TEST_CASE("SimpleExpressionsIT: Can parse UTF-8 BOM-prefixed input", "[parsing][simple][expression][it]")
{
SimpleExpressionTestsHelper helper;
helper.String("\xEF\xBB\xBF"
"6+5");
const auto result = helper.PerformIntegrationTest();
REQUIRE(result);
const auto& expression = helper.m_state->m_expression;
REQUIRE(expression->IsStatic());
const auto value = expression->EvaluateStatic();
REQUIRE(value.m_type == SimpleExpressionValue::Type::INT);
REQUIRE(value.m_int_value == 11);
}
TEST_CASE("SimpleExpressionsIT: Can parse subtraction without space", "[parsing][simple][expression][it]")
{
SimpleExpressionTestsHelper helper;