From 3743537f9ee767f2150a353628f881e47bd6da71 Mon Sep 17 00:00:00 2001
From: mo <michael.oliver_@outlook.com>
Date: Mon, 20 Jul 2026 20:08:59 +0100
Subject: [PATCH] fix: handle UTF-8 BOM-prefixed input (#919)

Fixes https://github.com/Laupetin/OpenAssetTools/issues/771
---
 src/Parser/Parsing/IParserLineStream.cpp      | 10 +++++++
 .../Impl/ParserSingleInputStreamTests.cpp     | 29 +++++++++++++++++++
 .../Parsing/Simple/SimpleExpressionTests.cpp  | 18 ++++++++++++
 3 files changed, 57 insertions(+)
 create mode 100644 test/ParserTests/Parsing/Impl/ParserSingleInputStreamTests.cpp

diff --git a/src/Parser/Parsing/IParserLineStream.cpp b/src/Parser/Parsing/IParserLineStream.cpp
index 60ba91c0..e9712182 100644
--- a/src/Parser/Parsing/IParserLineStream.cpp
+++ b/src/Parser/Parsing/IParserLineStream.cpp
@@ -1,5 +1,13 @@
 #include "IParserLineStream.h"
 
+#include <string_view>
+
+namespace
+{
+    // A UTF-8 BOM is an encoding signature and is only valid at the start of a file.
+    constexpr std::string_view UTF8_BOM("\xEF\xBB\xBF", 3);
+} // namespace
+
 ParserLine::ParserLine()
     : m_line_number(0)
 {
@@ -10,6 +18,8 @@ ParserLine::ParserLine(std::shared_ptr<std::string> filename, const int lineNumb
       m_line_number(lineNumber),
       m_line(std::move(line))
 {
+    if (m_line_number == 1 && m_line.starts_with(UTF8_BOM))
+        m_line.erase(0, UTF8_BOM.size());
 }
 
 bool ParserLine::IsEof() const
diff --git a/test/ParserTests/Parsing/Impl/ParserSingleInputStreamTests.cpp b/test/ParserTests/Parsing/Impl/ParserSingleInputStreamTests.cpp
new file mode 100644
index 00000000..5fd637d8
--- /dev/null
+++ b/test/ParserTests/Parsing/Impl/ParserSingleInputStreamTests.cpp
@@ -0,0 +1,29 @@
+#include "Parsing/Impl/ParserSingleInputStream.h"
+
+#include <catch2/catch_test_macros.hpp>
+#include <sstream>
+
+namespace test::parsing::impl::parser_single_input_stream
+{
+    TEST_CASE("ParserSingleInputStream: Strips UTF-8 BOM at start of input", "[parsing][parsingstream]")
+    {
+        std::istringstream input("\xEF\xBB\xBF"
+                                 "first line\nsecond line");
+        ParserSingleInputStream stream(input, "input");
+
+        REQUIRE(stream.NextLine().m_line == "first line");
+        REQUIRE(stream.NextLine().m_line == "second line");
+    }
+
+    TEST_CASE("ParserSingleInputStream: Preserves UTF-8 BOM outside start of input", "[parsing][parsingstream]")
+    {
+        std::istringstream input("first line\n\xEF\xBB\xBF"
+                                 "second line");
+        ParserSingleInputStream stream(input, "input");
+
+        REQUIRE(stream.NextLine().m_line == "first line");
+        REQUIRE(stream.NextLine().m_line
+                == "\xEF\xBB\xBF"
+                   "second line");
+    }
+} // namespace test::parsing::impl::parser_single_input_stream
diff --git a/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp b/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp
index 4c41db56..071245fa 100644
--- a/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp
+++ b/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp
@@ -808,6 +808,24 @@ namespace test::parsing::simple::expression
 
     namespace it
     {
+        TEST_CASE("SimpleExpressionsIT: Can parse UTF-8 BOM-prefixed input", "[parsing][simple][expression][it]")
+        {
+            SimpleExpressionTestsHelper helper;
+            helper.String("\xEF\xBB\xBF"
+                          "6+5");
+
+            const auto result = helper.PerformIntegrationTest();
+
+            REQUIRE(result);
+
+            const auto& expression = helper.m_state->m_expression;
+            REQUIRE(expression->IsStatic());
+
+            const auto value = expression->EvaluateStatic();
+            REQUIRE(value.m_type == SimpleExpressionValue::Type::INT);
+            REQUIRE(value.m_int_value == 11);
+        }
+
         TEST_CASE("SimpleExpressionsIT: Can parse subtraction without space", "[parsing][simple][expression][it]")
         {
             SimpleExpressionTestsHelper helper;