zone/OpenAssetTools
zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-04-19 15:52:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Leave in token joining operator but do stringize forward lookup instead

Browse Source
This commit is contained in:
Jan 2023-12-30 18:42:00 +01:00
parent edb88273e7
commit eece5bb91d
No known key found for this signature in database
GPG Key ID: 44B581F78FF5C57C
3 changed files with 64 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/Parser/Parsing/Impl/DefinesStreamProxy.cpp
View File

@ -26,11 +26,13 @@ DefinesStreamProxy::DefineParameterPosition::DefineParameterPosition(const unsig
{ {
} }
DefinesStreamProxy::Define::Define() = default; DefinesStreamProxy::Define::Define()
: m_contains_token_joining_operators(false){};
DefinesStreamProxy::Define::Define(std::string name, std::string value) DefinesStreamProxy::Define::Define(std::string name, std::string value)
: m_name(std::move(name)), : m_name(std::move(name)),
m_value(std::move(value)) m_value(std::move(value)),
m_contains_token_joining_operators(false)
{ {
} }
@ -39,10 +41,10 @@ DefinesStreamProxy::MacroParameterState::MacroParameterState()
{ {
} }
bool DefinesStreamProxy::Define::IsStringizeParameterBackwardsLookup(const std::string& value, unsigned pos) bool DefinesStreamProxy::Define::IsStringizeParameterForwardLookup(const std::string& value, unsigned pos)
{ {
// Check if # is prepended to the word // Check if # is prepended to the word
return pos > 0 && value[pos - 1] == '#'; return pos + 1 && (isalpha(value[pos + 1]) || value[pos + 1] == '_');
} }
bool DefinesStreamProxy::Define::IsTokenJoiningOperatorForwardLookup(const std::string& value, unsigned pos) bool DefinesStreamProxy::Define::IsTokenJoiningOperatorForwardLookup(const std::string& value, unsigned pos)
@ -54,15 +56,10 @@ void DefinesStreamProxy::Define::IdentifyTokenJoinsOnly()
{ {
for (auto i = 0u; i < m_value.size(); i++) for (auto i = 0u; i < m_value.size(); i++)
{ {
const auto c = m_value[i]; if (m_value[i] == '#' && IsTokenJoiningOperatorForwardLookup(m_value, i))
if (!isalnum(c) && c != '_')
{ {
if (c == '#' && IsTokenJoiningOperatorForwardLookup(m_value, i)) m_contains_token_joining_operators = true;
{ return;
m_token_joins.push_back(i);
m_value.erase(i, 2);
i -= 1;
}
} }
} }
} }
@ -76,6 +73,7 @@ void DefinesStreamProxy::Define::IdentifyParameters(const std::vector<std::strin
} }
auto inWord = false; auto inWord = false;
auto stringizeNext = false;
auto wordStart = 0u; auto wordStart = 0u;
for (auto i = 0u; i < m_value.size(); i++) for (auto i = 0u; i < m_value.size(); i++)
{ {
@ -95,22 +93,28 @@ void DefinesStreamProxy::Define::IdentifyParameters(const std::vector<std::strin
if (parameterIndex < parameterNames.size()) if (parameterIndex < parameterNames.size())
{ {
const auto stringize = IsStringizeParameterBackwardsLookup(m_value, wordStart); const auto stringizeOffset = stringizeNext ? 1 : 0;
const auto stringizeOffset = stringize ? 1 : 0;
m_value.erase(wordStart - stringizeOffset, i - wordStart + stringizeOffset); m_value.erase(wordStart - stringizeOffset, i - wordStart + stringizeOffset);
m_parameter_positions.emplace_back(parameterIndex, wordStart - stringizeOffset, stringize); m_parameter_positions.emplace_back(parameterIndex, wordStart - stringizeOffset, stringizeNext);
i = wordStart - stringizeOffset; i = wordStart - stringizeOffset;
} }
inWord = false; inWord = false;
stringizeNext = false;
} }
if (c == '#' && IsTokenJoiningOperatorForwardLookup(m_value, i)) if (c == '#')
{ {
m_token_joins.push_back(i); if (IsStringizeParameterForwardLookup(m_value, i))
m_value.erase(i, 2); stringizeNext = true;
i -= 1; else if (IsTokenJoiningOperatorForwardLookup(m_value, i))
{
m_contains_token_joining_operators = true;
// Skip next char since it's # anyway and we do not want to count it as stringize
i++;
}
} }
} }
else else
@ -136,11 +140,10 @@ void DefinesStreamProxy::Define::IdentifyParameters(const std::vector<std::strin
if (parameterIndex < parameterNames.size()) if (parameterIndex < parameterNames.size())
{ {
const auto stringize = IsStringizeParameterBackwardsLookup(m_value, wordStart); const auto stringizeOffset = stringizeNext ? 1 : 0;
const auto stringizeOffset = stringize ? 1 : 0;
m_value.erase(wordStart - stringizeOffset, m_value.size() - wordStart + stringizeOffset); m_value.erase(wordStart - stringizeOffset, m_value.size() - wordStart + stringizeOffset);
m_parameter_positions.emplace_back(parameterIndex, wordStart - stringizeOffset, stringize); m_parameter_positions.emplace_back(parameterIndex, wordStart - stringizeOffset, stringizeNext);
} }
} }
} }
@ -692,6 +695,7 @@ void DefinesStreamProxy::ExpandMacro(ParserLine& line,
std::string str = rawOutput.str(); std::string str = rawOutput.str();
unsigned nestedPos = 0; unsigned nestedPos = 0;
ProcessNestedMacros(line, linePos, callstack, str, nestedPos); ProcessNestedMacros(line, linePos, callstack, str, nestedPos);
out << str; out << str;
} }

4
src/Parser/Parsing/Impl/DefinesStreamProxy.h
View File

@ -42,14 +42,14 @@ public:
std::string m_name; std::string m_name;
std::string m_value; std::string m_value;
std::vector<DefineParameterPosition> m_parameter_positions; std::vector<DefineParameterPosition> m_parameter_positions;
std::vector<unsigned> m_token_joins; bool m_contains_token_joining_operators;
Define(); Define();
Define(std::string name, std::string value); Define(std::string name, std::string value);
void IdentifyParameters(const std::vector<std::string>& parameterNames); void IdentifyParameters(const std::vector<std::string>& parameterNames);
private: private:
static bool IsStringizeParameterBackwardsLookup(const std::string& value, unsigned pos); static bool IsStringizeParameterForwardLookup(const std::string& value, unsigned pos);
static bool IsTokenJoiningOperatorForwardLookup(const std::string& value, unsigned pos); static bool IsTokenJoiningOperatorForwardLookup(const std::string& value, unsigned pos);
void IdentifyTokenJoinsOnly(); void IdentifyTokenJoinsOnly();
}; };

36
test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp
View File

@ -1093,6 +1093,42 @@ namespace test::parsing::impl::defines_stream_proxy
REQUIRE(proxy.Eof()); REQUIRE(proxy.Eof());
} }
TEST_CASE("DefinesStreamProxy: Can use token-pasting operator after length deflating macros", "[parsing][parsingstream]")
{
const std::vector<std::string> lines{
"#define VERY_LONG_MACRO \"T\"",
"#define testMacro(a) VERY_LONG_MACRO VERY_LONG_MACRO \"Hello\"##a VERY_LONG_MACRO",
"testMacro(\"World\")",
};
MockParserLineStream mockStream(lines);
DefinesStreamProxy proxy(&mockStream);
ExpectLine(&proxy, 1, "");
ExpectLine(&proxy, 2, "");
ExpectLine(&proxy, 3, "\"T\" \"T\" \"HelloWorld\" \"T\"");
REQUIRE(proxy.Eof());
}
TEST_CASE("DefinesStreamProxy: Can use token-pasting operator after length inflating macros", "[parsing][parsingstream]")
{
const std::vector<std::string> lines{
"#define BLA \"This is very long\"",
"#define testMacro(a) BLA BLA \"Hello\"##a BLA",
"testMacro(\"World\")",
};
MockParserLineStream mockStream(lines);
DefinesStreamProxy proxy(&mockStream);
ExpectLine(&proxy, 1, "");
ExpectLine(&proxy, 2, "");
ExpectLine(&proxy, 3, "\"This is very long\" \"This is very long\" \"HelloWorld\" \"This is very long\"");
REQUIRE(proxy.Eof());
}
TEST_CASE("DefinesStreamProxy: Interprets nested macros in context of top-level macro", "[parsing][parsingstream]") TEST_CASE("DefinesStreamProxy: Interprets nested macros in context of top-level macro", "[parsing][parsingstream]")
{ {
const std::vector<std::string> lines{ const std::vector<std::string> lines{