diff --git a/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp b/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp index db324447..00b5d264 100644 --- a/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp +++ b/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp @@ -50,7 +50,10 @@ DefinesStreamProxy::DefineParameterPosition::DefineParameterPosition(const unsig } DefinesStreamProxy::Define::Define() - : m_contains_token_pasting_operators(false) {}; + : m_parameterized(false), + m_contains_token_pasting_operators(false) +{ +} DefinesStreamProxy::Define::Define(std::string name, std::string value) : Define(std::move(name), std::move(value), false) @@ -761,39 +764,27 @@ namespace unsigned m_end; }; - void EmitPastedTokens(const ParserLine& line, - const unsigned& linePos, - std::ostream& out, - const std::string& input, - const TokenPasteToken& token0, - const TokenPasteToken& token1) + void FlushTokenPastingResult(std::ostringstream& pasteOutput, std::ostringstream& pasted, const bool pastedIsString) { - if ((token0.m_type == TokenPasteTokenType::STRING) != (token1.m_type == TokenPasteTokenType::STRING)) - throw ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast(linePos + 1)), - "String token can only use token-pasting operator on other string token"); - if (token0.m_type == TokenPasteTokenType::STRING) - { - out << '"'; - token0.EmitValue(out, input); - token1.EmitValue(out, input); - out << '"'; - } + if (pastedIsString) + pasteOutput << '"' << pasted.str() << '"'; else - { - assert(token0.m_type == TokenPasteTokenType::IDENTIFIER || token0.m_type == TokenPasteTokenType::SYMBOL); + pasteOutput << pasted.str(); - token0.EmitValue(out, input); - token1.EmitValue(out, input); - } - } + pasted.str(std::string()); + }; } // namespace void DefinesStreamProxy::ProcessTokenPastingOperators(const ParserLine& line, const unsigned& linePos, std::string& input) { - std::ostringstream ss; + std::ostringstream pasteOutput; + + std::ostringstream pasted; + auto pasting = false; + auto pastedIsString = false; + auto copyStart = 0u; auto pasteNext = false; - TokenPasteToken previousToken; TokenPasteToken currentToken; const auto inputSize = input.size(); @@ -812,10 +803,17 @@ void DefinesStreamProxy::ProcessTokenPastingOperators(const ParserLine& line, co if (currentToken.m_type == TokenPasteTokenType::NONE) throw ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without previous token"); - if (previousToken.m_end < currentToken.m_start) - ss << std::string(input, previousToken.m_end, currentToken.m_start - previousToken.m_end); + if (!pasting) + { + if (copyStart < currentToken.m_start) + pasteOutput << std::string(input, copyStart, currentToken.m_start - copyStart); + + currentToken.EmitValue(pasted, input); + pastedIsString = currentToken.m_type == TokenPasteTokenType::STRING; + copyStart = currentToken.m_end; + pasting = true; + } - previousToken = currentToken; pasteNext = true; // Skip second # @@ -826,20 +824,34 @@ void DefinesStreamProxy::ProcessTokenPastingOperators(const ParserLine& line, co currentToken.SetFromInput(line, linePos, input, i); if (pasteNext) { - EmitPastedTokens(line, linePos, ss, input, previousToken, currentToken); - previousToken = currentToken; + if (pastedIsString != (currentToken.m_type == TokenPasteTokenType::STRING)) + { + throw ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast(linePos + 1)), + "String token can only use token-pasting operator on other string token"); + } + + currentToken.EmitValue(pasted, input); + copyStart = currentToken.m_end; pasteNext = false; } + else if (pasting) + { + FlushTokenPastingResult(pasteOutput, pasted, pastedIsString); + pasting = false; + } } } - if (inputSize > previousToken.m_end) - ss << std::string(input, previousToken.m_end, inputSize - previousToken.m_end); - if (pasteNext) throw ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without following token"); - input = ss.str(); + if (pasting) + FlushTokenPastingResult(pasteOutput, pasted, pastedIsString); + + if (inputSize > copyStart) + pasteOutput << std::string(input, copyStart, inputSize - copyStart); + + input = pasteOutput.str(); } void DefinesStreamProxy::InsertMacroParameters(std::ostringstream& out, const Define* macro, const std::vector& parameterValues) diff --git a/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp b/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp index be5e4863..aef72382 100644 --- a/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp +++ b/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp @@ -1147,6 +1147,26 @@ namespace test::parsing::impl::defines_stream_proxy REQUIRE(proxy.Eof()); } + TEST_CASE("DefinesStreamProxy: Can chain token-pasting operators", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define member(list, i) list##_member##i", + "member(party, 0)", + "#define glue(a, b, c) a ## b ## c", + "glue(\"Hello\", \" \", \"World\")", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "party_member0"); + ExpectLine(&proxy, 3, ""); + ExpectLine(&proxy, 4, "\"Hello World\""); + + REQUIRE(proxy.Eof()); + } + TEST_CASE("DefinesStreamProxy: Can use token-pasting operator with string and stringization outside macro", "[parsing][parsingstream]") { const std::vector lines{