Future/bison
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/bison.git synced 2026-03-19 01:03:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

d: change the return value of yylex from TokenKind to YYParser.Symbol

Browse Source 
The complete symbol approach was deemed to be the right approach for Dlang.
Now, the user can return from yylex() an instance of YYParser.Symbol structure,
which binds together the TokenKind, the semantic value and the location. Before,
the last two were reported separately to the parser.
Only the user API is changed, Bisons's internal structure is kept the same.

* data/skeletons/d.m4 (struct YYParser.Symbol): New.
* data/skeletons/lalr1.d: Change the return value.
* doc/bison.texi: Document it.
* examples/d/calc/calc.y, examples/d/simple/calc.y: Demonstrate it.
* tests/calc.at, tests/scanner.at: Test it.
This commit is contained in:
Adela Vais
2020-11-11 22:35:30 +02:00
committed by Akim Demaille
parent 2ca158c893
commit 10305f3e94
8 changed files with 95 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/calc.at
View File

@@ -590,7 +590,7 @@ class CalcLexer(R) : Lexer
return res;
}
TokenKind yylex ()
YYParser.Symbol yylex ()
{]AT_LOCATION_IF([[
location.begin = location.end;]])[
@@ -606,13 +606,13 @@ class CalcLexer(R) : Lexer
// EOF.
if (input.empty)
return TokenKind.]AT_TOKEN_PREFIX[EOF;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
// Numbers.
if (input.front.isNumber)
{
semanticVal_.ival = parseInt;
return TokenKind.]AT_TOKEN_PREFIX[NUM;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NUM, semanticVal_.ival]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
// Individual characters
@@ -630,22 +630,22 @@ class CalcLexer(R) : Lexer
if (c == '#')
{
stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error: invalid character: '#'");
return TokenKind.]AT_TOKEN_PREFIX[YYerror;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYerror]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
switch (c)
{
case '+': return TokenKind.]AT_TOKEN_PREFIX[PLUS;
case '-': return TokenKind.]AT_TOKEN_PREFIX[MINUS;
case '*': return TokenKind.]AT_TOKEN_PREFIX[STAR;
case '/': return TokenKind.]AT_TOKEN_PREFIX[SLASH;
case '(': return TokenKind.]AT_TOKEN_PREFIX[LPAR;
case ')': return TokenKind.]AT_TOKEN_PREFIX[RPAR;
case '\n': return TokenKind.]AT_TOKEN_PREFIX[EOL;
case '=': return TokenKind.]AT_TOKEN_PREFIX[EQUAL;
case '^': return TokenKind.]AT_TOKEN_PREFIX[POW;
case '!': return TokenKind.]AT_TOKEN_PREFIX[NOT;
default: return TokenKind.]AT_TOKEN_PREFIX[YYUNDEF;
case '+': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[PLUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '-': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[MINUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '*': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[STAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '/': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[SLASH]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '(': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[LPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case ')': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[RPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '\n': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '=': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EQUAL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '^': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[POW]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '!': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NOT]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
default: return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYUNDEF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
}
}

8
tests/d.at
View File

@@ -81,7 +81,7 @@ class CalcLexer(R) : Lexer
YYSemanticType semanticVal_;
YYSemanticType semanticVal() @property { return semanticVal_; }
TokenKind yylex()
YYParser.Symbol yylex()
{
$2
}
@@ -143,16 +143,16 @@ AT_KEYWORDS([d])
AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw true
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])
AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw false
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 258,]])
AT_CHECK_D_MINIMAL_W_LEXER([
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])
AT_CLEANUP

18
tests/scanner.at
View File

@@ -121,12 +121,12 @@ class YYLexer(R) : Lexer
return semanticVal_;
}
TokenKind yylex ()
YYParser.Symbol yylex ()
{
import std.uni : isNumber;
// Handle EOF.
if (input.empty)
return TokenKind.END;
return YYParser.Symbol(TokenKind.END);
auto c = input.front;
input.popFront;
@@ -136,13 +136,13 @@ class YYLexer(R) : Lexer
{
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
semanticVal_.val = c - '0';
return TokenKind.NUM;
case '+': return TokenKind.PLUS;
case '-': return TokenKind.MINUS;
case '*': return TokenKind.STAR;
case '/': return TokenKind.SLASH;
case '(': return TokenKind.LPAR;
case ')': return TokenKind.RPAR;
return YYParser.Symbol(TokenKind.NUM, semanticVal_.val);
case '+': return YYParser.Symbol(TokenKind.PLUS);
case '-': return YYParser.Symbol(TokenKind.MINUS);
case '*': return YYParser.Symbol(TokenKind.STAR);
case '/': return YYParser.Symbol(TokenKind.SLASH);
case '(': return YYParser.Symbol(TokenKind.LPAR);
case ')': return YYParser.Symbol(TokenKind.RPAR);
default: assert(0);
}
}