Future/bison
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/bison.git synced 2026-03-09 12:23:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

d: change the return value of yylex from TokenKind to YYParser.Symbol

Browse Source 
The complete symbol approach was deemed to be the right approach for Dlang.
Now, the user can return from yylex() an instance of YYParser.Symbol structure,
which binds together the TokenKind, the semantic value and the location. Before,
the last two were reported separately to the parser.
Only the user API is changed, Bisons's internal structure is kept the same.

* data/skeletons/d.m4 (struct YYParser.Symbol): New.
* data/skeletons/lalr1.d: Change the return value.
* doc/bison.texi: Document it.
* examples/d/calc/calc.y, examples/d/simple/calc.y: Demonstrate it.
* tests/calc.at, tests/scanner.at: Test it.
This commit is contained in:
Adela Vais
2020-11-11 22:35:30 +02:00
committed by Akim Demaille
parent 2ca158c893
commit 10305f3e94
8 changed files with 95 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
data/skeletons/d.m4
View File

@@ -446,3 +446,38 @@ m4_define([b4_var_decls],
], [$@])])
m4_define([b4_var_decl],
[ protected $1;])
# b4_symbol_type_define
# ---------------------
# Define symbol_type, the external type for symbols used for symbol
# constructors.
m4_define([b4_symbol_type_define],
[[
/**
* A complete symbol
*/
struct Symbol
{
private SymbolKind kind;
private ]b4_yystype[ value;]b4_locations_if([[
private YYLocation location_;]])[
this(TokenKind token]b4_locations_if([[, YYLocation loc]])[)
{
kind = yytranslate_(token);]b4_locations_if([
location_ = loc;])[
}
static foreach (member; __traits(allMembers, YYSemanticType))
{
this(TokenKind token, typeof(mixin("YYSemanticType." ~ member)) val]b4_locations_if([[, YYLocation loc]])[)
{
kind = yytranslate_(token);
mixin("value." ~ member ~ " = val;");]b4_locations_if([
location_ = loc;])[
}
}
SymbolKind token() { return kind; }
]b4_yystype[ semanticValue() { return value; }]b4_locations_if([[
YYLocation location() { return location_; }]])[
}
]])

19
data/skeletons/lalr1.d
View File

@@ -75,7 +75,7 @@ public interface Lexer
* to the next token and prepares to return the semantic value
* ]b4_locations_if([and beginning/ending positions ])[of the token.
* @@return the token identifier corresponding to the next token. */
TokenKind yylex ();
]b4_parser_class[.Symbol yylex ();
/**
* Entry point for error reporting. Emits an error
@@ -290,7 +290,7 @@ b4_user_union_members
yyDebugStream.writeln (s);
}
]])[
private final TokenKind yylex () {
private final ]b4_parser_class[.Symbol yylex () {
return yylexer.yylex ();
}
@@ -411,7 +411,9 @@ b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
yycdebugln (message);
}
}
]])[
]])
b4_symbol_type_define
[
/**
* Parse input from the scanner that was specified at object construction
* time. Return whether the end of the input was reached successfully.
@@ -493,13 +495,10 @@ m4_popdef([b4_at_dollar])])dnl
if (yychar == TokenKind.]b4_symbol(empty, id)[)
{]b4_parse_trace_if([[
yycdebugln ("Reading a token");]])[
yychar = yylex ();]b4_locations_if([[
static if (yy_location_is_class) {
yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos);
} else {
yylloc = ]b4_location_type[(yylexer.startPos, yylexer.endPos);
}]])
yylval = yylexer.semanticVal;[
Symbol yysymbol = yylex();
yychar = yysymbol.token();
yylval = yysymbol.semanticValue();]b4_locations_if([[
yylloc = yysymbol.location();]])[
}
/* Convert token to internal form. */

6
doc/bison.texi
View File

@@ -14011,9 +14011,9 @@ This method is defined by the user to emit an error message. The first
parameter is omitted if location tracking is not active.
@end deftypemethod
@deftypemethod {Lexer} {TokenKind} yylex()
Return the next token. Its type is the return value, its semantic value and
location are saved and returned by the their methods in the interface.
@deftypemethod {Lexer} {YYParser.Symbol} yylex()
Return the next token. The return value is of type YYParser.Symbol, which
binds together the TokenKind, the semantic value and the location.
@end deftypemethod
@deftypemethod {Lexer} {YYPosition} getStartPos()

20
examples/d/calc/calc.y
View File

@@ -114,7 +114,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_;
}
TokenKind yylex()
Calc.Symbol yylex()
{
import std.uni : isWhite, isNumber;
@@ -127,7 +127,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
}
if (input.empty)
return TokenKind.YYEOF;
return Calc.Symbol(TokenKind.YYEOF, new YYLocation(startPos, endPos));
// Numbers.
if (input.front.isNumber)
@@ -143,7 +143,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
}
start = end;
end.column += lenChars;
return TokenKind.NUM;
return Calc.Symbol(TokenKind.NUM, semanticVal_.ival, new YYLocation(startPos, endPos));
}
// Individual characters
@@ -153,17 +153,17 @@ if (isInputRange!R && is(ElementType!R : dchar))
end.column++;
switch (ch)
{
case '+': return TokenKind.PLUS;
case '-': return TokenKind.MINUS;
case '*': return TokenKind.STAR;
case '/': return TokenKind.SLASH;
case '(': return TokenKind.LPAR;
case ')': return TokenKind.RPAR;
case '+': return Calc.Symbol(TokenKind.PLUS, new YYLocation(startPos, endPos));
case '-': return Calc.Symbol(TokenKind.MINUS, new YYLocation(startPos, endPos));
case '*': return Calc.Symbol(TokenKind.STAR, new YYLocation(startPos, endPos));
case '/': return Calc.Symbol(TokenKind.SLASH, new YYLocation(startPos, endPos));
case '(': return Calc.Symbol(TokenKind.LPAR, new YYLocation(startPos, endPos));
case ')': return Calc.Symbol(TokenKind.RPAR, new YYLocation(startPos, endPos));
case '\n':
{
end.line++;
end.column = 1;
return TokenKind.EOL;
return Calc.Symbol(TokenKind.EOL, new YYLocation(startPos, endPos));
}
default: assert(0);
}

21
examples/d/simple/calc.y
View File

@@ -109,7 +109,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_;
}
TokenKind yylex()
Calc.Symbol yylex()
{
import std.uni : isWhite, isNumber;
@@ -118,14 +118,13 @@ if (isInputRange!R && is(ElementType!R : dchar))
input.popFront;
if (input.empty)
return TokenKind.YYEOF;
return Calc.Symbol(TokenKind.YYEOF);
// Numbers.
if (input.front.isNumber)
{
import std.conv : parse;
semanticVal_.ival = input.parse!int;
return TokenKind.NUM;
return Calc.Symbol(TokenKind.NUM, input.parse!int);
}
// Individual characters
@@ -133,13 +132,13 @@ if (isInputRange!R && is(ElementType!R : dchar))
input.popFront;
switch (ch)
{
case '+': return TokenKind.PLUS;
case '-': return TokenKind.MINUS;
case '*': return TokenKind.STAR;
case '/': return TokenKind.SLASH;
case '(': return TokenKind.LPAR;
case ')': return TokenKind.RPAR;
case '\n': return TokenKind.EOL;
case '+': return Calc.Symbol(TokenKind.PLUS);
case '-': return Calc.Symbol(TokenKind.MINUS);
case '*': return Calc.Symbol(TokenKind.STAR);
case '/': return Calc.Symbol(TokenKind.SLASH);
case '(': return Calc.Symbol(TokenKind.LPAR);
case ')': return Calc.Symbol(TokenKind.RPAR);
case '\n': return Calc.Symbol(TokenKind.EOL);
default: assert(0);
}
}

30
tests/calc.at
View File

@@ -590,7 +590,7 @@ class CalcLexer(R) : Lexer
return res;
}
TokenKind yylex ()
YYParser.Symbol yylex ()
{]AT_LOCATION_IF([[
location.begin = location.end;]])[
@@ -606,13 +606,13 @@ class CalcLexer(R) : Lexer
// EOF.
if (input.empty)
return TokenKind.]AT_TOKEN_PREFIX[EOF;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
// Numbers.
if (input.front.isNumber)
{
semanticVal_.ival = parseInt;
return TokenKind.]AT_TOKEN_PREFIX[NUM;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NUM, semanticVal_.ival]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
// Individual characters
@@ -630,22 +630,22 @@ class CalcLexer(R) : Lexer
if (c == '#')
{
stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error: invalid character: '#'");
return TokenKind.]AT_TOKEN_PREFIX[YYerror;
return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYerror]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
switch (c)
{
case '+': return TokenKind.]AT_TOKEN_PREFIX[PLUS;
case '-': return TokenKind.]AT_TOKEN_PREFIX[MINUS;
case '*': return TokenKind.]AT_TOKEN_PREFIX[STAR;
case '/': return TokenKind.]AT_TOKEN_PREFIX[SLASH;
case '(': return TokenKind.]AT_TOKEN_PREFIX[LPAR;
case ')': return TokenKind.]AT_TOKEN_PREFIX[RPAR;
case '\n': return TokenKind.]AT_TOKEN_PREFIX[EOL;
case '=': return TokenKind.]AT_TOKEN_PREFIX[EQUAL;
case '^': return TokenKind.]AT_TOKEN_PREFIX[POW;
case '!': return TokenKind.]AT_TOKEN_PREFIX[NOT;
default: return TokenKind.]AT_TOKEN_PREFIX[YYUNDEF;
case '+': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[PLUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '-': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[MINUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '*': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[STAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '/': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[SLASH]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '(': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[LPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case ')': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[RPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '\n': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '=': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EQUAL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '^': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[POW]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '!': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NOT]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
default: return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYUNDEF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
}
}

8
tests/d.at
View File

@@ -81,7 +81,7 @@ class CalcLexer(R) : Lexer
YYSemanticType semanticVal_;
YYSemanticType semanticVal() @property { return semanticVal_; }
TokenKind yylex()
YYParser.Symbol yylex()
{
$2
}
@@ -143,16 +143,16 @@ AT_KEYWORDS([d])
AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw true
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])
AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw false
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 258,]])
AT_CHECK_D_MINIMAL_W_LEXER([
%union { int ival; }], [return TokenKind.END;])
%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])
AT_CLEANUP

18
tests/scanner.at
View File

@@ -121,12 +121,12 @@ class YYLexer(R) : Lexer
return semanticVal_;
}
TokenKind yylex ()
YYParser.Symbol yylex ()
{
import std.uni : isNumber;
// Handle EOF.
if (input.empty)
return TokenKind.END;
return YYParser.Symbol(TokenKind.END);
auto c = input.front;
input.popFront;
@@ -136,13 +136,13 @@ class YYLexer(R) : Lexer
{
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
semanticVal_.val = c - '0';
return TokenKind.NUM;
case '+': return TokenKind.PLUS;
case '-': return TokenKind.MINUS;
case '*': return TokenKind.STAR;
case '/': return TokenKind.SLASH;
case '(': return TokenKind.LPAR;
case ')': return TokenKind.RPAR;
return YYParser.Symbol(TokenKind.NUM, semanticVal_.val);
case '+': return YYParser.Symbol(TokenKind.PLUS);
case '-': return YYParser.Symbol(TokenKind.MINUS);
case '*': return YYParser.Symbol(TokenKind.STAR);
case '/': return YYParser.Symbol(TokenKind.SLASH);
case '(': return YYParser.Symbol(TokenKind.LPAR);
case ')': return YYParser.Symbol(TokenKind.RPAR);
default: assert(0);
}
}