Future/bison
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/bison.git synced 2026-03-09 04:13:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

d: change the return value of yylex from TokenKind to YYParser.Symbol

Browse Source 
The complete symbol approach was deemed to be the right approach for Dlang.
Now, the user can return from yylex() an instance of YYParser.Symbol structure,
which binds together the TokenKind, the semantic value and the location. Before,
the last two were reported separately to the parser.
Only the user API is changed, Bisons's internal structure is kept the same.

* data/skeletons/d.m4 (struct YYParser.Symbol): New.
* data/skeletons/lalr1.d: Change the return value.
* doc/bison.texi: Document it.
* examples/d/calc/calc.y, examples/d/simple/calc.y: Demonstrate it.
* tests/calc.at, tests/scanner.at: Test it.
This commit is contained in:
Adela Vais
2020-11-11 22:35:30 +02:00
committed by Akim Demaille
parent 2ca158c893
commit 10305f3e94
8 changed files with 95 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
data/skeletons/d.m4
View File

@@ -446,3 +446,38 @@ m4_define([b4_var_decls],
], [$@])]) ], [$@])])
m4_define([b4_var_decl], m4_define([b4_var_decl],
[ protected $1;]) [ protected $1;])
# b4_symbol_type_define
# ---------------------
# Define symbol_type, the external type for symbols used for symbol
# constructors.
m4_define([b4_symbol_type_define],
[[
/**
* A complete symbol
*/
struct Symbol
{
private SymbolKind kind;
private ]b4_yystype[ value;]b4_locations_if([[
private YYLocation location_;]])[
this(TokenKind token]b4_locations_if([[, YYLocation loc]])[)
{
kind = yytranslate_(token);]b4_locations_if([
location_ = loc;])[
}
static foreach (member; __traits(allMembers, YYSemanticType))
{
this(TokenKind token, typeof(mixin("YYSemanticType." ~ member)) val]b4_locations_if([[, YYLocation loc]])[)
{
kind = yytranslate_(token);
mixin("value." ~ member ~ " = val;");]b4_locations_if([
location_ = loc;])[
}
}
SymbolKind token() { return kind; }
]b4_yystype[ semanticValue() { return value; }]b4_locations_if([[
YYLocation location() { return location_; }]])[
}
]])

19
data/skeletons/lalr1.d
View File

@@ -75,7 +75,7 @@ public interface Lexer
* to the next token and prepares to return the semantic value * to the next token and prepares to return the semantic value
* ]b4_locations_if([and beginning/ending positions ])[of the token. * ]b4_locations_if([and beginning/ending positions ])[of the token.
* @@return the token identifier corresponding to the next token. */ * @@return the token identifier corresponding to the next token. */
TokenKind yylex (); ]b4_parser_class[.Symbol yylex ();
/** /**
* Entry point for error reporting. Emits an error * Entry point for error reporting. Emits an error
@@ -290,7 +290,7 @@ b4_user_union_members
yyDebugStream.writeln (s); yyDebugStream.writeln (s);
} }
]])[ ]])[
private final TokenKind yylex () { private final ]b4_parser_class[.Symbol yylex () {
return yylexer.yylex (); return yylexer.yylex ();
} }
@@ -411,7 +411,9 @@ b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
yycdebugln (message); yycdebugln (message);
} }
} }
]])[ ]])
b4_symbol_type_define
[
/** /**
* Parse input from the scanner that was specified at object construction * Parse input from the scanner that was specified at object construction
* time. Return whether the end of the input was reached successfully. * time. Return whether the end of the input was reached successfully.
@@ -493,13 +495,10 @@ m4_popdef([b4_at_dollar])])dnl
if (yychar == TokenKind.]b4_symbol(empty, id)[) if (yychar == TokenKind.]b4_symbol(empty, id)[)
{]b4_parse_trace_if([[ {]b4_parse_trace_if([[
yycdebugln ("Reading a token");]])[ yycdebugln ("Reading a token");]])[
yychar = yylex ();]b4_locations_if([[ Symbol yysymbol = yylex();
static if (yy_location_is_class) { yychar = yysymbol.token();
yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos); yylval = yysymbol.semanticValue();]b4_locations_if([[
} else { yylloc = yysymbol.location();]])[
yylloc = ]b4_location_type[(yylexer.startPos, yylexer.endPos);
}]])
yylval = yylexer.semanticVal;[
} }
/* Convert token to internal form. */ /* Convert token to internal form. */

6
doc/bison.texi
View File

@@ -14011,9 +14011,9 @@ This method is defined by the user to emit an error message. The first
parameter is omitted if location tracking is not active. parameter is omitted if location tracking is not active.
@end deftypemethod @end deftypemethod
@deftypemethod {Lexer} {TokenKind} yylex() @deftypemethod {Lexer} {YYParser.Symbol} yylex()
Return the next token. Its type is the return value, its semantic value and Return the next token. The return value is of type YYParser.Symbol, which
location are saved and returned by the their methods in the interface. binds together the TokenKind, the semantic value and the location.
@end deftypemethod @end deftypemethod
@deftypemethod {Lexer} {YYPosition} getStartPos() @deftypemethod {Lexer} {YYPosition} getStartPos()

20
examples/d/calc/calc.y
View File

@@ -114,7 +114,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_; return semanticVal_;
} }
TokenKind yylex() Calc.Symbol yylex()
{ {
import std.uni : isWhite, isNumber; import std.uni : isWhite, isNumber;
@@ -127,7 +127,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
} }
if (input.empty) if (input.empty)
return TokenKind.YYEOF; return Calc.Symbol(TokenKind.YYEOF, new YYLocation(startPos, endPos));
// Numbers. // Numbers.
if (input.front.isNumber) if (input.front.isNumber)
@@ -143,7 +143,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
} }
start = end; start = end;
end.column += lenChars; end.column += lenChars;
return TokenKind.NUM; return Calc.Symbol(TokenKind.NUM, semanticVal_.ival, new YYLocation(startPos, endPos));
} }
// Individual characters // Individual characters
@@ -153,17 +153,17 @@ if (isInputRange!R && is(ElementType!R : dchar))
end.column++; end.column++;
switch (ch) switch (ch)
{ {
case '+': return TokenKind.PLUS; case '+': return Calc.Symbol(TokenKind.PLUS, new YYLocation(startPos, endPos));
case '-': return TokenKind.MINUS; case '-': return Calc.Symbol(TokenKind.MINUS, new YYLocation(startPos, endPos));
case '*': return TokenKind.STAR; case '*': return Calc.Symbol(TokenKind.STAR, new YYLocation(startPos, endPos));
case '/': return TokenKind.SLASH; case '/': return Calc.Symbol(TokenKind.SLASH, new YYLocation(startPos, endPos));
case '(': return TokenKind.LPAR; case '(': return Calc.Symbol(TokenKind.LPAR, new YYLocation(startPos, endPos));
case ')': return TokenKind.RPAR; case ')': return Calc.Symbol(TokenKind.RPAR, new YYLocation(startPos, endPos));
case '\n': case '\n':
{ {
end.line++; end.line++;
end.column = 1; end.column = 1;
return TokenKind.EOL; return Calc.Symbol(TokenKind.EOL, new YYLocation(startPos, endPos));
} }
default: assert(0); default: assert(0);
} }

21
examples/d/simple/calc.y
View File

@@ -109,7 +109,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_; return semanticVal_;
} }
TokenKind yylex() Calc.Symbol yylex()
{ {
import std.uni : isWhite, isNumber; import std.uni : isWhite, isNumber;
@@ -118,14 +118,13 @@ if (isInputRange!R && is(ElementType!R : dchar))
input.popFront; input.popFront;
if (input.empty) if (input.empty)
return TokenKind.YYEOF; return Calc.Symbol(TokenKind.YYEOF);
// Numbers. // Numbers.
if (input.front.isNumber) if (input.front.isNumber)
{ {
import std.conv : parse; import std.conv : parse;
semanticVal_.ival = input.parse!int; return Calc.Symbol(TokenKind.NUM, input.parse!int);
return TokenKind.NUM;
} }
// Individual characters // Individual characters
@@ -133,13 +132,13 @@ if (isInputRange!R && is(ElementType!R : dchar))
input.popFront; input.popFront;
switch (ch) switch (ch)
{ {
case '+': return TokenKind.PLUS; case '+': return Calc.Symbol(TokenKind.PLUS);
case '-': return TokenKind.MINUS; case '-': return Calc.Symbol(TokenKind.MINUS);
case '*': return TokenKind.STAR; case '*': return Calc.Symbol(TokenKind.STAR);
case '/': return TokenKind.SLASH; case '/': return Calc.Symbol(TokenKind.SLASH);
case '(': return TokenKind.LPAR; case '(': return Calc.Symbol(TokenKind.LPAR);
case ')': return TokenKind.RPAR; case ')': return Calc.Symbol(TokenKind.RPAR);
case '\n': return TokenKind.EOL; case '\n': return Calc.Symbol(TokenKind.EOL);
default: assert(0); default: assert(0);
} }
} }

30
tests/calc.at
View File

@@ -590,7 +590,7 @@ class CalcLexer(R) : Lexer
return res; return res;
} }
TokenKind yylex () YYParser.Symbol yylex ()
{]AT_LOCATION_IF([[ {]AT_LOCATION_IF([[
location.begin = location.end;]])[ location.begin = location.end;]])[
@@ -606,13 +606,13 @@ class CalcLexer(R) : Lexer
// EOF. // EOF.
if (input.empty) if (input.empty)
return TokenKind.]AT_TOKEN_PREFIX[EOF; return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
// Numbers. // Numbers.
if (input.front.isNumber) if (input.front.isNumber)
{ {
semanticVal_.ival = parseInt; semanticVal_.ival = parseInt;
return TokenKind.]AT_TOKEN_PREFIX[NUM; return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NUM, semanticVal_.ival]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
} }
// Individual characters // Individual characters
@@ -630,22 +630,22 @@ class CalcLexer(R) : Lexer
if (c == '#') if (c == '#')
{ {
stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error: invalid character: '#'"); stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error: invalid character: '#'");
return TokenKind.]AT_TOKEN_PREFIX[YYerror; return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYerror]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
} }
switch (c) switch (c)
{ {
case '+': return TokenKind.]AT_TOKEN_PREFIX[PLUS; case '+': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[PLUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '-': return TokenKind.]AT_TOKEN_PREFIX[MINUS; case '-': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[MINUS]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '*': return TokenKind.]AT_TOKEN_PREFIX[STAR; case '*': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[STAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '/': return TokenKind.]AT_TOKEN_PREFIX[SLASH; case '/': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[SLASH]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '(': return TokenKind.]AT_TOKEN_PREFIX[LPAR; case '(': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[LPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case ')': return TokenKind.]AT_TOKEN_PREFIX[RPAR; case ')': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[RPAR]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '\n': return TokenKind.]AT_TOKEN_PREFIX[EOL; case '\n': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '=': return TokenKind.]AT_TOKEN_PREFIX[EQUAL; case '=': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EQUAL]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '^': return TokenKind.]AT_TOKEN_PREFIX[POW; case '^': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[POW]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
case '!': return TokenKind.]AT_TOKEN_PREFIX[NOT; case '!': return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NOT]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
default: return TokenKind.]AT_TOKEN_PREFIX[YYUNDEF; default: return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYUNDEF]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
} }
} }
} }

8
tests/d.at
View File

@@ -81,7 +81,7 @@ class CalcLexer(R) : Lexer
YYSemanticType semanticVal_; YYSemanticType semanticVal_;
YYSemanticType semanticVal() @property { return semanticVal_; } YYSemanticType semanticVal() @property { return semanticVal_; }
TokenKind yylex() YYParser.Symbol yylex()
{ {
$2 $2
} }
@@ -143,16 +143,16 @@ AT_KEYWORDS([d])
AT_CHECK_D_MINIMAL_W_LEXER([ AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw true %define api.token.raw true
%union { int ival; }], [return TokenKind.END;]) %union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]]) AT_CHECK_D_GREP([[ END = 3,]])
AT_CHECK_D_MINIMAL_W_LEXER([ AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw false %define api.token.raw false
%union { int ival; }], [return TokenKind.END;]) %union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 258,]]) AT_CHECK_D_GREP([[ END = 258,]])
AT_CHECK_D_MINIMAL_W_LEXER([ AT_CHECK_D_MINIMAL_W_LEXER([
%union { int ival; }], [return TokenKind.END;]) %union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]]) AT_CHECK_D_GREP([[ END = 3,]])
AT_CLEANUP AT_CLEANUP

18
tests/scanner.at
View File

@@ -121,12 +121,12 @@ class YYLexer(R) : Lexer
return semanticVal_; return semanticVal_;
} }
TokenKind yylex () YYParser.Symbol yylex ()
{ {
import std.uni : isNumber; import std.uni : isNumber;
// Handle EOF. // Handle EOF.
if (input.empty) if (input.empty)
return TokenKind.END; return YYParser.Symbol(TokenKind.END);
auto c = input.front; auto c = input.front;
input.popFront; input.popFront;
@@ -136,13 +136,13 @@ class YYLexer(R) : Lexer
{ {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
semanticVal_.val = c - '0'; semanticVal_.val = c - '0';
return TokenKind.NUM; return YYParser.Symbol(TokenKind.NUM, semanticVal_.val);
case '+': return TokenKind.PLUS; case '+': return YYParser.Symbol(TokenKind.PLUS);
case '-': return TokenKind.MINUS; case '-': return YYParser.Symbol(TokenKind.MINUS);
case '*': return TokenKind.STAR; case '*': return YYParser.Symbol(TokenKind.STAR);
case '/': return TokenKind.SLASH; case '/': return YYParser.Symbol(TokenKind.SLASH);
case '(': return TokenKind.LPAR; case '(': return YYParser.Symbol(TokenKind.LPAR);
case ')': return TokenKind.RPAR; case ')': return YYParser.Symbol(TokenKind.RPAR);
default: assert(0); default: assert(0);
} }
} }