diff --git a/data/skeletons/c++.m4 b/data/skeletons/c++.m4 index 2d05699b..1971c7df 100644 --- a/data/skeletons/c++.m4 +++ b/data/skeletons/c++.m4 @@ -532,7 +532,9 @@ m4_define([b4_yytranslate_define], ]b4_parser_class[::yytranslate_ (]b4_token_ctor_if([token_type], [int])[ t) { - // YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to +]b4_api_token_raw_if( +[[ return static_cast (t);]], +[[ // YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to // TOKEN-NUM as returned by yylex. static const token_number_type @@ -548,7 +550,7 @@ m4_define([b4_yytranslate_define], else if (static_cast (t) <= user_token_number_max_) return translate_table[t]; else - return undef_token_; + return undef_token_;]])[ } ]]) diff --git a/data/skeletons/glr.c b/data/skeletons/glr.c index 7ef1a047..9b088959 100644 --- a/data/skeletons/glr.c +++ b/data/skeletons/glr.c @@ -346,7 +346,9 @@ static YYLTYPE yyloc_default][]b4_yyloc_default;])[ /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM as returned by yylex, with out-of-bounds checking. */ -#define YYTRANSLATE(YYX) \ +]b4_api_token_raw_if(dnl +[[#define YYTRANSLATE(YYX) (YYX)]], +[[#define YYTRANSLATE(YYX) \ ((unsigned) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) /* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM @@ -354,7 +356,7 @@ static YYLTYPE yyloc_default][]b4_yyloc_default;])[ static const ]b4_int_type_for([b4_translate])[ yytranslate[] = { ]b4_translate[ -}; +};]])[ #if ]b4_api_PREFIX[DEBUG /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ diff --git a/data/skeletons/lalr1.d b/data/skeletons/lalr1.d index dc155ef4..fd19e852 100644 --- a/data/skeletons/lalr1.d +++ b/data/skeletons/lalr1.d @@ -822,18 +822,21 @@ m4_popdef([b4_at_dollar])])dnl } ]])[ - /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ - private static immutable ]b4_int_type_for([b4_translate])[[] yytranslate_table_ = - @{ - ]b4_translate[ - @}; - private static ]b4_int_type_for([b4_translate])[ yytranslate_ (int t) { - if (t >= 0 && t <= yyuser_token_number_max_) - return yytranslate_table_[t]; +]b4_api_token_raw_if( +[[ import std.conv : to; + return to!byte (t);]], +[[ /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ + immutable ]b4_int_type_for([b4_translate])[[] translate_table = + @{ + ]b4_translate[ + @}; + + if (0 <= t && t <= yyuser_token_number_max_) + return translate_table[t]; else - return yyundef_token_; + return yyundef_token_;]])[ } private static immutable int yylast_ = ]b4_last[; diff --git a/data/skeletons/lalr1.java b/data/skeletons/lalr1.java index d0778bdf..af120404 100644 --- a/data/skeletons/lalr1.java +++ b/data/skeletons/lalr1.java @@ -1017,17 +1017,22 @@ b4_dollar_popdef[]dnl b4_rhs_location(yynrhs, yyi + 1)])[); }]])[ - /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + /* YYTRANSLATE_(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM as returned by yylex, with out-of-bounds checking. */ - ]b4_integral_parser_table_define([translate_table], [b4_translate])[ - private static final ]b4_int_type_for([b4_translate])[ yytranslate_ (int t) - { +]b4_api_token_raw_if(dnl +[[ { + return t; + } +]], +[[ { if (0 <= t && t <= yyuser_token_number_max_) return yytranslate_table_[t]; else return yyundef_token_; } + ]b4_integral_parser_table_define([translate_table], [b4_translate])[ +]])[ private static final int yylast_ = ]b4_last[; private static final int yynnts_ = ]b4_nterms_number[; diff --git a/tests/local.at b/tests/local.at index e70da61e..de396496 100644 --- a/tests/local.at +++ b/tests/local.at @@ -684,9 +684,9 @@ public void yyerror (]AT_LOCATION_IF([[YYLocation l, ]])[string m) m4_define([AT_MAIN_DEFINE(d)], [[int main () { - Lexer l = new Lexer (); - Parser p = new Parser (l); - p.parse (); + auto l = new ]AT_API_prefix[Lexer (); + auto p = new ]AT_API_PREFIX[Parser (l); + return p.parse (); }]]) diff --git a/tests/scanner.at b/tests/scanner.at index a3a8b479..543d5cfc 100644 --- a/tests/scanner.at +++ b/tests/scanner.at @@ -18,6 +18,115 @@ AT_BANNER([[Interface with the scanner.]]) +# -------------- # +# AT_RAW_YYLEX. # +# -------------- # + +m4_pushdef([AT_RAW_YYLEX], [AT_LANG_DISPATCH([$0], $@)]) + +m4_define([AT_RAW_YYLEX(c)], +[#include /* abort */ +AT_YYLEX_PROTOTYPE[ +{ + static const char* input = "0-(1+2)*3/9"; + int c = *input++; + switch (c) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + ]AT_VAL[.val = c - '0'; + return ]AT_CXX_IF([yy::parser::token::])[NUM; + case '+': return ]AT_CXX_IF([yy::parser::token::])[PLUS; + case '-': return ]AT_CXX_IF([yy::parser::token::])[MINUS; + case '*': return ]AT_CXX_IF([yy::parser::token::])[STAR; + case '/': return ]AT_CXX_IF([yy::parser::token::])[SLASH; + case '(': return ]AT_CXX_IF([yy::parser::token::])[LPAR; + case ')': return ]AT_CXX_IF([yy::parser::token::])[RPAR; + case 0: return 0; + } + abort (); +} +]]) + +m4_copy([AT_RAW_YYLEX(c)], [AT_RAW_YYLEX(c++)]) + +m4_define([AT_RAW_YYLEX(d)], +[[import std.range.primitives; +import std.stdio; + +auto yyLexer(R)(R range) + if (isInputRange!R && is (ElementType!R : dchar)) +{ + return new YYLexer!R(range); +} + +auto yyLexer () +{ + return yyLexer("0-(1+2)*3/9"); +} + +class YYLexer(R) : Lexer + if (isInputRange!R && is (ElementType!R : dchar)) +{ + R input; + + this(R r) { + input = r; + } + + ]AT_YYERROR_DEFINE[ + + YYSemanticType semanticVal_; + public final @property YYSemanticType semanticVal () + { + return semanticVal_; + } + + int yylex () + { + import std.uni : isNumber; + // Handle EOF. + if (input.empty) + return YYTokenType.EOF; + + auto c = input.front; + input.popFront; + + // Numbers. + switch (c) + { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + semanticVal_.val = c - '0'; + return YYTokenType.NUM; + case '+': return YYTokenType.PLUS; + case '-': return YYTokenType.MINUS; + case '*': return YYTokenType.STAR; + case '/': return YYTokenType.SLASH; + case '(': return YYTokenType.LPAR; + case ')': return YYTokenType.RPAR; + default: assert(0); + } + } +} +]]) + +m4_pushdef([AT_MAIN_DEFINE(d)], +[[int main () +{ + auto l = yyLexer (); + auto p = new YYParser (l); + return !p.parse (); +}]]) + + ## ------------------- ## ## Raw token numbers. ## ## ------------------- ## @@ -30,12 +139,13 @@ AT_BISON_OPTION_PUSHDEFS([%debug $1]) AT_DATA_GRAMMAR([[input.y]], [[$1 %debug +]AT_D_IF([], [[ %code { #include ]AT_YYERROR_DECLARE[ ]AT_YYLEX_DECLARE[ -} +}]])[ %union { int val; @@ -70,40 +180,18 @@ exp %% ]AT_YYERROR_DEFINE[ +]AT_RAW_YYLEX[ ]AT_MAIN_DEFINE[ - -int yylex (void) -{ - static const char* input = "0-(1+2)*3/9"; - int c = *input++; - switch (c) - { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - yylval.val = c - '0'; - return NUM; - case '+': return PLUS; - case '-': return MINUS; - case '*': return STAR; - case '/': return SLASH; - case '(': return LPAR; - case ')': return RPAR; - case 0: return 0; - } -} ]]) AT_FULL_COMPILE([input]) -AT_CHECK([grep -c yytranslate input.c], [ignore], [AT_TOKEN_RAW_IF([0], [2])[ +# yacc.c, glr.c and glr.cc use 'yytranslate' (and YYTRANSLATE). +# lalr1.cc uses 'translate_table' (and yytranslate_). +# lalr1.d uses 'byte[] translate_table =' (and yytranslate_). +AT_CHECK([[grep -Ec 'yytranslate\[\]|translate_table\[\]|translate_table =' input.]AT_LANG_EXT], + [ignore], + [AT_TOKEN_RAW_IF([0], [1])[ ]]) AT_PARSER_CHECK([input], 0, @@ -114,7 +202,10 @@ AT_BISON_OPTION_POPDEFS AT_CLEANUP ]) -AT_TEST([]) -AT_TEST([%define api.token.raw]) +m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], [glr.cc], [lalr1.d]], +[AT_TEST([%skeleton "]b4_skel["]) + AT_TEST([%skeleton "]b4_skel[" %define api.token.raw])]) + +m4_popdef([AT_MAIN_DEFINE(d)]) m4_popdef([AT_TEST])