From b254b36db8d281e11a2ff062ea5bbfd222a820eb Mon Sep 17 00:00:00 2001 From: Akim Demaille Date: Sun, 26 Apr 2020 14:48:59 +0200 Subject: [PATCH] all: don't emit an error message when the scanner returns YYERRCODE I'm quite pleased to see that the tricky case of glr.c was already prepared by the changes to support syntax_error exceptions. Better yet, it is actually syntax_error that becomes a special case of the general pattern: make yytoken be YYERRCODE. * data/skeletons/glr.c (YYFAULTYTOK): Remove the now useless (Basil) Faulty token. Instead, use the error token. * data/skeletons/lalr1.d, data/skeletons/lalr1.java: When computing the action, first check the case of the error token. * tests/calc.at: Check cases for the error token symbols before and after it. --- data/skeletons/glr.c | 28 +++++++------- data/skeletons/glr.cc | 2 + data/skeletons/lalr1.d | 70 +++++++++++++++++++++-------------- data/skeletons/lalr1.java | 77 +++++++++++++++++++++++---------------- data/skeletons/yacc.c | 5 +-- tests/calc.at | 15 ++++++++ 6 files changed, 121 insertions(+), 76 deletions(-) diff --git a/data/skeletons/glr.c b/data/skeletons/glr.c index 6b046349..1cb99662 100644 --- a/data/skeletons/glr.c +++ b/data/skeletons/glr.c @@ -334,10 +334,7 @@ static YYLTYPE yyloc_default][]b4_yyloc_default;])[ #define YYMAXLEFT ]b4_max_left_semantic_context[ /* YYMAXUTOK -- Last valid token number (for yychar). */ -#define YYMAXUTOK ]b4_user_token_number_max[]b4_glr_cc_if([[ -/* YYFAULTYTOK -- Token number (for yychar) that denotes a - syntax_error thrown from the scanner. */ -#define YYFAULTYTOK (YYMAXUTOK + 1)]])[ +#define YYMAXUTOK ]b4_user_token_number_max[ /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM as returned by yylex, with out-of-bounds checking. */ @@ -830,7 +827,7 @@ yygetToken (int *yycharp][]b4_pure_if([, yyGLRStack* yystackp])[]b4_user_formals // Map errors caught in the scanner to the undefined token, // so that error handling is started. However, record this // with this special value of yychar. - *yycharp = YYFAULTYTOK; + *yycharp = ]b4_symbol(1, id)[; } #endif // YY_EXCEPTIONS]], [[ *yycharp = ]b4_lex[;]])[ @@ -1035,8 +1032,14 @@ static inline int yygetLRActions (yy_state_t yystate, yysymbol_kind_t yytoken, const short** yyconflicts) { int yyindex = yypact[yystate] + yytoken; - if (yyisDefaultedState (yystate) - || yyindex < 0 || YYLAST < yyindex || yycheck[yyindex] != yytoken) + if (yytoken == ]b4_symbol(1, kind)[) + { + // This is the error token. + *yyconflicts = yyconfl; + return 0; + } + else if (yyisDefaultedState (yystate) + || yyindex < 0 || YYLAST < yyindex || yycheck[yyindex] != yytoken) { *yyconflicts = yyconfl; return -yydefact[yystate]; @@ -2486,12 +2489,11 @@ b4_dollar_popdef])[]dnl } else if (yyisErrorAction (yyaction)) {]b4_locations_if([[ - yystack.yyerror_range[1].yystate.yyloc = yylloc;]])[]b4_glr_cc_if([[ - /* Don't issue an error message again for exceptions - thrown from the scanner. */ - if (yychar != YYFAULTYTOK) - ]], [[ -]])[ yyreportSyntaxError (&yystack]b4_user_args[); + yystack.yyerror_range[1].yystate.yyloc = yylloc;]])[ + /* Issue an error message unless the scanner already + did. */ + if (yychar != ]b4_symbol(1, id)[) + yyreportSyntaxError (&yystack]b4_user_args[); goto yyuser_error; } else diff --git a/data/skeletons/glr.cc b/data/skeletons/glr.cc index 05c9f3eb..af15a9a6 100644 --- a/data/skeletons/glr.cc +++ b/data/skeletons/glr.cc @@ -344,6 +344,8 @@ b4_percent_define_flag_if([[global_tokens_and_yystype]], #define ]b4_symbol(-2, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(-2, [id])[ #undef ]b4_symbol(0, [id])[ #define ]b4_symbol(0, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(0, [id])[ +#undef ]b4_symbol(1, [id])[ +#define ]b4_symbol(1, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(1, [id])[ #ifndef ]b4_api_PREFIX[STYPE # define ]b4_api_PREFIX[STYPE ]b4_namespace_ref[::]b4_parser_class[::semantic_type diff --git a/data/skeletons/lalr1.d b/data/skeletons/lalr1.d index 0569f3df..430dfc78 100644 --- a/data/skeletons/lalr1.d +++ b/data/skeletons/lalr1.d @@ -509,39 +509,53 @@ m4_popdef([b4_at_dollar])])dnl yytoken = yytranslate_ (yychar);]b4_parse_trace_if([[ yy_symbol_print ("Next token is", yytoken, yylval]b4_locations_if([, yylloc])[);]])[ - /* If the proper action on seeing token YYTOKEN is to reduce or to - detect an error, take that action. */ - yyn += yytoken; - if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken) - label = YYDEFAULT; - - /* <= 0 means reduce or error. */ - else if ((yyn = yytable_[yyn]) <= 0) + if (yytoken == SymbolKind.]b4_symbol(1, kind)[) { - if (yy_table_value_is_error_ (yyn)) - label = YYERRLAB; - else - { - yyn = -yyn; - label = YYREDUCE; - } + // The scanner already issued an error message, process directly + // to error recovery. But do not keep the error token as + // lookahead, it is too special and may lead us to an endless + // loop in error recovery. */ + yychar = TokenKind.YYUNDEF; + yytoken = SymbolKind.]b4_symbol_prefix[YYUNDEF;]b4_locations_if([[ + yyerrloc = yylloc;]])[ + label = YYERRLAB1; } else { - /* Shift the lookahead token. */]b4_parse_trace_if([[ - yy_symbol_print ("Shifting", yytoken, yylval]b4_locations_if([, yylloc])[);]])[ + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken) + label = YYDEFAULT; - /* Discard the token being shifted. */ - yychar = TokenKind.YYEMPTY; + /* <= 0 means reduce or error. */ + else if ((yyn = yytable_[yyn]) <= 0) + { + if (yy_table_value_is_error_ (yyn)) + label = YYERRLAB; + else + { + yyn = -yyn; + label = YYREDUCE; + } + } + else + { + /* Shift the lookahead token. */]b4_parse_trace_if([[ + yy_symbol_print ("Shifting", yytoken, yylval]b4_locations_if([, yylloc])[);]])[ - /* Count tokens shifted since error; after three, turn off error - * status. */ - if (yyerrstatus_ > 0) - --yyerrstatus_; + /* Discard the token being shifted. */ + yychar = TokenKind.YYEMPTY; - yystate = yyn; - yystack.push (yystate, yylval]b4_locations_if([, yylloc])[); - label = YYNEWSTATE; + /* Count tokens shifted since error; after three, turn off error + * status. */ + if (yyerrstatus_ > 0) + --yyerrstatus_; + + yystate = yyn; + yystack.push (yystate, yylval]b4_locations_if([, yylloc])[); + label = YYNEWSTATE; + } } break; @@ -577,8 +591,8 @@ m4_popdef([b4_at_dollar])])dnl yytoken = SymbolKind.]b4_symbol(-2, kind)[; yyerror (]b4_locations_if([yylloc, ])[yysyntax_error (yystate, yytoken)); } - -]b4_locations_if([ yyerrloc = yylloc;])[ +]b4_locations_if([ + yyerrloc = yylloc;])[ if (yyerrstatus_ == 3) { /* If just tried and failed to reuse lookahead token after an diff --git a/data/skeletons/lalr1.java b/data/skeletons/lalr1.java index bd19787b..8cb000d4 100644 --- a/data/skeletons/lalr1.java +++ b/data/skeletons/lalr1.java @@ -164,12 +164,12 @@ import java.text.MessageFormat; } } - private ]b4_location_type[ yylloc (YYStack rhs, int n) + private ]b4_location_type[ yylloc(YYStack rhs, int n) { if (0 < n) - return new ]b4_location_type[ (rhs.locationAt (n-1).begin, rhs.locationAt (0).end); + return new ]b4_location_type[(rhs.locationAt(n-1).begin, rhs.locationAt(0).end); else - return new ]b4_location_type[ (rhs.locationAt (0).end); + return new ]b4_location_type[(rhs.locationAt(0).end); }]])[ ]b4_declare_symbol_enum[ @@ -615,41 +615,55 @@ b4_dollar_popdef[]dnl yySymbolPrint ("Next token is", yytoken, yylval]b4_locations_if([, yylloc])[);]])[ - /* If the proper action on seeing token YYTOKEN is to reduce or to - detect an error, take that action. */ - yyn += yytoken.getCode (); - if (yyn < 0 || YYLAST_ < yyn || yycheck_[yyn] != yytoken.getCode ()) - label = YYDEFAULT; - - /* <= 0 means reduce or error. */ - else if ((yyn = yytable_[yyn]) <= 0) + if (yytoken == SymbolKind.]b4_symbol_prefix[YYERRCODE) { - if (yyTableValueIsError (yyn)) - label = YYERRLAB; - else - { - yyn = -yyn; - label = YYREDUCE; - } + // The scanner already issued an error message, process directly + // to error recovery. But do not keep the error token as + // lookahead, it is too special and may lead us to an endless + // loop in error recovery. */ + yychar = Lexer.]b4_percent_define_get([api.token.prefix])[YYUNDEF; + yytoken = SymbolKind.]b4_symbol_prefix[YYUNDEF;]b4_locations_if([[ + yyerrloc = yylloc;]])[ + label = YYERRLAB1; } - else { - /* Shift the lookahead token. */]b4_parse_trace_if([[ - yySymbolPrint ("Shifting", yytoken, - yylval]b4_locations_if([, yylloc])[); + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken.getCode (); + if (yyn < 0 || YYLAST_ < yyn || yycheck_[yyn] != yytoken.getCode ()) + label = YYDEFAULT; + + /* <= 0 means reduce or error. */ + else if ((yyn = yytable_[yyn]) <= 0) + { + if (yyTableValueIsError (yyn)) + label = YYERRLAB; + else + { + yyn = -yyn; + label = YYREDUCE; + } + } + + else + { + /* Shift the lookahead token. */]b4_parse_trace_if([[ + yySymbolPrint ("Shifting", yytoken, + yylval]b4_locations_if([, yylloc])[); ]])[ - /* Discard the token being shifted. */ - yychar = YYEMPTY_; + /* Discard the token being shifted. */ + yychar = YYEMPTY_; - /* Count tokens shifted since error; after three, turn off error - status. */ - if (yyerrstatus_ > 0) - --yyerrstatus_; + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus_ > 0) + --yyerrstatus_; - yystate = yyn; - yystack.push (yystate, yylval]b4_locations_if([, yylloc])[); - label = YYNEWSTATE; + yystate = yyn; + yystack.push (yystate, yylval]b4_locations_if([, yylloc])[); + label = YYNEWSTATE; + } } break; @@ -685,7 +699,6 @@ b4_dollar_popdef[]dnl yytoken = null; yyreportSyntaxError (new Context (yystack, yytoken]b4_locations_if([[, yylloc]])[)); } - ]b4_locations_if([[ yyerrloc = yylloc;]])[ if (yyerrstatus_ == 3) diff --git a/data/skeletons/yacc.c b/data/skeletons/yacc.c index f7a12dc7..2f3c19a1 100644 --- a/data/skeletons/yacc.c +++ b/data/skeletons/yacc.c @@ -1963,9 +1963,8 @@ yyerrlab: goto yyexhaustedlab; }]])[ } - -]b4_locations_if([[ yyerror_range[1] = yylloc;]])[ - +]b4_locations_if([[ + yyerror_range[1] = yylloc;]])[ if (yyerrstatus == 3) { /* If just tried and failed to reuse lookahead token after an diff --git a/tests/calc.at b/tests/calc.at index a3711612..67fb21aa 100644 --- a/tests/calc.at +++ b/tests/calc.at @@ -1036,6 +1036,21 @@ _AT_CHECK_CALC_ERROR([$1], [0], [(#) + (#) = 2222], [[1.2: syntax error: invalid character: '#' 1.8: syntax error: invalid character: '#']]) +_AT_CHECK_CALC_ERROR([$1], [0], [(1 + #) = 1111], + [[final: 1111 0 0]], + [102], +[[1.6: syntax error: invalid character: '#']]) + +_AT_CHECK_CALC_ERROR([$1], [0], [(# + 1) = 1111], + [[final: 1111 0 0]], + [102], +[[1.2: syntax error: invalid character: '#']]) + +_AT_CHECK_CALC_ERROR([$1], [0], [(1 + # + 1) = 1111], + [[final: 1111 0 0]], + [102], +[[1.6: syntax error: invalid character: '#']]) + AT_BISON_OPTION_POPDEFS