glr.cc: fix the handling of syntax_error from the scanner

Commit 90a8537e62 was right, but issued
two error messages.  Commit 80ef7e7639
tried to address that by mapping yychar and yytoken to empty, but that
completely breaks the invariants of glr.c.  In particular, yygetToken
can be called repeatedly and is expected to return the latest result,
unless yytoken is YYEMPTY.  Since the previous attempt was "recording"
that the token was coming from an exception by setting it to YYEMPTY,
instead of getting again the faulty token, we fetched another one.

Rather, revert to the first approach: map yytoken to "invalid token",
but record in yychar the fact that we come from an exception thrown in
the scanner.

* data/skeletons/glr.c (YYFAULTYTOK): New.
(yygetToken): Use it to record syntax errors from the scanner.
* tests/c++.at (Syntax error as exception): In addition to checking
syntax_error with error recovery, make sure it also behaves as
expected without.

data/skeletons/glr.c

@@ -327,7 +327,7 @@ static YYLTYPE yyloc_default][]b4_yyloc_default;])[
 #define YYNNTS  ]b4_nterms_number[
 /* YYNRULES -- Number of rules.  */
 #define YYNRULES  ]b4_rules_number[
-/* YYNRULES -- Number of states.  */
+/* YYNSTATES -- Number of states.  */
 #define YYNSTATES  ]b4_states_number[
 /* YYMAXRHS -- Maximum number of symbols on right-hand side of rule.  */
 #define YYMAXRHS ]b4_r2_max[
@@ -335,8 +335,14 @@ static YYLTYPE yyloc_default][]b4_yyloc_default;])[
    accessed by $0, $-1, etc., in any rule.  */
 #define YYMAXLEFT ]b4_max_left_semantic_context[
 
+/* YYMAXUTOK -- Last valid token number (for yychar).  */
+#define YYMAXUTOK   ]b4_user_token_number_max[]b4_glr_cc_if([[
+/* YYFAULTYTOK -- Token number (for yychar) that denotes a
+   syntax_error thrown from the scanner.  */
+#define YYFAULTYTOK (YYMAXUTOK + 1)]])[
+/* YYUNDEFTOK -- Symbol number (for yytoken) that denotes an unknown
+   token.  */
 #define YYUNDEFTOK  ]b4_undef_token_number[
-#define YYMAXUTOK   ]b4_user_token_number_max[
 
 /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
    as returned by yylex, with out-of-bounds checking.  */
@@ -782,8 +788,10 @@ yygetToken (int *yycharp][]b4_pure_if([, yyGLRStack* yystackp])[]b4_user_formals
           YYDPRINTF ((stderr, "Caught exception: %s\n", yyexc.what()));]b4_locations_if([
           yylloc = yyexc.location;])[
           yyerror (]b4_lyyerror_args[yyexc.what ());
-          // Leave yytoken/yychar to YYEMPTY.
+          // Map errors caught in the scanner to the undefined token
-          return YYEMPTY;
+          // (YYUNDEFTOK), so that error handling is started.
+          // However, record this with this special value of yychar.
+          *yycharp = YYFAULTYTOK;
         }
 #endif // YY_EXCEPTIONS]], [[
       *yycharp = ]b4_lex[;]])[
@@ -2352,11 +2360,11 @@ b4_dollar_popdef])[]dnl
                 }
               else if (yyisErrorAction (yyaction))
                 {]b4_locations_if([[
-                  yystack.yyerror_range[1].yystate.yyloc = yylloc;]])[
+                  yystack.yyerror_range[1].yystate.yyloc = yylloc;]])[]b4_glr_cc_if([[
-                  /* If yylex returned no token (YYEMPTY), it already
+                  /* Don't issue an error message again for exceptions
-                     issued an error message.  */
+                     thrown from the scanner.  */
-                  if (yytoken != YYEMPTY)
+                  if (yychar != YYFAULTYTOK)
-                    yyreportSyntaxError (&yystack]b4_user_args[);
+  ]])[                  yyreportSyntaxError (&yystack]b4_user_args[);
                   goto yyuser_error;
                 }
               else

data/skeletons/glr.cc

@@ -284,6 +284,7 @@ b4_percent_code_get([[requires]])[
     /// \returns  0 iff parsing succeeded.
     virtual int parse ();
 
+#if ]b4_api_PREFIX[DEBUG
     /// The current debugging stream.
     std::ostream& debug_stream () const;
     /// Set the current debugging stream.
@@ -295,8 +296,8 @@ b4_percent_code_get([[requires]])[
     debug_level_type debug_level () const;
     /// Set the current debugging level.
     void set_debug_level (debug_level_type l);
+#endif
 
-  public:
     /// Report a syntax error.]b4_locations_if([[
     /// \param loc    where the syntax error is found.]])[
     /// \param msg    a description of the syntax error.

tests/c++.at

@@ -957,14 +957,17 @@ AT_DATA_GRAMMAR([[input.yy]],
 %define parse.trace
 %%
 
-start:
+start: with-recovery | '!' without-recovery;
-  thing
+
-| start thing
+with-recovery:
+  %empty
+| with-recovery item
+| with-recovery error   { std::cerr << "caught error\n"; }
 ;
 
-thing:
+without-recovery:
-  error   { std::cerr << "caught error\n"; }
+  %empty
-| item
+| without-recovery item
 ;
 
 item:
@@ -988,17 +991,15 @@ yy::parser::error (const std::string &m)
 AT_DATA_SOURCE([scan.cc],
 [[#include "input.hh"
 
+// 'a': valid item, 's': syntax error, 'l': lexical error.
 int
-yylex (yy::parser::semantic_type *)
+yylex (yy::parser::semantic_type *lval)
 {
-  // 's': syntax error, 'l': lexical error.
+  switch (int res = getchar ())
-  //
-  // Leave enough valid tokens to make sure we recovered from the
-  // previous error, otherwise we might hide some error messages
-  // (discarded during error recovery).
-  static char const *input = "asaaalaa";
-  switch (int res = *input++)
   {
+    // Don't choke on echo's \n.
+    case '\n':
+      return yylex (lval);
     case 'l':
       throw yy::parser::syntax_error ("invalid character");
     default:
@@ -1010,15 +1011,27 @@ yylex (yy::parser::semantic_type *)
 AT_BISON_CHECK([[-o input.cc input.yy]])
 
 AT_FOR_EACH_CXX([
-AT_LANG_COMPILE([[input]], [[input.cc scan.cc]])
+  AT_LANG_COMPILE([[input]], [[input.cc scan.cc]])
 
-AT_PARSER_CHECK([[./input]], [[0]], [[]],
+  # Leave enough valid tokens to make sure we recovered from the
+  # previous error, otherwise we might hide some error messages
+  # (discarded during error recovery).
+  AT_PARSER_CHECK([[echo "asaaalaa" | ./input ]], [[0]], [[]],
 [[error: invalid expression
 caught error
 error: invalid character
 caught error
 ]])
-])
+
+  AT_PARSER_CHECK([[echo "!as" | ./input ]], [1], [],
+[[error: invalid expression
+]])
+
+  AT_PARSER_CHECK([[echo "!al" | ./input ]], [1], [],
+[[error: invalid character
+]])
+
+]) # AT_FOR_EACH_CXX
 
 AT_BISON_OPTION_POPDEFS
 AT_CLEANUP
@@ -1029,6 +1042,8 @@ AT_TEST([%skeleton "glr.cc"])
 
 m4_popdef([AT_TEST])
 
+
+
 ## ------------------ ##
 ## Exception safety.  ##
 ## ------------------ ##