c: don't emit an error message when the scanner returns YYERRCODE

* data/skeletons/yacc.c (yyparse): When the scanner returns YYERRCODE,
go directly to error recovery (yyerrlab1).
However, don't keep the error token as lookahead, that token is too
special.
* data/skeletons/lalr1.cc: Likewise.

* examples/c/bistromathic/parse.y (yylex): Use that feature to report
nicely invalid characters.
* examples/c/bistromathic/bistromathic.test: Check that.
* examples/test: Neutralize gratuitous differences such as rule
position.

* tests/calc.at: Check that case in C only.
The other case seem to be working, but that's an illusion that the
next commit will address (in fact, they can enter endless loops, and
report the error several times anyway).

data/skeletons/lalr1.cc

@@ -981,6 +981,16 @@ b4_dollar_popdef])[]dnl
       }
     YY_SYMBOL_PRINT ("Next token is", yyla);
 
+    if (yyla.kind () == ]symbol_kind::b4_symbol(1, kind)[)
+    {
+      // The scanner already issued an error message, process directly
+      // to error recovery.  But do not keep the error token as
+      // lookahead, it is too special and may lead us to an endless
+      // loop in error recovery. */
+      yyla.type = ]symbol_kind::b4_symbol(2, kind)[;
+      goto yyerrlab1;
+    }
+
     /* If the proper action on seeing token YYLA.TYPE is to reduce or
        to detect an error, take that action.  */
     yyn += yyla.kind ();

data/skeletons/yacc.c

@@ -1770,6 +1770,17 @@ yyread_pushed_token:]])[
       yytoken = ]b4_symbol(0, [kind])[;
       YYDPRINTF ((stderr, "Now at end of input.\n"));
     }
+  else if (yychar == ]b4_symbol(1, [id])[)
+    {
+      /* The scanner already issued an error message, process directly
+         to error recovery.  But do not keep the error token as
+         lookahead, it is too special and may lead us to an endless
+         loop in error recovery. */
+      yychar = ]b4_symbol(2, [id])[;
+      yytoken = ]b4_symbol(1, [kind])[;]b4_locations_if([[
+      yyerror_range[1] = yylloc;]])[
+      goto yyerrlab1;
+    }
   else
     {
       yytoken = YYTRANSLATE (yychar);

examples/c/bistromathic/bistromathic.test

@@ -82,7 +82,11 @@ run 0 '> 1 / 0
 > ''
 err: 1.1-5: error: division by zero'
 
-# Error recovery.
+
+## ---------------- ##
+## Error recovery.  ##
+## ---------------- ##
+
 cat >input <<EOF
 ((1 ++ 2) ** 3)
 (1 ++ 2) + (3 ** 4)
@@ -96,14 +100,124 @@ err: 1.6: syntax error: expected - or ( or number or function or variable before
 err: 2.5: syntax error: expected - or ( or number or function or variable before +
 err: 2.16: syntax error: expected - or ( or number or function or variable before *'
 
-# This is really stupid: we just discarded % and did not enter error recovery.
+# The rule "( error )" should work even if there are no tokens between "(" and ")".
+cat >input <<EOF
+()
+EOF
+run 0 '> ()
+666
+> ''
+err: 1.2: syntax error: expected - or ( or number or function or variable before )'
+
+
 cat >input <<EOF
 100% + 10
 EOF
 run 0 '> 100% + 10
-110
 > ''
-err: 1.4: error: invalid character'
+err: 1.4: syntax error: invalid character: %'
+
+# Traces.  This allows to check the location of the error.  If we
+# forget to map YYERRCODE to YYUNDEF, error recovery enters an endless
+# loop with this input.
+cat >input <<EOF
+(+_)
+EOF
+run 0 '> (+_)
+666
+> ''
+err: Starting parse
+err: Entering state 0
+err: Stack now 0
+err: Reading a token
+err: Next token is token ( (1.1: )
+err: Shifting token ( (1.1: )
+err: Entering state 2
+err: Stack now 0 2
+err: Return for a new token:
+err: Reading a token
+err: Next token is token + (1.2: )
+err: LAC: initial context established for +
+err: LAC: checking lookahead +: Err
+err: LAC: checking lookahead end of file: Err
+err: LAC: checking lookahead +: Err
+err: LAC: checking lookahead -: S1
+err: LAC: checking lookahead *: Err
+err: LAC: checking lookahead /: Err
+err: LAC: checking lookahead ^: Err
+err: LAC: checking lookahead (: S2
+err: LAC: checking lookahead ): Err
+err: LAC: checking lookahead =: Err
+err: LAC: checking lookahead exit: Err
+err: LAC: checking lookahead number: S4
+err: LAC: checking lookahead function: S5
+err: LAC: checking lookahead variable: S6
+err: LAC: checking lookahead NEG: Err
+err: 1.2: syntax error: expected - or ( or number or function or variable before +
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Next token is token + (1.2: )
+err: LAC: initial context established for +
+err: LAC: checking lookahead +: Err
+err: Error: discarding token + (1.2: )
+err: Error: popping token error (1.2: )
+err: Stack now 0 2
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Return for a new token:
+err: 1.3: syntax error: invalid character: _
+err: Reading a token
+err: Error: popping token error (1.2: )
+err: Stack now 0 2
+err: Shifting token error (1.2-3: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Next token is token invalid token (1.3: )
+err: LAC: initial context established for invalid token
+err: LAC: checking lookahead invalid token: Always Err
+err: Error: discarding token invalid token (1.3: )
+err: Error: popping token error (1.2-3: )
+err: Stack now 0 2
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2-3: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Return for a new token:
+err: Reading a token
+err: Next token is token ) (1.4: )
+err: Shifting token ) (1.4: )
+err: Entering state 20
+err: Stack now 0 2 10 20
+err: Reducing stack by rule 15 (line 151):
+err:    $1 = token ( (1.1: )
+err:    $2 = token error (1.2-3: )
+err:    $3 = token ) (1.4: )
+err: -> $$ = nterm exp (1.1-4: 666)
+err: Entering state 7
+err: Stack now 0 7
+err: Return for a new token:
+err: Reading a token
+err: Now at end of input.
+err: LAC: initial context established for end of file
+err: LAC: checking lookahead end of file: R2 G8 S19
+err: Reducing stack by rule 2 (line 126):
+err:    $1 = nterm exp (1.1-4: 666)
+err: -> $$ = nterm input (1.1-4: )
+err: Entering state 8
+err: Stack now 0 8
+err: Now at end of input.
+err: Shifting token end of file (1.5: )
+err: LAC: initial context discarded due to shift
+err: Entering state 19
+err: Stack now 0 8 19
+err: Stack now 0 8 19
+err: Cleanup: popping token end of file (1.5: )
+err: Cleanup: popping nterm input (1.1-4: )' -p
+
 
 
 ## ------------ ##

examples/c/bistromathic/parse.y

@@ -3,6 +3,7 @@
 %code top {
   #include <ctype.h>  // isdigit
   #include <math.h>   // cos, sin, etc.
+  #include <stdarg.h> // va_start
   #include <stdio.h>  // printf
   #include <stdlib.h> // calloc
   #include <string.h> // strcmp
@@ -45,8 +46,14 @@
 }
 
 %code provides {
+# ifndef __attribute__
+#  ifndef __GNUC__
+#   define __attribute__(Spec) /* empty */
+#  endif
+# endif
   int yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc);
-  void yyerror (YYLTYPE *yylloc, char const *msg);
+  void yyerror (YYLTYPE *loc, char const *format, ...)
+    __attribute__ ((__format__ (__printf__, 2, 3)));
 }
 
 %code {
@@ -249,6 +256,8 @@ yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc)
     case '(': return TOK_LPAREN;
     case ')': return TOK_RPAREN;
 
+    case '!': return TOK_YYUNDEF;
+
     case '\0': return TOK_YYEOF;
 
       // Numbers.
@@ -290,8 +299,8 @@ yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc)
 
       // Stray characters.
     default:
-      yyerror (yylloc, "error: invalid character");
+      yyerror (yylloc, "syntax error: invalid character: %c", c);
-      return yylex (line, yylval, yylloc);
+      return TOK_YYERRCODE;
     }
 }
 
@@ -371,10 +380,15 @@ yyreport_syntax_error (const yypcontext_t *ctx)
 
 
 // Called by yyparse on error.
-void yyerror (YYLTYPE *loc, char const *msg)
+void yyerror (YYLTYPE *loc, char const *format, ...)
 {
   YY_LOCATION_PRINT (stderr, *loc);
-  fprintf (stderr, ": %s\n", msg);
+  fputs (": ", stderr);
+  va_list args;
+  va_start (args, format);
+  vfprintf (stderr, format, args);
+  va_end (args);
+  putc ('\n', stderr);
 }
 
 
@@ -535,7 +549,7 @@ int main (int argc, char const* argv[])
 #endif
 
   // Enable parse traces on option -p.
-  if (argc == 2 && strcmp (argv[1], "-p") == 0)
+  if (1 < argc && strcmp (argv[1], "-p") == 0)
     yydebug = 1;
   init_table ();
   init_readline ();

examples/test

@@ -89,11 +89,13 @@ run ()
   shift
 
   # Expected output.
-  if $strip_prompt; then
+  $echo "$1" |
-      $echo "$1" | sed -e '/^> /d' >exp
+    sed -e 's/Reducing stack by rule .* (line .*):/Reducing stack by rule XX (line XXX):/g' |
-  else
+    if $strip_prompt; then
-      $echo "$1" >exp
+      sed -e '/^> /d'
-  fi
+    else
+      cat
+    fi >exp
   shift
 
   # Effective exit status.
@@ -102,7 +104,14 @@ run ()
   prog "$@" - <input >out_eff 2>err_eff || sta_eff=$?
 
   # Combine effective output and error streams.
-  { cat out_eff && $noerr || sed -e 's/^/err: /g' err_eff; } >eff
+  {
+    cat out_eff
+    if ! $noerr; then
+       sed -e 's/^/err: /g' \
+           -e 's/Reducing stack by rule .* (line .*):/Reducing stack by rule XX (line XXX):/g' \
+           err_eff
+    fi
+  } >eff
 
   if test $sta_eff -eq $sta_exp; then
     if cmp eff exp 2>/dev/null; then

tests/calc.at

@@ -201,8 +201,8 @@ read_integer (]AT_YYLEX_FORMALS[)
   /* Skip white spaces.  */
   do
     {
-]AT_LOCATION_IF(
+]AT_LOCATION_IF([
-[     AT_LOC_FIRST_COLUMN = AT_LOC_LAST_COLUMN;
+      AT_LOC_FIRST_COLUMN = AT_LOC_LAST_COLUMN;
       AT_LOC_FIRST_LINE   = AT_LOC_LAST_LINE;
 ])[
     }
@@ -220,6 +220,15 @@ read_integer (]AT_YYLEX_FORMALS[)
   if (c == EOF)
     return ]AT_TOKEN_PREFIX[CALC_EOF;
 
+  /* An explicit error raised by the scanner. */
+  if (c == '#')
+    {]AT_LOCATION_IF([
+      fprintf (stderr, "%d.%d: ",
+               AT_LOC_FIRST_LINE, AT_LOC_FIRST_COLUMN);])[
+      fputs ("syntax error: invalid character: '#'\n", stderr);
+      return ]AT_TOKEN_PREFIX[]AT_API_PREFIX[ERRCODE;
+    }
+
   /* Return single chars. */
   return c;
 }
@@ -304,7 +313,7 @@ class CalcLexer(R) : Lexer
         location.end.column += 1;]])[
       }
 
-    // Handle EOF.
+    // EOF.
     if (input.empty)
       return TokenKind.CALC_EOF;
 
@@ -325,6 +334,14 @@ class CalcLexer(R) : Lexer
     else
       location.end.column += 1;]])[
     input.popFront;
+
+    // An explicit error raised by the scanner. */
+    if (c == '#')
+      {
+        stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error: invalid character: '#'");
+        return TokenKind.YYERRCODE;
+      }
+
     return c;
   }
 }
@@ -370,9 +387,9 @@ m4_define([AT_CALC_YYLEX(java)],
 
   public int yylex () throws IOException {;]AT_LOCATION_IF([[
     start.set (reader.getPosition ());]])[
-    int ttype = st.nextToken ();]AT_LOCATION_IF([[
+    int tkind = st.nextToken ();]AT_LOCATION_IF([[
     end.set (reader.getPosition ());]])[
-    switch (ttype)
+    switch (tkind)
       {
       case StreamTokenizer.TT_EOF:
         return EOF;
@@ -386,8 +403,11 @@ m4_define([AT_CALC_YYLEX(java)],
         return NUM;
       case ' ': case '\t':
         return yylex ();
+      case '#':
+        System.err.println(]AT_LOCATION_IF([[start + ": " + ]])["syntax error: invalid character: '#'");
+        return YYERRCODE;
       default:
-        return ttype;
+        return tkind;
       }
   }
 ]AT_LEXPARAM_IF([], [[}]])[
@@ -1004,6 +1024,20 @@ _AT_CHECK_CALC_ERROR([$1], [0], [(* *) + (*) + (*)],
 ]AT_JAVA_IF([1.10-1.11], [1.10])[: syntax error on token ['*'] (expected: [number] ['-'] ['('] ['!'])
 ]AT_JAVA_IF([1.16-1.17], [1.16])[: syntax error on token ['*'] (expected: [number] ['-'] ['('] ['!'])]])
 
+
+# YYERRCODE.
+# ---------
+# Check that returning YYERRCODE from the scanner properly enters
+# error-recovery without issuing a second error message.
+
+_AT_CHECK_CALC_ERROR([$1], [0], [(#) + (#) = 2222],
+                     [[final: 2222 0 0]],
+                     [102],
+[[1.2: syntax error: invalid character: '#'
+1.8: syntax error: invalid character: '#']])
+
+
+
 AT_BISON_OPTION_POPDEFS
 
 AT_CLEANUP