version 3.5.4

* NEWS: Record release date.

.prev-version

@@ -1 +1 @@
-3.5
+3.5.3

NEWS

@@ -1,5 +1,44 @@
 GNU Bison NEWS
 
+* Noteworthy changes in release 3.5.4 (2020-04-05) [stable]
+
+** WARNING: Future backward-incompatibilities!
+
+  TL;DR: replace "#define YYERROR_VERBOSE 1" by "%define parse.error verbose".
+
+  Bison 3.6 will no longer support the YYERROR_VERBOSE macro; the parsers
+  that still depend on it will produce Yacc-like error messages (just
+  "syntax error").  It was superseded by the "%error-verbose" directive in
+  Bison 1.875 (2003-01-01).  Bison 2.6 (2012-07-19) clearly announced that
+  support for YYERROR_VERBOSE would be removed.  Note that since Bison 3.0
+  (2013-07-25), "%error-verbose" is deprecated in favor of "%define
+  parse.error verbose".
+
+** Bug fixes
+
+  Fix portability issues of the package itself on old compilers.
+
+  Fix api.token.raw support in Java.
+
+* Noteworthy changes in release 3.5.3 (2020-03-08) [stable]
+
+** Bug fixes
+
+  Error messages could quote lines containing zero-width characters (such as
+  \005) with incorrect styling.  Fixes for similar issues with unexpectedly
+  short lines (e.g., the file was changed between parsing and diagnosing).
+
+  Several unlikely crashes found by fuzzing have been fixed.
+
+* Noteworthy changes in release 3.5.2 (2020-02-13) [stable]
+
+** Bug fixes
+
+  Portability issues and minor cosmetic issues.
+
+  The lalr1.cc skeleton properly rejects unsupported values for parse.lac
+  (as yacc.c does).
+
 * Noteworthy changes in release 3.5.1 (2020-01-19) [stable]
 
 ** Bug fixes

README

@@ -1,4 +1,19 @@
-This package contains the GNU Bison parser generator.
+GNU Bison is a general-purpose parser generator that converts an annotated
+context-free grammar into a deterministic LR or generalized LR (GLR) parser
+employing LALR(1) parser tables.  Bison can also generate IELR(1) or
+canonical LR(1) parser tables.  Once you are proficient with Bison, you can
+use it to develop a wide range of language parsers, from those used in
+simple desk calculators to complex programming languages.
+
+Bison is upward compatible with Yacc: all properly-written Yacc grammars
+work with Bison with no change.  Anyone familiar with Yacc should be able to
+use Bison with little trouble.  You need to be fluent in C, C++ or Java
+programming in order to use Bison.
+
+Bison and the parsers it generates are portable, they do not require any
+specific compilers.
+
+GNU Bison's home page is https://gnu.org/software/bison/.
 
 # Installation
 ## Build from git
@@ -17,13 +32,18 @@ See the file INSTALL for generic compilation and installation instructions.
 Bison requires GNU m4 1.4.6 or later.  See
 https://ftp.gnu.org/gnu/m4/m4-1.4.6.tar.gz.
 
+## Running a non installed bison
+Once you ran `make`, you might want to toy with this fresh bison before
+installing it.  In that case, do not use `src/bison`: it would use the
+*installed* files (skeletons, etc.), not the local ones.  Use `tests/bison`.
+
 ## Colored diagnostics
 As an experimental feature, diagnostics are now colored, controlled by the
 `--color` and `--style` options.
 
 To use them, install the libtextstyle library before configuring Bison.  It
 is available from https://alpha.gnu.org/gnu/gettext/, for instance
-https://alpha.gnu.org/gnu/gettext/libtextstyle-0.8.tar.gz.
+https://alpha.gnu.org/pub/gnu/gettext/libtextstyle-0.20.5.tar.gz.
 
 The option --color supports the following arguments:
 - always, yes: Enable colors.
@@ -85,18 +105,12 @@ Software Foundation, Inc.
 
 This file is part of GNU bison, the GNU Compiler Compiler.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
 Local Variables:
 mode: markdown
@@ -104,7 +118,8 @@ fill-column: 76
 ispell-dictionary: "american"
 End:
 
-LocalWords:  parsers ngettext Texinfo pdf html YYYY ZZZZ ispell american
-LocalWords:  MERCHANTABILITY
+LocalWords:  parsers ngettext Texinfo pdf html YYYY ZZZZ ispell american md
+LocalWords:  MERCHANTABILITY GLR LALR IELR submodule init README src bw
+LocalWords:  Relocatability symlinks symlink
 
 -->

README-hacking.md

@@ -34,13 +34,28 @@ Only user visible strings are to be translated: error messages, bits of the
 assert/abort), and all the --trace output which is meant for the maintainers
 only.
 
-## Horizontal tabs
+## Coding style
+Follow the GNU Coding Standards.
+
+Don't reinvent the wheel: we use gnulib, which features many components.
+Actually, Bison has legacy code that we should replace with gnulib modules
+(e.g., many adhoc implementations of lists).
+
 Do not add horizontal tab characters to any file in Bison's repository
 except where required.  For example, do not use tabs to format C code.
 However, make files, ChangeLog, and some regular expressions require tabs.
 Also, test cases might need to contain tabs to check that Bison properly
 processes tabs in its input.
 
+## Commit messages
+Please, imitate the style we use.  Use `git log` to get sources of
+inspiration.
+
+## Debugging
+Bison supports tracing of its various steps, via the `--trace` option.
+Since it is not meant for the end user, it is not displayed by `bison
+--help`, nor is it documented in the manual.  Instead, run `bison
+--trace=help`.
 
 # Working from the repository
 

THANKS

@@ -4,8 +4,9 @@ it is today without the invaluable help of these people:
 Aaro Koskinen             [email protected]
 Аскар Сафин               [email protected]
 Adam Sampson              [email protected]
+Ahcheong Lee              [email protected]
 Airy Andre                [email protected]
-Akim Demaille             akim@lrde.epita.fr
+Akim Demaille             akim@gnu.org
 Albert Chin-A-Young       [email protected]
 Alexander Belopolsky      [email protected]
 Alexandre Duret-Lutz      [email protected]
@@ -60,6 +61,7 @@ Enrico Scholz             [email protected]
 Eric Blake                [email protected]
 Eric S. Raymond           [email protected]
 Étienne Renault           [email protected]
+Evan Lavelle              [email protected]
 Evan Nemerson             [email protected]
 Evgeny Stambulchik        [email protected]
 Fabrice Bauzac            [email protected]
@@ -130,6 +132,7 @@ Nick Bowler               [email protected]
 Nicolas Bedon             [email protected]
 Nicolas Burrus            [email protected]
 Nicolas Tisserand         [email protected]
+Nikki Valen               [email protected]
 Noah Friedman             [email protected]
 Odd Arild Olsen           [email protected]
 Oleg Smolsky              [email protected]

TODO

@@ -315,13 +315,8 @@ It would be a very nice source of inspiration for the other languages.
 Valentin Tolmer is working on this.
 
 ** YYERRCODE
-Defined to 256, but not used, not documented.  Probably the token
-number for the error token, which POSIX wants to be 256, but which
-Bison might renumber if the user used number 256.  Keep fix and doc?
-Throw away?
-
-Also, why don't we output the token name of the error token in the
-output?  It is explicitly skipped:
+Why don't we output the token name of the error token in the output?  It is
+explicitly skipped:
 
       /* Skip error token and tokens without identifier.  */
       if (sym != errtoken && id)

build-aux/cross-options.pl

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
-# Written by Akimn Demaille.
+# Written by Akim Demaille.
 
 use warnings;
 use 5.005;

data/skeletons/bison.m4

@@ -532,9 +532,9 @@ m4_define([b4_any_token_visible_if],
 # ----------------------------
 m4_define([b4_token_format],
 [b4_token_visible_if([$2],
-[m4_quote(m4_format([$1],
-                     [b4_symbol([$2], [id])],
-                     [b4_symbol([$2], b4_api_token_raw_if([[number]], [[user_number]]))]))])])
+[m4_format([[$1]],
+           m4_quote(b4_symbol([$2], [id])),
+           m4_quote(b4_symbol([$2], b4_api_token_raw_if([[number]], [[user_number]]))))])])
 
 
 ## ------- ##

data/skeletons/java.m4

@@ -109,7 +109,7 @@ m4_define([b4_null], [null])
 m4_define([b4_typed_parser_table_define],
 [m4_ifval([$4], [b4_comment([$4])
   ])dnl
-[private static final ]$1[ yy$2_[] = yy$2_init();
+[private static final ]$1[[] yy$2_ = yy$2_init();
   private static final ]$1[[] yy$2_init()
   {
     return new ]$1[[]
@@ -147,18 +147,20 @@ b4_symbol_foreach([b4_token_enum])])])
 # b4-case(ID, CODE)
 # -----------------
 # We need to fool Java's stupid unreachable code detection.
-m4_define([b4_case], [  case $1:
+m4_define([b4_case],
+[  case $1:
   if (yyn == $1)
     $2;
   break;
-    ])
+])
 
 # b4_predicate_case(LABEL, CONDITIONS)
 # ------------------------------------
-m4_define([b4_predicate_case], [  case $1:
+m4_define([b4_predicate_case],
+[  case $1:
      if (! ($2)) YYERROR;
     break;
-    ])
+])
 
 
 ## -------- ##
@@ -361,4 +363,4 @@ m4_define([b4_var_decl],
 # -----------------------
 # Expand to either an empty string or "throws THROWS".
 m4_define([b4_maybe_throws],
-          [m4_ifval($1, [throws $1])])
+          [m4_ifval($1, [ throws $1])])

data/skeletons/lalr1.cc

@@ -23,6 +23,7 @@ m4_define([b4_value_type_setup_variant])
 # Check the value of %define parse.lac, where LAC stands for lookahead
 # correction.
 b4_percent_define_default([[parse.lac]], [[none]])
+b4_percent_define_check_values([[[[parse.lac]], [[full]], [[none]]]])
 b4_define_flag_if([lac])
 m4_define([b4_lac_flag],
           [m4_if(b4_percent_define_get([[parse.lac]]),
@@ -1339,7 +1340,7 @@ b4_error_verbose_if([state_type yystate, const symbol_type& yyla],
         yyarg[yycount++] = yytname_[yytoken];]b4_lac_if([[
 
 #if ]b4_api_PREFIX[DEBUG
-        // Execute LAC once. We don't care if it is succesful, we
+        // Execute LAC once. We don't care if it is successful, we
         // only do it for the sake of debugging output.
         if (!yy_lac_established_)
           yy_lac_check_ (yytoken);

data/skeletons/lalr1.java

@@ -65,8 +65,8 @@ m4_define([b4_define_state],[[
     int label = YYNEWSTATE;
 
     /* Error handling.  */
-    int yynerrs_ = 0;
-    ]b4_locations_if([[/* The location where the error started.  */
+    int yynerrs_ = 0;]b4_locations_if([[
+    /* The location where the error started.  */
     ]b4_location_type[ yyerrloc = null;
 
     /* Location. */
@@ -182,7 +182,7 @@ b4_locations_if([[
 
 ]b4_token_enums[
 
-    ]b4_locations_if([[/**
+]b4_locations_if([[/**
      * Method to retrieve the beginning position of the last scanned token.
      * @@return the position at which the last scanned token starts.
      */
@@ -206,13 +206,13 @@ b4_locations_if([[
      * ]b4_locations_if([and beginning/ending positions ])[of the token.
      * @@return the token identifier corresponding to the next token.
      */
-    int yylex () ]b4_maybe_throws([b4_lex_throws])[;
+    int yylex ()]b4_maybe_throws([b4_lex_throws])[;
 
     /**
      * Entry point for error reporting.  Emits an error
      * ]b4_locations_if([referring to the given location ])[in a user-defined way.
      *
-     * ]b4_locations_if([[@@param loc The location of the element to which the
+     *]b4_locations_if([[ @@param loc The location of the element to which the
      *                error message is related]])[
      * @@param msg The string for the error message.
      */
@@ -224,22 +224,23 @@ b4_locations_if([[
 ]b4_percent_code_get([[lexer]])[
   }
 
-  ]])[/**
+]])[
+  /**
    * The object doing lexical analysis for us.
    */
   private Lexer yylexer;
 
-  ]b4_parse_param_vars[
+]b4_parse_param_vars[
 
 ]b4_lexer_if([[
   /**
    * Instantiates the Bison-generated parser.
    */
-  public ]b4_parser_class (b4_parse_param_decl([b4_lex_param_decl])[) ]b4_maybe_throws([b4_init_throws])[
+  public ]b4_parser_class (b4_parse_param_decl([b4_lex_param_decl])[)]b4_maybe_throws([b4_init_throws])[
   {
-    ]b4_percent_code_get([[init]])[
+]b4_percent_code_get([[init]])[
     this.yylexer = new YYLexer(]b4_lex_param_call[);
-    ]b4_parse_param_cons[
+]b4_parse_param_cons[
   }
 ]])[
 
@@ -247,11 +248,11 @@ b4_locations_if([[
    * Instantiates the Bison-generated parser.
    * @@param yylexer The scanner that will supply tokens to the parser.
    */
-  ]b4_lexer_if([[protected]], [[public]]) b4_parser_class[ (]b4_parse_param_decl([[Lexer yylexer]])[) ]b4_maybe_throws([b4_init_throws])[
+  ]b4_lexer_if([[protected]], [[public]]) b4_parser_class[ (]b4_parse_param_decl([[Lexer yylexer]])[)]b4_maybe_throws([b4_init_throws])[
   {
-    ]b4_percent_code_get([[init]])[
+]b4_percent_code_get([[init]])[
     this.yylexer = yylexer;
-    ]b4_parse_param_cons[
+]b4_parse_param_cons[
   }
 
 ]b4_parse_trace_if([[
@@ -320,8 +321,8 @@ b4_locations_if([[
   }]])[
 
   private final class YYStack {
-    private int[] stateStack = new int[16];
-    ]b4_locations_if([[private ]b4_location_type[[] locStack = new ]b4_location_type[[16];]])[
+    private int[] stateStack = new int[16];]b4_locations_if([[
+    private ]b4_location_type[[] locStack = new ]b4_location_type[[16];]])[
     private ]b4_yystype[[] valueStack = new ]b4_yystype[[16];
 
     public int size = 16;
@@ -334,8 +335,7 @@ b4_locations_if([[
         {
           int[] newStateStack = new int[size * 2];
           System.arraycopy (stateStack, 0, newStateStack, 0, height);
-          stateStack = newStateStack;
-          ]b4_locations_if([[
+          stateStack = newStateStack;]b4_locations_if([[
           ]b4_location_type[[] newLocStack = new ]b4_location_type[[size * 2];
           System.arraycopy (locStack, 0, newLocStack, 0, height);
           locStack = newLocStack;]])
@@ -347,8 +347,8 @@ b4_locations_if([[
           size *= 2;
         }
 
-      stateStack[height] = state;
-      ]b4_locations_if([[locStack[height] = loc;]])[
+      stateStack[height] = state;]b4_locations_if([[
+      locStack[height] = loc;]])[
       valueStack[height] = value;
     }
 
@@ -359,8 +359,8 @@ b4_locations_if([[
     public final void pop (int num) {
       // Avoid memory leaks... garbage collection is a white lie!
       if (0 < num) {
-        java.util.Arrays.fill (valueStack, height - num + 1, height + 1, null);
-        ]b4_locations_if([[java.util.Arrays.fill (locStack, height - num + 1, height + 1, null);]])[
+        java.util.Arrays.fill (valueStack, height - num + 1, height + 1, null);]b4_locations_if([[
+        java.util.Arrays.fill (locStack, height - num + 1, height + 1, null);]])[
       }
       height -= num;
     }
@@ -368,8 +368,9 @@ b4_locations_if([[
     public final int stateAt (int i) {
       return stateStack[height - i];
     }
+]b4_locations_if([[
 
-    ]b4_locations_if([[public final ]b4_location_type[ locationAt (int i) {
+    public final ]b4_location_type[ locationAt (int i) {
       return locStack[height - i];
     }
 
@@ -452,7 +453,7 @@ b4_locations_if([[
       return yydefgoto_[yysym - yyntokens_];
   }
 
-  private int yyaction (int yyn, YYStack yystack, int yylen) ]b4_maybe_throws([b4_throws])[
+  private int yyaction (int yyn, YYStack yystack, int yylen)]b4_maybe_throws([b4_throws])[
   {
     /* If YYLEN is nonzero, implement the default value of the action:
        '$$ = $1'.  Otherwise, use the top of the stack.
@@ -460,8 +461,8 @@ b4_locations_if([[
        Otherwise, the following line sets YYVAL to garbage.
        This behavior is undocumented and Bison
        users should not rely upon it.  */
-    ]b4_yystype[ yyval = (0 < yylen) ? yystack.valueAt (yylen - 1) : yystack.valueAt (0);
-    ]b4_locations_if([b4_location_type[ yyloc = yylloc (yystack, yylen);]])[]b4_parse_trace_if([[
+    ]b4_yystype[ yyval = (0 < yylen) ? yystack.valueAt (yylen - 1) : yystack.valueAt (0);]b4_locations_if([[
+    ]b4_location_type[ yyloc = yylloc (yystack, yylen);]])[]b4_parse_trace_if([[
 
     yyReducePrint (yyn, yystack);]])[
 
@@ -541,29 +542,28 @@ b4_locations_if([[
    * @@return <tt>true</tt> if the parsing succeeds.  Note that this does not
    *          imply that there were no syntax errors.
    */
-  public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[
+  public boolean parse ()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[
 ]b4_push_if([
   /**
    * Push Parse input from external lexer
    *
    * @@param yylextoken current token
-   * @@param yylexval current lval
-]b4_locations_if([   * @@param yylexloc current position])[
+   * @@param yylexval current lval]b4_locations_if([[
+   * @@param yylexloc current position]])[
    *
    * @@return <tt>YYACCEPT, YYABORT, YYPUSH_MORE</tt>
    */
-  public int push_parse (int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc]))
-      b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[
-  {
-    ]b4_locations_if([/* @@$.  */
-    b4_location_type yyloc;])[
+  public int push_parse (int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc]))b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[
+  {]b4_locations_if([[
+    /* @@$.  */
+    ]b4_location_type[ yyloc;]])[
 ]b4_push_if([],[[
 ]b4_define_state[]b4_parse_trace_if([[
     yycdebug ("Starting parse\n");]])[
     yyerrstatus_ = 0;
 
     /* Initialize the stack.  */
-    yystack.push (yystate, yylval ]b4_locations_if([, yylloc])[);
+    yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
 ]m4_ifdef([b4_initial_action], [
 b4_dollar_pushdef([yylval], [], [], [yylloc])dnl
     b4_user_initial_action
@@ -706,7 +706,8 @@ b4_dollar_popdef[]dnl
             yyerror (]b4_locations_if([yylloc, ])[yysyntax_error (yystate, yytoken));
           }
 
-        ]b4_locations_if([yyerrloc = yylloc;])[
+]b4_locations_if([[
+        yyerrloc = yylloc;]])[
         if (yyerrstatus_ == 3)
           {
             /* If just tried and failed to reuse lookahead token after an
@@ -730,8 +731,8 @@ b4_dollar_popdef[]dnl
       /*-------------------------------------------------.
       | errorlab -- error raised explicitly by YYERROR.  |
       `-------------------------------------------------*/
-      case YYERROR:
-        ]b4_locations_if([yyerrloc = yystack.locationAt (yylen - 1);])[
+      case YYERROR:]b4_locations_if([[
+        yyerrloc = yystack.locationAt (yylen - 1);]])[
         /* Do not reclaim the symbols of the rule which action triggered
            this YYERROR.  */
         yystack.pop (yylen);
@@ -765,7 +766,8 @@ b4_dollar_popdef[]dnl
             if (yystack.height == 0)
               ]b4_push_if([{label = YYABORT; break;}],[return false;])[
 
-            ]b4_locations_if([yyerrloc = yystack.locationAt (0);])[
+]b4_locations_if([[
+            yyerrloc = yystack.locationAt (0);]])[
             yystack.pop ();
             yystate = yystack.stateAt (0);]b4_parse_trace_if([[
             if (0 < yydebug)
@@ -776,12 +778,12 @@ b4_dollar_popdef[]dnl
             /* Leave the switch.  */
             break;
 
-]b4_locations_if([
+]b4_locations_if([[
         /* Muck with the stack to setup for yylloc.  */
         yystack.push (0, null, yylloc);
         yystack.push (0, null, yyerrloc);
         yyloc = yylloc (yystack, 2);
-        yystack.pop (2);])[
+        yystack.pop (2);]])[
 
         /* Shift the error token.  */]b4_parse_trace_if([[
         yySymbolPrint ("Shifting", yystos_[yyn],
@@ -823,10 +825,10 @@ b4_dollar_popdef[]dnl
     this.label = YYNEWSTATE;
 
     /* Error handling.  */
-    this.yynerrs_ = 0;
-    ]b4_locations_if([/* The location where the error started.  */
+    this.yynerrs_ = 0;]b4_locations_if([[
+    /* The location where the error started.  */
     this.yyerrloc = null;
-    this.yylloc = new b4_location_type (null, null);])[
+    this.yylloc = new ]b4_location_type[ (null, null);]])[
 
     /* Semantic value of the lookahead.  */
     this.yylval = null;
@@ -846,8 +848,7 @@ b4_dollar_popdef[]dnl
    *
    * @@return <tt>YYACCEPT, YYABORT, YYPUSH_MORE</tt>
    */
-  public int push_parse (int yylextoken, b4_yystype yylexval, b4_position_type yylexpos)
-      b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])
+  public int push_parse (int yylextoken, b4_yystype yylexval, b4_position_type yylexpos)b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])
   {
     return push_parse (yylextoken, yylexval, new b4_location_type (yylexpos));
   }
@@ -862,7 +863,7 @@ b4_dollar_popdef[]dnl
    * @@return <tt>true</tt> if the parsing succeeds.  Note that this does not
    *          imply that there were no syntax errors.
    */
-  public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[
+  public boolean parse ()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[
   {
     if (yylexer == null)
       throw new NullPointerException("Null Lexer");
@@ -872,9 +873,9 @@ b4_dollar_popdef[]dnl
       ]b4_yystype[ lval = yylexer.getLVal();
 ]b4_locations_if([dnl
       b4_location_type yyloc = new b4_location_type (yylexer.getStartPos (),
-                                            yylexer.getEndPos ());])[
-      ]b4_locations_if([status = push_parse(token,lval,yyloc);],[
-      status = push_parse(token,lval);])[
+                                            yylexer.getEndPos ());])[]b4_locations_if([[
+      status = push_parse(token,lval,yyloc);]], [[
+      status = push_parse(token,lval);]])[
     } while (status == YYPUSH_MORE);
     return (status == YYACCEPT);
   }
@@ -1011,7 +1012,7 @@ b4_dollar_popdef[]dnl
   private static final ]b4_int_type_for([b4_translate])[ yytranslate_ (int t)
 ]b4_api_token_raw_if(dnl
 [[  {
-    return t;
+    return (]b4_int_type_for([b4_translate])[) t;
   }
 ]],
 [[  {

data/skeletons/yacc.c

@@ -673,7 +673,7 @@ static const ]b4_int_type_for([b4_toknum])[ yytoknum[] =
 
 /* Error token number */
 #define YYTERROR        1
-#define YYERRCODE       256
+#define YYERRCODE       ]b4_symbol(1, user_number)[
 
 ]b4_locations_if([[
 ]b4_yylloc_default_define[

doc/bison.texi

@@ -5841,7 +5841,10 @@ states and what is done for each type of lookahead token in that state.
 
 @deffn {Directive} %yacc
 Pretend the option @option{--yacc} was given, i.e., imitate Yacc, including
-its naming conventions.  @xref{Tuning the Parser}, for more.
+its naming conventions.  Only makes sense with the @file{yacc.c}
+skeleton. @xref{Tuning the Parser}, for more.
+
+Of course @code{%yacc} is a Bison extension@dots{}
 @end deffn
 
 
@@ -9972,7 +9975,7 @@ When a Bison grammar compiles properly but parses ``incorrectly'', the
 @node Enabling Traces
 @subsection  Enabling Traces
 There are several means to enable compilation of trace facilities, in
-decresing order of preference:
+decreasing order of preference:
 
 @table @asis
 @item the variable @samp{parse.trace}
@@ -10228,7 +10231,7 @@ Cleanup: popping nterm input ()
 @subsection The @code{YYPRINT} Macro
 @findex YYPRINT
 
-The @code{%printer} directive was introduced in Bison 1.50 (Novembre 2002).
+The @code{%printer} directive was introduced in Bison 1.50 (November 2002).
 Before then, @code{YYPRINT} provided a similar feature, but only for
 terminal symbols and only with the @file{yacc.c} skeleton.
 
@@ -10857,11 +10860,10 @@ different diagnostics to be generated (it implies @option{-Wyacc}), and may
 change behavior in other minor ways.  Most importantly, imitate Yacc's
 output file name conventions, so that the parser implementation file is
 called @file{y.tab.c}, and the other outputs are called @file{y.output} and
-@file{y.tab.h}.  Also, if generating a deterministic parser in C, generate
-@code{#define} statements in addition to an @code{enum} to associate token
-numbers with token names.  Thus, the following shell script can substitute
-for Yacc, and the Bison distribution contains such a script for
-compatibility with POSIX:
+@file{y.tab.h}.  Also, generate @code{#define} statements in addition to an
+@code{enum} to associate token numbers with token names.  Thus, the
+following shell script can substitute for Yacc, and the Bison distribution
+contains such a script for compatibility with POSIX:
 
 @example
 #! /bin/sh
@@ -10869,9 +10871,9 @@ bison -y "$@@"
 @end example
 
 The @option{-y}/@option{--yacc} option is intended for use with traditional
-Yacc grammars.  If your grammar uses Bison extensions like
-@samp{%glr-parser}, Bison might not be Yacc-compatible even if this option
-is specified.
+Yacc grammars.  This option only makes sense for the default C skeleton,
+@file{yacc.c}.  If your grammar uses Bison extensions Bison cannot be
+Yacc-compatible, even if this option is specified.
 @end table
 
 @node Output Files
@@ -12103,13 +12105,24 @@ designed the grammar for.
 %defines
 @end example
 
+@noindent
+@findex %define api.token.raw
+Because our scanner returns only genuine tokens and never simple characters
+(i.e., it returns @samp{PLUS}, not @samp{'+'}), we can avoid conversions.
+
+@comment file: calc++/parser.yy
+@example
+%define api.token.raw
+@end example
+
 @noindent
 @findex %define api.token.constructor
 @findex %define api.value.type variant
-This example will use genuine C++ objects as semantic values, therefore, we
-require the variant-based interface.  To make sure we properly use it, we
-enable assertions.  To fully benefit from type-safety and more natural
-definition of ``symbol'', we enable @code{api.token.constructor}.
+This example uses genuine C++ objects as semantic values, therefore, we
+require the variant-based storage of semantic values.  To make sure we
+properly use it, we enable assertions.  To fully benefit from type-safety
+and more natural definition of ``symbol'', we enable
+@code{api.token.constructor}.
 
 @comment file: calc++/parser.yy
 @example
@@ -14604,7 +14617,7 @@ London, Department of Computer Science, TR-00-12 (December 2000).
 @c LocalWords: strncmp intval tindex lvalp locp llocp typealt YYBACKUP subrange
 @c LocalWords: YYEMPTY YYEOF YYRECOVERING yyclearin GE def UMINUS maybeword loc
 @c LocalWords: Johnstone Shamsa Sadaf Hussain Tomita TR uref YYMAXDEPTH inline
-@c LocalWords: YYINITDEPTH stmts ref initdcl maybeasm notype Lookahead
+@c LocalWords: YYINITDEPTH stmts ref initdcl maybeasm notype Lookahead ctx
 @c LocalWords: hexflag STR exdent itemset asis DYYDEBUG YYFPRINTF args Autoconf
 @c LocalWords: ypp yxx itemx tex leaderfill Troubleshouting sqrt Graphviz
 @c LocalWords: hbox hss hfill tt ly yyin fopen fclose ofirst gcc ll lookahead
@@ -14648,10 +14661,12 @@ London, Department of Computer Science, TR-00-12 (December 2000).
 @c LocalWords: Relocatability exprs fixit Wyacc parseable fixits ffixit svg
 @c LocalWords: DNDEBUG cstring Wzero workalike POPL workalikes byacc UCB
 @c LocalWords: Penello's Penello Byson Byson's Corbett's CSD TOPLAS PDP
-@c LocalWords: Beazley's goyacc ocamlyacc SIGACT SIGPLAN colorWarning
+@c LocalWords: Beazley's goyacc ocamlyacc SIGACT SIGPLAN colorWarning exVal
 @c LocalWords: setcolor rgbError colorError rgbNotice colorNotice derror
 @c LocalWords: colorOff maincolor inlineraw darkviolet darkcyan dwarning
-@c LocalWords: dnotice copyable stdint ptrdiff bufsize
+@c LocalWords: dnotice copyable stdint ptrdiff bufsize yyreport invariants
+@c LocalWords: xrefautomaticsectiontitle yysyntax yysymbol ARGMAX cond
+@c LocalWords: Wdangling
 
 @c Local Variables:
 @c ispell-dictionary: "american"

etc/bench.pl.in

@@ -100,7 +100,7 @@ Artificial grammar with very long rules.
 
 =item B<-h>, B<--help>
 
-Display this message and exit succesfully.  The more verbose, the more
+Display this message and exit successfully.  The more verbose, the more
 details.
 
 =item B<-i>, B<--iterations>=I<integer>

examples/c++/README.md

@@ -55,5 +55,5 @@ Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
 Texts.  A copy of the license is included in the "GNU Free
 Documentation License" file as part of this distribution.
 
-# LocalWords:  mfcalc calc parsers yy ispell american
+LocalWords:  mfcalc calc parsers yy ispell american
 --->

examples/c++/calc++/README.md

@@ -38,20 +38,12 @@ End:
 
 Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
-This file is part of Bison, the GNU Compiler Compiler.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# LocalWords:  calc parsers yy MERCHANTABILITY Ctrl ispell american
+LocalWords:  calc parsers yy MERCHANTABILITY Ctrl ispell american
 --->

examples/c/README.md

@@ -47,6 +47,8 @@ End:
 
 Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
+This file is part of GNU bison, the GNU Compiler Compiler.
+
 Permission is granted to copy, distribute and/or modify this document
 under the terms of the GNU Free Documentation License, Version 1.3 or
 any later version published by the Free Software Foundation; with no

examples/c/calc/README.md

@@ -13,16 +13,11 @@ Copyright (C) 2019-2020 Free Software Foundation, Inc.
 
 This file is part of Bison, the GNU Compiler Compiler.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
 --->

examples/c/lexcalc/README.md

@@ -13,16 +13,10 @@ Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
 This file is part of Bison, the GNU Compiler Compiler.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 --->

examples/c/reccalc/README.md

@@ -28,18 +28,11 @@ End:
 
 Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
-This file is part of Bison, the GNU Compiler Compiler.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
 --->

examples/c/reccalc/parse.y

@@ -4,7 +4,11 @@
 // Emitted in the header file, before the definition of YYSTYPE.
 %code requires
 {
+  #ifndef YY_TYPEDEF_YY_SCANNER_T
+  # define YY_TYPEDEF_YY_SCANNER_T
   typedef void* yyscan_t;
+  #endif
+
   typedef struct
   {
     // Whether to print the intermediate results.
@@ -132,7 +136,6 @@ exp:
 
 %%
 // Epilogue (C code).
-
 #include "scan.h"
 
 result

examples/test

@@ -31,18 +31,18 @@ exit=true
 cwd=$(pwd)
 
 # The exercised program.
-for p in "$cwd/examples/$medir/$me"
-do
-    if test -x "$p"; then
-        prog=$p
-        break
-    elif test -f "$p.class"; then
-        pwd
-        prog="$SHELL $cwd/javaexec.sh -cp $(dirname $p) $(basename $p)"
-        break
-    fi
-done
-if test x"$prog" = x; then
+abs_medir=$cwd/examples/$medir
+if test -x "$abs_medir/$me"; then
+    prog ()
+    {
+        "$abs_medir/$me" "$@"
+    }
+elif test -f "$abs_medir/$me.class"; then
+    prog ()
+    {
+        "$SHELL" "$cwd/javaexec.sh" -cp "$abs_medir" "$me" "$@"
+    }
+else
     echo "$me: ERROR: cannot find program to exercise in:"
     echo "$me: ERROR:   $cwd/examples/$medir/$me"
     exit 1
@@ -55,7 +55,7 @@ cleanup ()
 {
   status=$?
   if test -z "$DEBUG"; then
-     cd $cwd
+     cd "$cwd"
      rm -rf $$.dir
   fi
   exit $status
@@ -82,7 +82,7 @@ run ()
   shift
   # Effective exit status.
   sta_eff=0
-  $prog "$@" - <input >out_eff 2>err_eff || sta_eff=$?
+  prog "$@" - <input >out_eff 2>err_eff || sta_eff=$?
   # Combine effective output and error streams.
   out_eff=$(cat out_eff && $noerr || sed -e 's/^/err: /g' err_eff)
   if test $sta_eff -eq $sta_exp; then

lib/.gitignore

@@ -138,6 +138,8 @@
 /isnanl-nolibm.h
 /isnanl.c
 /iswblank.c
+/iswdigit.c
+/iswxdigit.c
 /itold.c
 /lc-charset-dispatch.c
 /lc-charset-dispatch.h
@@ -147,6 +149,7 @@
 /limits.in.h
 /localcharset.c
 /localcharset.h
+/locale.h
 /locale.in.h
 /localtime-buffer.c
 /localtime-buffer.h

m4/.gitignore

@@ -69,6 +69,8 @@
 /isnanf.m4
 /isnanl.m4
 /iswblank.m4
+/iswdigit.m4
+/iswxdigit.m4
 /javacomp.m4
 /javaexec.m4
 /jm-winsz1.m4
@@ -210,3 +212,4 @@
 /xalloc.m4
 /xsize.m4
 /xstrndup.m4
+/zzgnulib.m4

src/complain.c

@@ -41,9 +41,9 @@ bool warnings_are_errors = false;
 /** Whether -Werror/-Wno-error was applied to a warning.  */
 typedef enum
   {
-    errority_unset = 0,     /** No explict status.  */
-    errority_disabled = 1,  /** Explictly disabled with -Wno-error=foo.  */
-    errority_enabled = 2    /** Explictly enabled with -Werror=foo. */
+    errority_unset = 0,     /** No explicit status.  */
+    errority_disabled = 1,  /** Explicitly disabled with -Wno-error=foo.  */
+    errority_enabled = 2    /** Explicitly enabled with -Werror=foo. */
   } errority;
 
 /** For each warning type, its errority.  */

src/gram.h

@@ -42,9 +42,9 @@
 
    Internally, we cannot use the number 0 for a rule because for
    instance RITEM stores both symbol (the RHS) and rule numbers: the
-   symbols are shorts >= 0, and rule number are stored negative.
+   symbols are integers >= 0, and rule numbers are stored negative.
    Therefore 0 cannot be used, since it would be both the rule number
-   0, and the token $end).
+   0, and the token $end.
 
    Actions are accessed via the rule number.
 
@@ -55,8 +55,7 @@
 
    RULES[R].lhs -- the symbol of the left hand side of rule R.
 
-   RULES[R].rhs -- the index in RITEM of the beginning of the portion
-   for rule R.
+   RULES[R].rhs -- the beginning of the portion of RITEM for rule R.
 
    RULES[R].prec -- the symbol providing the precedence level of R.
 
@@ -75,16 +74,16 @@
 
    RULES[R].line -- the line where R was defined.
 
-   RULES[R].useful -- whether the rule is used (i.e., false if thrown
-   away by reduce).
+   RULES[R].useful -- whether the rule is used.  False if thrown away
+   by reduce().
 
    The right hand side is stored as symbol numbers in a portion of
    RITEM.
 
    The length of the portion is one greater than the number of symbols
    in the rule's right hand side.  The last element in the portion
-   contains minus R, which identifies it as the end of a portion and
-   says which rule it is for.
+   contains -R, which identifies it as the end of a portion and says
+   which rule it is for.
 
    The portions of RITEM come in order of increasing rule number.
    NRITEMS is the total length of RITEM.  Each element of RITEM is

src/location.c

@@ -175,6 +175,8 @@ location_print (location loc, FILE *out)
     }
   else
     {
+      aver (loc.start.file);
+      aver (loc.end.file);
       int end_col = 0 != loc.end.column ? loc.end.column - 1 : 0;
       res += fprintf (out, "%s",
                       quotearg_n_style (3, escape_quoting_style, loc.start.file));
@@ -317,7 +319,7 @@ caret_getc_internal (mbchar_t *res)
 
 /* Move CARET_INFO (which has a valid FILE) to the line number LINE.
    Compute and cache that line's length in CARET_INFO.LINE_LEN.
-   Return whether succesful.*/
+   Return whether successful.  */
 static bool
 caret_set_line (int line)
 {
@@ -421,12 +423,14 @@ location_caret (location loc, const char *style, FILE *out)
       {
         /* The last column to highlight.  Only the first line of
            multiline locations are quoted, in which case the ending
-           column is the end of line.  Single point locations (with
-           equal boundaries) denote the character that they
-           follow.  */
-        int col_end
+           column is the end of line.
+
+           We used to work with byte offsets, and that was much
+           easier.  However, we went back to using (visual) columns to
+           support truncating of long lines.  */
+        const int col_end
           = loc.start.line == loc.end.line
-          ? loc.end.column + (loc.start.column == loc.end.column)
+          ? loc.end.column
           : caret_info.line_len;
         /* Quote the file (at most the first line in the case of
            multiline locations).  */
@@ -436,24 +440,28 @@ location_caret (location loc, const char *style, FILE *out)
              expected (maybe the file was changed since the scanner
              ran), we might reach the end before we actually saw the
              opening column.  */
-          bool opened = false;
+          enum { before, inside, after } state = before;
           while (!mb_iseof (c) && !mb_iseq (c, '\n'))
             {
-              if (caret_info.pos.column == loc.start.column)
+              // We might have already opened (and even closed!) the
+              // style and yet have the equality of the columns if we
+              // just saw zero-width characters.
+              if (state == before
+                  && caret_info.pos.column == loc.start.column)
                 {
                   begin_use_class (style, out);
-                  opened = true;
+                  state = inside;
                 }
               if (skip < caret_info.pos.column)
                 mb_putc (c, out);
               boundary_compute (&caret_info.pos, mb_ptr (c), mb_len (c));
               caret_getc (c);
-              if (opened
+              if (state == inside
                   && (caret_info.pos.column == col_end
                       || width < caret_info.pos.column - skip))
                 {
                   end_use_class (style, out);
-                  opened = false;
+                  state = after;
                 }
               if (width < caret_info.pos.column - skip)
                 {
@@ -461,6 +469,12 @@ location_caret (location loc, const char *style, FILE *out)
                   break;
                 }
             }
+          if (state == inside)
+            {
+              // The line is shorter than expected.
+              end_use_class (style, out);
+              state = after;
+            }
           putc ('\n', out);
         }
 

src/location.h

@@ -42,16 +42,14 @@ typedef struct
 
   /* If positive, the column (starting at 1) just after the boundary.
      This is neither a byte count, nor a character count; it is a
-     column count.  If this is INT_MAX, the column number has
+     (visual) column count.  If this is INT_MAX, the column number has
      overflowed.
 
-     Meaningless and not displayed if nonpositive.
-  */
+     Meaningless and not displayed if nonpositive.  */
   int column;
 
-  /* If nonnegative, the byte number (starting at 0) in the current line.
-     Never displayed, used when printing error messages with colors to
-     know where colors start and end.  */
+  /* If nonnegative, the byte number (starting at 0) in the current
+     line.  Not displayed (unless --trace=location).  */
   int byte;
 
 } boundary;

src/lr0.c

@@ -58,10 +58,11 @@ core_print (size_t core_size, item_number *core, FILE *out)
     }
 }
 
-/*------------------------------------------------------------------.
-| A state was just discovered from another state.  Queue it for     |
-| later examination, in order to find its transitions.  Return it.  |
-`------------------------------------------------------------------*/
+/*-----------------------------------------------------------------.
+| A state was just discovered by transitioning on SYM from another |
+| state.  Queue this state for later examination, in order to find |
+| its outgoing transitions.  Return it.                            |
+`-----------------------------------------------------------------*/
 
 static state *
 state_list_append (symbol_number sym, size_t core_size, item_number *core)
@@ -98,7 +99,7 @@ static state **shiftset;
 
 
 /* KERNEL_BASE[symbol-number] -> list of item numbers (offsets inside
-   RITEM) of lenngth KERNEL_SIZE[symbol-number]. */
+   RITEM) of length KERNEL_SIZE[symbol-number]. */
 static item_number **kernel_base;
 static int *kernel_size;
 

src/lr0.h

@@ -21,8 +21,6 @@
 #ifndef LR0_H_
 # define LR0_H_
 
-# include "state.h"
-
 void generate_states (void);
 
 #endif /* !LR0_H_ */

src/muscle-tab.c

@@ -292,7 +292,6 @@ muscle_location_grow (char const *key, location loc)
 
 #define COMMON_DECODE(Value)                                    \
   case '$':                                                     \
-    ++(Value); aver (*(Value) == '[');                          \
     ++(Value); aver (*(Value) == ']');                          \
     ++(Value); aver (*(Value) == '[');                          \
     obstack_sgrow (&muscle_obstack, "$");                       \

src/parse-gram.c

@@ -1,4 +1,4 @@
-/* A Bison parser, made by GNU Bison 3.5.22-948cd-dirty.  */
+/* A Bison parser, made by GNU Bison 3.5.2.  */
 
 /* Bison implementation for Yacc-like parsers in C
 
@@ -48,7 +48,7 @@
 #define YYBISON 1
 
 /* Bison version.  */
-#define YYBISON_VERSION "3.5.22-948cd-dirty"
+#define YYBISON_VERSION "3.5.2"
 
 /* Skeleton name.  */
 #define YYSKELETON_NAME "yacc.c"

src/parse-gram.h

@@ -1,4 +1,4 @@
-/* A Bison parser, made by GNU Bison 3.5.22-948cd-dirty.  */
+/* A Bison parser, made by GNU Bison 3.5.2.  */
 
 /* Bison interface for Yacc-like parsers in C
 

src/print-xml.c

@@ -329,7 +329,7 @@ print_reductions (FILE *out, int level, state *s)
 
 /*--------------------------------------------------------------.
 | Report on OUT all the actions (shifts, gotos, reductions, and |
-| explicit erros from %nonassoc) of S.                          |
+| explicit errors from %nonassoc) of S.                         |
 `--------------------------------------------------------------*/
 
 static void

src/print.c

@@ -319,7 +319,7 @@ print_reductions (FILE *out, state *s)
 
 /*--------------------------------------------------------------.
 | Report on OUT all the actions (shifts, gotos, reductions, and |
-| explicit erros from %nonassoc) of S.                          |
+| explicit errors from %nonassoc) of S.                         |
 `--------------------------------------------------------------*/
 
 static void

src/reader.c

@@ -610,7 +610,7 @@ packgram (void)
 {
   int itemno = 0;
   ritem = xnmalloc (nritems + 1, sizeof *ritem);
-  /* This sentinel is used by build_relations in gram.c.  */
+  /* This sentinel is used by build_relations() in lalr.c.  */
   *ritem++ = 0;
 
   rule_number ruleno = 0;

src/reader.h

@@ -35,12 +35,6 @@ typedef struct merger_list
   location type_declaration_loc;
 } merger_list;
 
-/* From the parser.  */
-extern int gram_debug;
-int gram_parse (void);
-
-
-/* From reader.c. */
 void grammar_start_symbol_set (symbol *sym, location loc);
 void grammar_current_rule_begin (symbol *lhs, location loc,
                                  named_ref *lhs_named_ref);

src/scan-code.l

@@ -81,7 +81,7 @@ static bool untyped_var_seen;
    historically almost any character is allowed in a tag.  We disallow
    NUL and newline, as this simplifies our implementation.  We allow
    "->" as a means to dereference a pointer.  */
-tag      ([^\0\n>]|->)+
+tag      ([^\0\n>]|->)*[^-]
 
 /* Zero or more instances of backslash-newline.  Following GCC, allow
    white space between the backslash and the newline.  */
@@ -617,7 +617,7 @@ handle_action_dollar (symbol_list *rule, char *text, const location *dollar_loc)
   char *cp = fetch_type_name (text + 1, &type_name, dollar_loc);
   int n = parse_ref (cp, effective_rule, effective_rule_length,
                      rule->midrule_parent_rhs_index, text, dollar_loc, '$');
-  /* End type_name.  Don't do it ealier: parse_ref depends on TEXT.  */
+  /* End type_name.  Don't do it earlier: parse_ref depends on TEXT.  */
   if (type_name)
     cp[-1] = '\0';
 

src/symtab.c

@@ -77,10 +77,12 @@ sym_content_new (symbol *s)
   res->symbol = s;
 
   res->type_name = NULL;
+  res->type_loc = empty_loc;
   for (int i = 0; i < CODE_PROPS_SIZE; ++i)
     code_props_none_init (&res->props[i]);
 
   res->number = NUMBER_UNDEFINED;
+  res->prec_loc = empty_loc;
   res->prec = 0;
   res->assoc = undef_assoc;
   res->user_token_number = USER_NUMBER_UNDEFINED;
@@ -553,7 +555,10 @@ symbol_class_set (symbol *sym, symbol_class class, location loc, bool declaring)
                                _("previous declaration"));
             }
           else
-            s->status = declared;
+            {
+              sym->location = loc;
+              s->status = declared;
+            }
         }
     }
 }

src/symtab.h

@@ -130,7 +130,7 @@ struct sym_content
   /** Its \c \%type's location.  */
   location type_loc;
 
-  /** Any \c \%destructor (resp. \%printer) declared specificially for this
+  /** Any \c \%destructor (resp. \%printer) declared specifically for this
       symbol.
 
       Access this field only through <tt>symbol</tt>'s interface functions. For
@@ -225,7 +225,8 @@ void symbol_precedence_set (symbol *sym, int prec, assoc a, location loc);
 /** Set the \c class associated with \c sym.
 
     Whether \c declaring means whether this class definition comes
-    from %nterm or %token (but not %type, prec/assoc, etc.).  */
+    from %nterm or %token (but not %type, prec/assoc, etc.).  A symbol
+    can have "declaring" set only at most once.  */
 void symbol_class_set (symbol *sym, symbol_class class, location loc,
                        bool declaring);
 

src/system.h

@@ -73,9 +73,14 @@ typedef size_t uintptr_t;
 # include <verify.h>
 # include <xalloc.h>
 
+// Clang and ICC like to pretend they are GCC.
+# if defined __GNUC__ && !defined __clang__ && !defined __ICC
+#  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+# endif
 
-/* See https://lists.gnu.org/archive/html/bug-bison/2019-10/msg00061.html. */
-# if defined __GNUC__ && ! defined __clang__ && ! defined __ICC && __GNUC__ < 5
+// See https://lists.gnu.org/archive/html/bug-bison/2019-10/msg00061.html
+// and https://trac.macports.org/ticket/59927.
+# if defined GCC_VERSION && 405 <= GCC_VERSION
 #  define IGNORE_TYPE_LIMITS_BEGIN \
      _Pragma ("GCC diagnostic push") \
      _Pragma ("GCC diagnostic ignored \"-Wtype-limits\"")
@@ -209,10 +214,10 @@ typedef size_t uintptr_t;
 
 /* Output Str both quoted for M4 (i.e., embed in [[...]]), and escaped
    for our postprocessing (i.e., escape M4 special characters).  If
-   Str is empty (or NULL), output "[]" instead of "[[]]" as it make M4
-   programming easier (m4_ifval can be used).
+   Str is empty (or NULL), output "[]" instead of "[[]]" as it makes
+   M4 programming easier (m4_ifval can be used).
 
-   For instance "[foo]" -> "[[@{foo@}]]", "$$" -> "[[$][$][]]". */
+   For instance "[foo]" -> "[[@{foo@}]]", "$$" -> "[[$][$][]]".  */
 
 # define obstack_quote(Obs, Str)                \
   do {                                          \

tests/calc.at

@@ -720,7 +720,7 @@ _AT_CHECK_CALC_ERROR([$1], [0], [(- *) + (1 2) = 1], [113],
 calc: error: 2222 != 1])
 
 # Check that yyerrok works properly: second error is not reported,
-# third and fourth are.  Parse status is succesful.
+# third and fourth are.  Parse status is successful.
 _AT_CHECK_CALC_ERROR([$1], [0], [(* *) + (*) + (*)], [113],
 [1.2: syntax error, unexpected '*', expecting number or '-' or '(' or '!'
 1.10: syntax error, unexpected '*', expecting number or '-' or '(' or '!'

tests/diagnostics.at

@@ -37,15 +37,15 @@ AT_BISON_OPTION_PUSHDEFS
 
 AT_DATA_GRAMMAR([[input.y]], [$2])
 
+AT_DATA([experr], [$4])
+
 # For some reason, literal ^M in the input are removed and don't end
 # in `input.y`.  So use the two-character ^M represent it, and let
 # Perl insert real CR characters.
-if grep '\^M' input.y >/dev/null; then
-  AT_PERL_REQUIRE([-pi -e 's{\^M}{\r}gx' input.y])
+if $EGREP ['\^M|\\[0-9][0-9][0-9]'] input.y experr >/dev/null; then
+  AT_PERL_REQUIRE([-pi -e 's{\^M}{\r}g;s{\\(\d{3}|.)}{$v = $[]1; $v =~ /\A\d+\z/ ? chr($v) : $v}ge' input.y experr])
 fi
 
-AT_DATA([experr], [$4])
-
 AT_CHECK([LC_ALL="$locale" $5 bison -fcaret --color=debug -Wall input.y], [$3], [], [experr])
 
 # When no style, same messages, but without style.
@@ -152,6 +152,65 @@ input.y: <warning>warning:</warning> fix-its can be applied.  Rerun with option
 ]])
 
 
+
+## ------------------------------------- ##
+## Line is too short, and then you die.  ##
+## ------------------------------------- ##
+
+# We trust the "#line", since that's what allows us to quote the
+# actual source from which the gramar file was generated.  But #line
+# can also be wrong, and point to a line which is shorter that the bad
+# one.  In which case we can easily forget to close the styling.
+#
+# Be sure to have #line point to a line long enough to open the
+# styling, but not enough to close it.
+
+AT_TEST([[Line is too short, and then you die]],
+[[// Beware that there are 9 lines inserted before (including this one).
+#line 12
+%token foo 123
+%token foo 123123
+%token foo 123
+%%
+exp:
+]],
+[1],
+[[input.y:13.8-10: <warning>warning:</warning> symbol foo redeclared [<warning>-Wother</warning>]
+   13 | %token <warning>foo</warning> 123
+      |        <warning>^~~</warning>
+input.y:12.8-10:     previous declaration
+   12 | %token <note>foo</note> 123123
+      |        <note>^~~</note>
+input.y:13.12-17: <error>error:</error> redefining user token number of foo
+   13 | %token foo <error>123</error>
+      |            <error>^~~~~~</error>
+input.y:14.8-10: <warning>warning:</warning> symbol foo redeclared [<warning>-Wother</warning>]
+   14 | %%
+      |        <warning>^~~</warning>
+input.y:12.8-10:     previous declaration
+   12 | %token <note>foo</note> 123123
+      |        <note>^~~</note>
+]])
+
+
+## ----------------------- ##
+## Zero-width characters.  ##
+## ----------------------- ##
+
+# We used to open twice the styling for characters that have a
+# zero-width on display (e.g., \005).
+
+AT_TEST([[Zero-width characters]],
+[[%%
+exp: an\005error.
+]],
+[1],
+[[input.y:10.8: <error>error:</error> invalid character: '\\005'
+   10 | exp: an<error>\005</error>error.
+      |        <error>^</error>
+]])
+
+
 ## -------------------------------------- ##
 ## Tabulations and multibyte characters.  ##
 ## -------------------------------------- ##
@@ -262,7 +321,7 @@ input.y:10.1-27: <error>error:</error> %define variable 'error2' is not used
 ## ----------------- ##
 
 # Carriage-return used to count as a newline in the scanner, and not
-# in diagnostics.  Resulting in all sort of nice bugs.
+# in diagnostics.  Resulting in all kinds of nice bugs.
 
 AT_TEST([[Carriage return]],
 [[^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M

tests/input.at

@@ -298,6 +298,52 @@ input.y:8.14: error: syntax error, unexpected integer
 AT_CLEANUP
 
 
+## ---------------------------- ##
+## Redefining the error token.  ##
+## ---------------------------- ##
+
+AT_SETUP([Redefining the error token])
+
+# We used to crash when trying to display the original definition of
+# "error", which is a builtin without any location.
+
+AT_DATA([input.y],
+[[%token error 123
+%token error 124
+%%
+exp:
+]])
+
+AT_BISON_CHECK([-fcaret input.y], [1], [],
+[[input.y:2.8-12: warning: symbol error redeclared [-Wother]
+    2 | %token error 124
+      |        ^~~~~
+input.y:1.8-12:     previous declaration
+    1 | %token error 123
+      |        ^~~~~
+input.y:2.14-16: error: redefining user token number of error
+    2 | %token error 124
+      |              ^~~
+]])
+
+# While at it, make sure we properly used the user's number for
+# "error".
+AT_DATA([input.y],
+[[%token error 123
+%%
+exp:
+]])
+
+AT_BISON_CHECK([input.y])
+
+AT_CHECK([$EGREP -E '123|256' input.tab.c], [],
+[[       0,   123,   257
+#define YYERRCODE       123
+]])
+
+AT_CLEANUP
+
+
 ## ------------------ ##
 ## Dangling aliases.  ##
 ## ------------------ ##
@@ -2069,6 +2115,25 @@ input.y:1.1-34:     accepted value: 'consistent'
 input.y:1.1-34:     accepted value: 'accepting'
 ]])
 
+# Check escapes.
+AT_DATA([[input.y]],
+[[%define lr.default-reduction {[$@]}
+%%
+start: %empty;
+]])
+AT_BISON_CHECK([[-fcaret input.y]], [[1]], [[]],
+[[input.y:1.1-35: warning: %define variable 'lr.default-reduction' requires keyword values [-Wdeprecated]
+    1 | %define lr.default-reduction {[$@]}
+      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+input.y:1.1-35: error: invalid value for %define variable 'lr.default-reduction': '[$@]'
+    1 | %define lr.default-reduction {[$@]}
+      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+input.y:1.1-35:     accepted value: 'most'
+input.y:1.1-35:     accepted value: 'consistent'
+input.y:1.1-35:     accepted value: 'accepting'
+]])
+
+
 # Back-end.
 AT_DATA([[input.y]],
 [[%define api.push-pull neither
@@ -2367,6 +2432,25 @@ AT_DATA([[input.y]],
 start: %empty;
 ]])
 
+# Only "full" and "none" are accepted for parse.lac
+# Unknown values (such as "unsupported") are rejected
+AT_BISON_CHECK([[-Dparse.lac=none input.y]])
+AT_BISON_CHECK([[-Dparse.lac=none -L c++ input.y]])
+AT_BISON_CHECK([[-Dparse.lac=full input.y]])
+AT_BISON_CHECK([[-Dparse.lac=full -L c++ input.y]])
+AT_BISON_CHECK([[-Dparse.lac=unsupported input.y]],
+               [[1]], [],
+[[<command line>:3: error: invalid value for %define variable 'parse.lac': 'unsupported'
+<command line>:3:     accepted value: 'full'
+<command line>:3:     accepted value: 'none'
+]])
+AT_BISON_CHECK([[-Dparse.lac=unsupported -L c++ input.y]],
+               [[1]], [],
+[[<command line>:3: error: invalid value for %define variable 'parse.lac': 'unsupported'
+<command line>:3:     accepted value: 'full'
+<command line>:3:     accepted value: 'none'
+]])
+
 # parse.lac.* options are useless if LAC isn't actually activated.
 AT_BISON_CHECK([[-Dparse.lac.es-capacity-initial=1 input.y]],
                [[1]], [],
@@ -2378,11 +2462,11 @@ AT_BISON_CHECK([[-Dparse.lac.memory-trace=full input.y]],
 ]])
 
 # parse.lac.* options are useless in C++ even if LAC isn't actually activated.
-AT_BISON_CHECK([[-Dparse.lac.es-capacity-initial=1 -L C++ -Dparse.lac input.y]],
+AT_BISON_CHECK([[-Dparse.lac.es-capacity-initial=1 -L C++ -Dparse.lac=full input.y]],
                [[1]], [],
 [[<command line>:3: error: %define variable 'parse.lac.es-capacity-initial' is not used
 ]])
-AT_BISON_CHECK([[-Dparse.lac.memory-trace=full -L C++ -Dparse.lac  input.y]],
+AT_BISON_CHECK([[-Dparse.lac.memory-trace=full -L C++ -Dparse.lac=full input.y]],
                [[1]], [],
 [[<command line>:3: error: %define variable 'parse.lac.memory-trace' is not used
 ]])
@@ -2529,7 +2613,9 @@ AT_DATA_GRAMMAR([[input.y]],
 %printer        { $%; @%; } <*> exp TOK;
 %{ $ @ %} // Should not warn.
 %%
-exp: TOK        { $%; @%; $$ = $1; };
+exp: TOK        { $%; @%; $$ = $1; }
+   | 'a'        { $<->1; $$ = 1; }
+   | 'b'        { $<foo->bar>$; }
 %%
 $ @ // Should not warn.
 ]])
@@ -2543,6 +2629,7 @@ input.y:13.19: warning: stray '$' [-Wother]
 input.y:13.23: warning: stray '@' [-Wother]
 input.y:16.19: warning: stray '$' [-Wother]
 input.y:16.23: warning: stray '@' [-Wother]
+input.y:17.19: warning: stray '$' [-Wother]
 ]])
 
 AT_BISON_OPTION_POPDEFS

tests/local.at

@@ -233,6 +233,11 @@ m4_pushdef([AT_PARSE_PARAMS])
 m4_bpatsubst([$3], [%parse-param { *\([^{}]*[^{} ]\) *}],
              [m4_append([AT_PARSE_PARAMS], [\1, ])])
 
+m4_pushdef([AT_PARSER_CLASS],
+[m4_bmatch([$3], [%define  *api\.parser\.class {\([^\}]*\)}],
+           [m4_bregexp([$3], [%define  *api\.parser\.class {\([^\}]*\)}], [\1])],
+           [AT_API_PREFIX[]Parser])])
+
 m4_pushdef([AT_PURE_IF],
 [m4_bmatch([$3], [%define  *api\.pure\|%pure-parser],
            [m4_bmatch([$3], [%define  *api\.pure *false], [$2], [$1])],
@@ -382,6 +387,7 @@ m4_popdef([AT_LOCATION_IF])
 m4_popdef([AT_PARSE_PARAMS])
 m4_popdef([AT_PUSH_IF])
 m4_popdef([AT_PURE_IF])
+m4_popdef([AT_PARSER_CLASS])
 m4_popdef([AT_PARAM_IF])
 m4_popdef([AT_LEXPARAM_IF])
 m4_popdef([AT_YACC_IF])
@@ -409,6 +415,12 @@ m4_define([AT_LANG_CASE],
 [m4_case(AT_LANG, $@)])
 
 
+# AT_LANG_MATCH(LANG1, IF-LANG1, LANG2, IF-LANG2, ..., DEFAULT)
+# ------------------------------------------------------------
+m4_define([AT_LANG_MATCH],
+[m4_bmatch(AT_LANG, $@)])
+
+
 # _AT_LANG_DISPATCH(LANG, MACRO, ARGS)
 # ------------------------------------
 # Call the specialization of MACRO for LANG with ARGS.  Complain if
@@ -747,6 +759,9 @@ m4_define([AT_MAIN_DEFINE(d)],
 # ------------------------------
 m4_copy([AT_DATA], [AT_DATA_GRAMMAR(java)])
 
+# No need to declare, it's part of the class interface.
+m4_define([AT_YYERROR_DECLARE(java)],        [])
+m4_define([AT_YYERROR_DECLARE_EXTERN(java)], [])
 
 # AT_JAVA_POSITION_DEFINE
 # -----------------------
@@ -829,10 +844,12 @@ m4_define([AT_YYLEX_DEFINE(java)],
 m4_define([AT_MAIN_DEFINE(java)],
 [[class input
 {
-  public static void main (String args[]) throws IOException
+  public static void main (String[] args) throws IOException
   {
     ]AT_API_prefix[Parser p = new ]AT_API_prefix[Parser ();
-    System.exit (p.parse () ? 0 : 1);
+    boolean success = p.parse ();
+    if (!success)
+      System.exit (1);
   }
 }]])
 

tests/local.mk

@@ -96,10 +96,10 @@ clean-local-tests:
 
 .PHONY: recheck
 recheck: $(RUN_TESTSUITE_deps)
-	$(RUN_TESTSUITE)							\
-	  $$(perl -n								\
-	     -e 'if (/Summary of the failures/../Detailed failed tests/)'	\
-	     -e '{ /^ *[0-9]+:/ && s/:.*//s && print }' %D%/testsuite.log)
+	$(RUN_TESTSUITE)					\
+	  $$(perl -n						\
+	     -e 'eof && /^(\d+).*: FAILED/ && print "$$1 "'	\
+		%D%/testsuite.dir/*/testsuite.log)
 
 check-local: $(RUN_TESTSUITE_deps)
 	$(RUN_TESTSUITE)

tests/scanner.at

@@ -158,6 +158,78 @@ m4_pushdef([AT_MAIN_DEFINE(d)],
 }]])
 
 
+m4_pushdef([AT_MAIN_DEFINE(java)],
+[[class input
+{
+  public static void main (String[] args) throws IOException
+  {]AT_LEXPARAM_IF([[
+    ]AT_PARSER_CLASS[ p = new ]AT_PARSER_CLASS[ (System.in);]], [[
+    ]AT_API_prefix[Lexer l = new ]AT_API_prefix[Lexer (System.in);
+    ]AT_PARSER_CLASS[ p = new ]AT_PARSER_CLASS[ (l);]])AT_DEBUG_IF([[
+    //p.setDebugLevel (1);]])[
+    boolean success = p.parse ();
+    if (!success)
+      System.exit (1);
+  }
+}]])
+
+m4_define([AT_RAW_YYLEX(java)],
+[[class CalcLexer implements Calc.Lexer {
+
+  StreamTokenizer st;
+
+  public CalcLexer (InputStream is)
+  {
+    st = new StreamTokenizer (new StringReader ("0-(1+2)*3/9"));
+    st.resetSyntax ();
+    st.eolIsSignificant (true);
+    st.whitespaceChars ('\t', '\t');
+    st.whitespaceChars (' ', ' ');
+    st.wordChars ('0', '9');
+  }
+
+  public void yyerror (String s)
+  {
+    System.err.println (s);
+  }
+
+  Integer yylval;
+
+  public Object getLVal () {
+    return yylval;
+  }
+
+  public int yylex () throws IOException {
+    int ttype = st.nextToken ();
+    switch (ttype)
+      {
+      case StreamTokenizer.TT_EOF:
+        return EOF;
+      case StreamTokenizer.TT_EOL:
+        return (int) '\n';
+      case StreamTokenizer.TT_WORD:
+        yylval = new Integer (st.sval);
+        return NUM;
+      case '+':
+        return PLUS;
+      case '-':
+        return MINUS;
+      case '*':
+        return STAR;
+      case '/':
+        return SLASH;
+      case '(':
+        return LPAR;
+      case ')':
+        return RPAR;
+      default:
+        throw new AssertionError ("invalid character: " + ttype);
+      }
+  }
+}
+]])
+
+
 ## ------------------- ##
 ## Raw token numbers.  ##
 ## ------------------- ##
@@ -166,19 +238,31 @@ m4_pushdef([AT_TEST],
 [
 AT_SETUP([Token numbers: $1])
 
-AT_BISON_OPTION_PUSHDEFS([%debug $1])
+AT_BISON_OPTION_PUSHDEFS([%debug ]m4_bmatch([$1], [java], [[%define api.prefix {Calc} %define api.parser.class {Calc}]])[ $1])
 AT_DATA_GRAMMAR([[input.y]],
 [[$1
 %debug
-]AT_D_IF([], [[
+]AT_LANG_MATCH([[c\|c++]], [[
 %code
 {
 #include <stdio.h>
 ]AT_YYERROR_DECLARE[
 ]AT_YYLEX_DECLARE[
-}]])[
+}]],
+  [java], [[
+%define api.prefix {Calc}
+%define api.parser.class {Calc}
+%code imports {
+  import java.io.IOException;
+  import java.io.InputStream;
+  import java.io.StringReader;
+  import java.io.Reader;
+  import java.io.StreamTokenizer;
+}
+]])[
 
-]AT_VARIANT_IF([[
+]AT_LANG_MATCH([c\|c++\|d],
+[AT_VARIANT_IF([[
 %token <int> NUM "number"
 %nterm <int> exp
 ]], [[
@@ -187,6 +271,10 @@ AT_DATA_GRAMMAR([[input.y]],
 }
 %token <val> NUM "number"
 %nterm <val> exp
+]])],
+    [java],
+[[%token <Integer> NUM "number"
+%type  <Integer> exp
 ]])[
 
 %token
@@ -204,7 +292,7 @@ AT_DATA_GRAMMAR([[input.y]],
 %%
 
 input
-: exp         { printf ("%d\n", $][1); }
+: exp         { ]AT_JAVA_IF([[System.out.println ($][1)]], [[printf ("%d\n", $][1)]])[; }
 ;
 
 exp
@@ -217,21 +305,26 @@ exp
 ;
 
 %%
-]AT_YYERROR_DEFINE[
+]AT_LANG_MATCH([c\|c++\|d],
+               [AT_YYERROR_DEFINE])[
 ]AT_RAW_YYLEX[
 ]AT_MAIN_DEFINE[
 ]])
 
 AT_FULL_COMPILE([input])
 
+# When api.token.raw, the yytranslate table should not be included.
+#
 # yacc.c, glr.c and glr.cc use 'yytranslate' (and YYTRANSLATE).
 # lalr1.cc uses 'translate_table' (and yytranslate_).
 # lalr1.d uses 'byte[] translate_table =' (and yytranslate_).
-AT_CHECK([[$EGREP -c 'yytranslate\[\]|translate_table\[\]|translate_table =' input.]AT_LANG_EXT],
+# lalr1.java uses 'byte[] translate_table_ =' (and yytranslate_).
+AT_CHECK([[$EGREP -c 'yytranslate\[\]|translate_table\[\]|translate_table =|translate_table_ =' input.]AT_LANG_EXT],
          [ignore],
          [AT_TOKEN_RAW_IF([0], [1])[
 ]])
 
+
 AT_PARSER_CHECK([input], 0,
 [[-1
 ]])
@@ -240,11 +333,12 @@ AT_BISON_OPTION_POPDEFS
 AT_CLEANUP
 ])
 
-m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], [glr.cc], [lalr1.d]],
+m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], [glr.cc], [lalr1.java], [lalr1.d]],
 [AT_TEST([%skeleton "]b4_skel["])
  AT_TEST([%skeleton "]b4_skel[" %define api.token.raw])])
 
 AT_TEST([%skeleton "lalr1.cc" %define api.token.raw %define api.value.type variant %define api.token.constructor])])
 
+m4_popdef([AT_MAIN_DEFINE(java)])
 m4_popdef([AT_MAIN_DEFINE(d)])
 m4_popdef([AT_TEST])