version 3.7.2

* NEWS: Record release date.

.prev-version

@@ -1 +1 @@
-3.7
+3.7.1

.travis.yml

@@ -50,6 +50,7 @@ jobs:
         - make -j2 dist-xz
         # Can help understanding why we get "dirty" tarballs.
         - git status
+        - git diff
         - dist=$(echo bison*.xz)
 
         # Unfortunately we cannot deterministically know the name of the tarball without the full
@@ -129,7 +130,7 @@ jobs:
         - CC=icc
         - CXX=icpc
       install:
-        - source /opt/intel/inteloneapi/compiler/latest/env/vars.sh
+        - source /opt/intel/oneapi/compiler/latest/env/vars.sh
       addons:
         apt:
           sources:

NEWS

@@ -1,5 +1,28 @@
 GNU Bison NEWS
 
+* Noteworthy changes in release 3.7.2 (2020-09-05) [stable]
+
+  This release of Bison fixes all known bugs reported for Bison in MITRE's
+  Common Vulnerabilities and Exposures (CVE) system.  These vulnerabilities
+  are only about bison-the-program itself, not the generated code.
+
+  Although these bugs are typically irrelevant to how Bison is used, they
+  are worth fixing if only to give users peace of mind.
+
+  There is no known vulnerability in the generated parsers.
+
+** Bug fixes
+
+  Fix concurrent build issues (introduced in Bison 3.5).
+
+  Push parsers always use YYMALLOC/YYFREE (no direct calls to malloc/free).
+
+  Fix portability issues of the test suite, and of bison itself.
+
+  Some unlikely crashes found by fuzzing have been fixed.  This is only
+  about bison itself, not the generated parsers.
+
+
 * Noteworthy changes in release 3.7.1 (2020-08-02) [stable]
 
 ** Bug fixes
@@ -554,7 +577,8 @@ GNU Bison NEWS
   \005) with incorrect styling.  Fixes for similar issues with unexpectedly
   short lines (e.g., the file was changed between parsing and diagnosing).
 
-  Several unlikely crashes found by fuzzing have been fixed.
+  Some unlikely crashes found by fuzzing have been fixed.  This is only
+  about bison itself, not the generated parsers.
 
 
 * Noteworthy changes in release 3.5.2 (2020-02-13) [stable]

TODO

@@ -1,4 +1,12 @@
-* Bison 3.7
+* Soon
+** gnulib
+Bruno notes:
+
+> I haven't looked deeply, but it strikes me that gnulib/lib/bitset/array.c
+> does not make use of the 'ffsl' function, nor or the 'integer_length_l'
+> function. Maybe because in Bison, all bitsets are so dense that it does
+> not give a performance advantage?
+
 ** Cex
 *** Improve gnulib
 Don't do this (counterexample.c):

cfg.mk

@@ -126,7 +126,7 @@ _sed_rm_comments_q = $(subst ','\'',$(_sed_remove_comments))
 
 _space_before_paren_exempt =? \\n\\$$
 _space_before_paren_exempt = \
-  (^ *\#|(LA)?LR\([01]\)|percent_(code|define)|b4_syncline|m4_(define|init)|symbol)
+  (^ *\#|(LA)?LR\([01]\)|percent_(code|define)|b4_syncline|m4_(define|init))
 # Ensure that there is a space before each open parenthesis in C code.
 sc_space_before_open_paren:
 	@if $(VC_LIST_EXCEPT) | grep -l '\.[ch]$$' > /dev/null; then	\
@@ -156,6 +156,7 @@ exclude = \
 $(call exclude,                                                                 \
   bindtextdomain=^lib/main.c$$                                                  \
   cast_of_argument_to_free=^src/muscle-tab.c$$                                  \
+  error_message_uppercase=etc/bench.pl.in$$                                     \
   po_check=^tests|(^po/POTFILES.in|.md)$$                                       \
   preprocessor_indentation=^data/|^lib/|^src/parse-gram.[ch]$$                  \
   program_name=^lib/main.c$$                                                    \

data/skeletons/yacc.c

@@ -1486,7 +1486,7 @@ yypstate_new (void)
   yypstate *yyps;]b4_pure_if([], [[
   if (yypstate_allocated)
     return YY_NULLPTR;]])[
-  yyps = YY_CAST (yypstate *, malloc (sizeof *yyps));
+  yyps = YY_CAST (yypstate *, YYMALLOC (sizeof *yyps));
   if (!yyps)
     return YY_NULLPTR;]b4_pure_if([], [[
   yypstate_allocated = 1;]])[
@@ -1515,7 +1515,7 @@ yypstate_delete (yypstate *yyps)
 #endif]b4_lac_if([[
       if (yyes != yyesa)
         YYSTACK_FREE (yyes);]])[
-      free (yyps);]b4_pure_if([], [[
+      YYFREE (yyps);]b4_pure_if([], [[
       yypstate_allocated = 0;]])[
     }
 }

doc/bison.texi

@@ -6296,7 +6296,10 @@ Introduced in Bison 3.3 to replace @code{parser_class_name}.
 
 @item Default Value: @code{YY} for Java, @code{yy} otherwise.
 
-@item History: introduced in Bison 2.6
+@item History:
+introduced in Bison 2.6, with its argument in double quotes.  Uses braces
+since Bison 3.0 (double quotes are still supported for backward
+compatibility).
 @end itemize
 @end deffn
 

doc/local.mk

@@ -57,7 +57,7 @@ MAINTAINERCLEANFILES = $(CROSS_OPTIONS_TEXI)
 
 # Fix Info's @code in @deftype
 # https://lists.gnu.org/archive/html/help-texinfo/2019-11/msg00004.html
-all: $(srcdir)/$(%C%_bison).info.bak
+all-local: $(srcdir)/$(%C%_bison).info.bak
 $(srcdir)/$(%C%_bison).info.bak: $(srcdir)/$(%C%_bison).info
 	$(AM_V_GEN) $(PERL) -pi.bak -0777	\
 	  -e 's{(^ --.*\n(?: {10}.*\n)*)}'	\

etc/bench.pl.in

@@ -185,13 +185,13 @@ my $verbose = 1;
 
 =over 4
 
-=item C<verbose($level, $message)>
+=item C<verbose ($level, $message)>
 
 Report the C<$message> is C<$level> E<lt>= C<$verbose>.
 
 =cut
 
-sub verbose($$)
+sub verbose ($$)
 {
   my ($level, $message) = @_;
   print STDERR $message
@@ -201,13 +201,13 @@ sub verbose($$)
 
 ######################################################################
 
-=item C<directives($bench, @directive)>
+=item C<directives ($bench, @directive)>
 
 Format the list of directives for Bison for bench named C<$bench>.
 
 =cut
 
-sub directives($@)
+sub directives ($@)
 {
   my ($bench, @directive) = @_;
   my $res = "/* Directives for bench '$bench'. */\n";
@@ -218,6 +218,27 @@ sub directives($@)
 
 ######################################################################
 
+=item C<is_pure (@directive)>
+
+Whether api.pure is set.
+
+=cut
+
+sub is_pure (@)
+{
+  my (@directive) = @_;
+  for my $dir (@directive)
+    {
+      if ($dir =~ /\A%define api.pure/)
+        {
+          return 1;
+        }
+    }
+  return 0;
+}
+
+######################################################################
+
 =item C<generate_grammar_triangular ($base, $max, @directive)>
 
 Create a large triangular grammar which looks like :
@@ -389,18 +410,14 @@ sub generate_grammar_calc ($$@)
 %define api.value.type union
 $directives
 
-%{
+%code provides {
 static int power (int base, int exponent);
 /* yyerror receives the location if:
    - %location & %pure & %glr
    - %location & %pure & %yacc & %parse-param. */
 static void yyerror (const char *s);
-#if YYPURE
+static int yylex (@{[is_pure (@directive) ? "YYSTYPE *yylvalp" : "void"]});
-static int yylex (YYSTYPE* yylvalp);
+}
-#else
-static int yylex (void);
-#endif
-%}
 
 /* Bison Declarations */
 %token
@@ -467,12 +484,7 @@ yyerror (const char *s)
 }
 
 static int
-#if YYPURE
+yylex (@{[is_pure (@directive) ? "YYSTYPE *yylvalp" : "void"]})
-# define yylval (*yylvalp)
-yylex (YYSTYPE* yylvalp)
-#else
-yylex (void)
-#endif
 {
   int c;
 
@@ -498,7 +510,7 @@ yylex (void)
     case '5': case '6': case '7': case '8': case '9':
       {
         int nchars = 0;
-        int n = sscanf (input - 1, "%d%n", &yylval.NUM, &nchars);
+        int n = sscanf (input - 1, "%d%n", &@{[is_pure (@directive) ? "yylvalp->" : "yylval."]}NUM, &nchars);
         assert (n == 1);
         input += nchars - 1;
         return NUM;
@@ -506,7 +518,7 @@ yylex (void)
 
     default:
       yyerror ("error: invalid character");
-      return yylex ();
+      return yylex (@{[is_pure (@directive) ? "yylvalp" : ""]});
     }
 }
 EOF
@@ -592,10 +604,10 @@ $directives
   // Prototype of the yylex function providing subsequent tokens.
   static
 #if USE_TOKEN_CTOR
-  yy::parser::symbol_type yylex();
+  yy::parser::symbol_type yylex ();
 #else
-  yy::parser::token_type yylex(yy::parser::semantic_type* yylvalp,
+  yy::parser::token_type yylex (yy::parser::semantic_type *yylvalp,
-                               yy::parser::location_type* yyllocp);
+                                yy::parser::location_type *yyllocp);
 #endif
 
   // Conversion to string.
@@ -618,8 +630,8 @@ EOF
       print $out <<'EOF';
 %token <std::string> TEXT
 %token <int> NUMBER
-%printer { std::cerr << "Number: " << $$; } <int>
+%printer { yyo << "Number: " << $$; } <int>
-%printer { std::cerr << "Text: " << $$; } <std::string>
+%printer { yyo << "Text: " << $$; } <std::string>
 %type <std::string> text result
 
 %%
@@ -641,8 +653,8 @@ EOF
 %union {int ival; std::string* sval;}
 %token <sval> TEXT
 %token <ival> NUMBER
-%printer { std::cerr << "Number: " << $$; } <ival>
+%printer { yyo << "Number: " << $$; } <ival>
-%printer { std::cerr << "Text: " << *$$; } <sval>
+%printer { yyo << "Text: " << *$$; } <sval>
 %type <sval> text result
 
 %%
@@ -664,10 +676,10 @@ EOF
 
 static
 #if USE_TOKEN_CTOR
-yy::parser::symbol_type yylex()
+yy::parser::symbol_type yylex ()
 #else
-yy::parser::token_type yylex(yy::parser::semantic_type* yylvalp,
+yy::parser::token_type yylex (yy::parser::semantic_type *yylvalp,
-                             yy::parser::location_type* yyllocp)
+                              yy::parser::location_type *yyllocp)
 #endif
 {
   typedef yy::parser::location_type location_type;

examples/c/bistromathic/bistromathic.test

@@ -251,29 +251,29 @@ err: Next token is token ) (1.4: )
 err: Shifting token ) (1.4: )
 err: Entering state 20
 err: Stack now 0 2 10 20
-err: Reducing stack by rule 15 (line 151):
+err: Reducing stack by rule XX (line XXX):
 err:    $1 = token ( (1.1: )
 err:    $2 = token error (1.2-3: )
 err:    $3 = token ) (1.4: )
 err: -> $$ = nterm exp (1.1-4: 666)
-err: Entering state 7
+err: Entering state 8
-err: Stack now 0 7
+err: Stack now 0 8
 err: Return for a new token:
 err: Reading a token
 err: Now at end of input.
 err: LAC: initial context established for end of file
-err: LAC: checking lookahead end of file: R2 G8 S19
+err: LAC: checking lookahead end of file: R2 G7 S14
-err: Reducing stack by rule 2 (line 126):
+err: Reducing stack by rule XX (line XXX):
 err:    $1 = nterm exp (1.1-4: 666)
 err: -> $$ = nterm input (1.1-4: )
-err: Entering state 8
+err: Entering state 7
-err: Stack now 0 8
+err: Stack now 0 7
 err: Now at end of input.
 err: Shifting token end of file (1.5: )
 err: LAC: initial context discarded due to shift
-err: Entering state 19
+err: Entering state 14
-err: Stack now 0 8 19
+err: Stack now 0 7 14
-err: Stack now 0 8 19
+err: Stack now 0 7 14
 err: Cleanup: popping token end of file (1.5: )
 err: Cleanup: popping nterm input (1.1-4: )' -p
 

examples/d/local.mk

@@ -27,7 +27,7 @@ EXTRA_DIST += %D%/calc.test
 
 %D%/calc.d: %D%/calc.y $(dependencies)
 	$(AM_V_GEN)$(MKDIR_P) %D%
-	$(AM_V_at)$(BISON) $(srcdir)/%D%/calc.y -o $@
+	$(AM_V_at)$(BISON) -o $@ $(srcdir)/%D%/calc.y
 
 %D%/calc: %D%/calc.d
 	$(AM_V_GEN) $(DC) $(DCFLAGS) -of$@ %D%/calc.d

examples/java/calc/local.mk

@@ -27,7 +27,7 @@ EXTRA_DIST += %D%/Calc.test
 
 %D%/Calc.java: %D%/Calc.y $(dependencies)
 	$(AM_V_GEN)$(MKDIR_P) %D%
-	$(AM_V_at)$(BISON) $(srcdir)/%D%/Calc.y -o $@
+	$(AM_V_at)$(BISON) -o $@ $(srcdir)/%D%/Calc.y
 
 %D%/Calc.class: %D%/Calc.java
 	$(AM_V_GEN) $(SHELL) $(top_builddir)/javacomp.sh %D%/Calc.java

examples/java/simple/local.mk

@@ -27,7 +27,7 @@ EXTRA_DIST += %D%/Calc.test
 
 %D%/Calc.java: %D%/Calc.y $(dependencies)
 	$(AM_V_GEN)$(MKDIR_P) %D%
-	$(AM_V_at)$(BISON) $(srcdir)/%D%/Calc.y -o $@
+	$(AM_V_at)$(BISON) -o $@ $(srcdir)/%D%/Calc.y
 
 %D%/Calc.class: %D%/Calc.java
 	$(AM_V_GEN) $(SHELL) $(top_builddir)/javacomp.sh %D%/Calc.java

lib/.gitignore

@@ -336,9 +336,11 @@
 /sys_types.in.h
 /sys_wait.in.h
 /sysexits.in.h
+/termios.h
 /termios.in.h
 /textstyle.h
 /textstyle.in.h
+/thread-optim.h
 /time.h
 /time.in.h
 /timespec.c

m4/.gitignore

@@ -67,6 +67,7 @@
 /intlmacosx.m4
 /intmax.m4
 /intmax_t.m4
+/inttypes-pri.m4
 /inttypes.m4
 /inttypes_h.m4
 /isnan.m4
@@ -127,6 +128,7 @@
 /open.m4
 /pathmax.m4
 /perror.m4
+/pid_t.m4
 /pipe2.m4
 /po.m4
 /posix_spawn.m4

src/counterexample.c

@@ -712,10 +712,10 @@ ssb_equals (const search_state_bundle *s1, const search_state_bundle *s2)
 typedef gl_list_t ssb_list;
 
 static size_t
-visited_hasher (const search_state *ss, size_t maximum)
+visited_hasher (const search_state *ss, size_t max)
 {
-  return (parse_state_hasher (ss->states[0], maximum)
+  return (parse_state_hasher (ss->states[0], max)
-          + parse_state_hasher (ss->states[1], maximum)) % maximum;
+          + parse_state_hasher (ss->states[1], max)) % max;
 }
 
 static bool

src/location.c

@@ -155,7 +155,9 @@ int
 location_print (location loc, FILE *out)
 {
   int res = 0;
-  if (trace_flag & trace_locations)
+  if (location_empty (loc))
+    res += fprintf (out, "(empty location)");
+  else if (trace_flag & trace_locations)
     {
       res += boundary_print (&loc.start, out);
       res += fprintf (out, "-");

src/output.c

@@ -249,7 +249,7 @@ prepare_symbol_names (char const *muscle_name)
       if (i)
         obstack_1grow (&format_obstack, ' ');
       if (translatable)
-        obstack_sgrow (&format_obstack, "]b4_symbol_translate([");
+        obstack_sgrow (&format_obstack, "]b4_symbol_translate""([");
       obstack_escape (&format_obstack, cp);
       if (translatable)
         obstack_sgrow (&format_obstack, "])[");
@@ -554,7 +554,7 @@ prepare_symbol_definitions (void)
   /* Map "orig NUM" to new numbers.  See data/README.  */
   for (symbol_number i = ntokens; i < nsyms + nuseless_nonterminals; ++i)
     {
-      obstack_printf (&format_obstack, "symbol(orig %d, number)", i);
+      obstack_printf (&format_obstack, "symbol""(orig %d, number)", i);
       const char *key = obstack_finish0 (&format_obstack);
       MUSCLE_INSERT_INT (key, nterm_map ? nterm_map[i - ntokens] : i);
     }
@@ -565,12 +565,12 @@ prepare_symbol_definitions (void)
       const char *key;
 
 #define SET_KEY(Entry)                                          \
-      obstack_printf (&format_obstack, "symbol(%d, %s)",        \
+      obstack_printf (&format_obstack, "symbol""(%d, %s)",        \
                       i, Entry);                                \
       key = obstack_finish0 (&format_obstack);
 
 #define SET_KEY2(Entry, Suffix)                                 \
-      obstack_printf (&format_obstack, "symbol(%d, %s_%s)",     \
+      obstack_printf (&format_obstack, "symbol""(%d, %s_%s)",     \
                       i, Entry, Suffix);                        \
       key = obstack_finish0 (&format_obstack);
 

src/parse-gram.c

@@ -3147,8 +3147,7 @@ char_name (char c)
     }
 }
 
-static
+static void
-void
 current_lhs (symbol *sym, location loc, named_ref *ref)
 {
   current_lhs_symbol = sym;

src/parse-gram.y

@@ -1158,8 +1158,7 @@ char_name (char c)
     }
 }
 
-static
+static void
-void
 current_lhs (symbol *sym, location loc, named_ref *ref)
 {
   current_lhs_symbol = sym;

src/parse-simulation.c

@@ -28,7 +28,7 @@
 #include "lssi.h"
 #include "nullable.h"
 
-typedef struct parse_state
+struct parse_state
 {
   // Path of state-items the parser has traversed.
   struct si_chunk
@@ -58,7 +58,7 @@ typedef struct parse_state
   // Causes chunk contents to be freed when the reference count is
   // one. Used when only the chunk metadata will be needed.
   bool free_contents_early;
-} parse_state;
+};
 
 
 static void

src/reader.c

@@ -406,8 +406,8 @@ grammar_midrule_action (void)
      action.  Create the MIDRULE.  */
   location dummy_loc = current_rule->action_props.location;
   symbol *dummy = dummy_symbol_get (dummy_loc);
-  symbol_type_set(dummy,
+  symbol_type_set (dummy,
-                  current_rule->action_props.type, current_rule->action_props.location);
+                   current_rule->action_props.type, current_rule->action_props.location);
   symbol_list *midrule = symbol_list_sym_new (dummy, dummy_loc);
 
   /* Remember named_ref of previous action. */
@@ -815,8 +815,11 @@ check_and_convert_grammar (void)
     grammar = p;
   }
 
-  aver (nsyms <= SYMBOL_NUMBER_MAXIMUM);
+  if (SYMBOL_NUMBER_MAXIMUM - nnterms < ntokens)
-  aver (nsyms == ntokens + nnterms);
+    complain (NULL, fatal, "too many symbols in input grammar (limit is %d)",
+              SYMBOL_NUMBER_MAXIMUM);
+
+  nsyms = ntokens + nnterms;
 
   /* Assign the symbols their symbol numbers.  */
   symbols_pack ();

src/scan-gram.l

@@ -567,6 +567,8 @@ eqopt    ({sp}=)?
               _("POSIX Yacc does not support string literals"));
     RETURN_VALUE (STRING, last_string);
   }
+  <<EOF>>   unexpected_eof (token_start, "\"");
+  "\n"      unexpected_newline (token_start, "\"");
 }
 
 <SC_ESCAPED_TSTRING>
@@ -580,13 +582,10 @@ eqopt    ({sp}=)?
               _("POSIX Yacc does not support string literals"));
     RETURN_VALUE (TSTRING, last_string);
   }
+  <<EOF>>   unexpected_eof (token_start, "\")");
+  "\n"      unexpected_newline (token_start, "\")");
 }
 
-<SC_ESCAPED_STRING,SC_ESCAPED_TSTRING>
-{
-  <<EOF>>   unexpected_eof (token_start, "\"");
-  "\n"      unexpected_newline (token_start, "\"");
-}
 
 
   /*----------------------------------------------------------.
@@ -692,6 +691,15 @@ eqopt    ({sp}=)?
               p);
     STRING_1GROW ('?');
   }
+
+  "\\"             {
+    // None of the other rules matched: the last character of this
+    // file is "\".  But Flex does not support "\\<<EOF>>".
+    unexpected_eof (token_start,
+                    YY_START == SC_ESCAPED_CHARACTER ? "?'"
+                    : YY_START == SC_ESCAPED_STRING ? "?\""
+                    : "?\")");
+  }
 }
 
   /*--------------------------------------------.

src/symtab.c

@@ -137,11 +137,6 @@ symbol_new (uniqstr tag, location loc)
   res->alias = NULL;
   res->content = sym_content_new (res);
   res->is_alias = false;
-
-  if (nsyms == SYMBOL_NUMBER_MAXIMUM)
-    complain (NULL, fatal, _("too many symbols in input grammar (limit is %d)"),
-              SYMBOL_NUMBER_MAXIMUM);
-  nsyms++;
   return res;
 }
 
@@ -182,11 +177,11 @@ symbol_free (void *ptr)
 */
 
 static void
-symbols_sort (symbol **first, symbol **second)
+symbols_sort (const symbol **first, const symbol **second)
 {
   if (0 < location_cmp ((*first)->location, (*second)->location))
     {
-      symbol* tmp = *first;
+      const symbol* tmp = *first;
       *first = *second;
       *second = tmp;
     }
@@ -243,7 +238,11 @@ semantic_type_new (uniqstr tag, const location *loc)
 | Print a symbol.  |
 `-----------------*/
 
-#define SYMBOL_ATTR_PRINT(Attr)                         \
+#define SYMBOL_INT_ATTR_PRINT(Attr)                     \
+  if (s->content)                                       \
+    fprintf (f, " %s = %d", #Attr, s->content->Attr)
+
+#define SYMBOL_STR_ATTR_PRINT(Attr)                     \
   if (s->content && s->content->Attr)                   \
     fprintf (f, " %s { %s }", #Attr, s->content->Attr)
 
@@ -264,7 +263,11 @@ symbol_print (symbol const *s, FILE *f)
                : c == nterm_sym    ? "nterm"
                : NULL, /* abort.  */
                s->tag);
-      SYMBOL_ATTR_PRINT (type_name);
+      putc (' ', f);
+      location_print (s->location, f);
+      SYMBOL_INT_ATTR_PRINT (code);
+      SYMBOL_INT_ATTR_PRINT (number);
+      SYMBOL_STR_ATTR_PRINT (type_name);
       SYMBOL_CODE_PRINT (destructor);
       SYMBOL_CODE_PRINT (printer);
     }
@@ -371,7 +374,7 @@ symbol_from_uniqstr_fuzzy (const uniqstr key)
 }
 
 static void
-complain_symbol_undeclared (symbol *sym)
+complain_symbol_undeclared (const symbol *sym)
 {
   assert (sym->content->status != declared);
   const symbol *best = symbol_from_uniqstr_fuzzy (sym->tag);
@@ -398,7 +401,10 @@ void
 symbol_location_as_lhs_set (symbol *sym, location loc)
 {
   if (!sym->location_of_lhs)
-    sym->location = loc;
+    {
+      sym->location = loc;
+      sym->location_of_lhs = true;
+    }
 }
 
 
@@ -548,10 +554,6 @@ symbol_class_set (symbol *sym, symbol_class class, location loc, bool declaring)
       if (class == token_sym && s->class == pct_type_sym)
         complain_pct_type_on_token (&sym->location);
 
-      if (class == nterm_sym && s->class != nterm_sym)
-        s->number = nnterms++;
-      else if (class == token_sym && s->number == NUMBER_UNDEFINED)
-        s->number = ntokens++;
       s->class = class;
 
       if (declaring)
@@ -573,9 +575,9 @@ symbol_class_set (symbol *sym, symbol_class class, location loc, bool declaring)
 }
 
 
-/*------------------------------------------------.
+/*----------------------------.
-| Set the USER_TOKEN_NUMBER associated with SYM.  |
+| Set the token code of SYM.  |
-`------------------------------------------------*/
+`----------------------------*/
 
 void
 symbol_code_set (symbol *sym, int code, location loc)
@@ -598,10 +600,6 @@ symbol_code_set (symbol *sym, int code, location loc)
       if (code == 0 && !eoftoken)
         {
           eoftoken = sym->content->symbol;
-          /* It is always mapped to 0, so it was already counted in
-             NTOKENS.  */
-          if (eoftoken->content->number != NUMBER_UNDEFINED)
-            --ntokens;
           eoftoken->content->number = 0;
         }
     }
@@ -621,9 +619,11 @@ symbol_check_defined (symbol *sym)
     {
       complain_symbol_undeclared (sym);
       s->class = nterm_sym;
-      s->number = nnterms++;
     }
 
+  if (s->number == NUMBER_UNDEFINED)
+    s->number = s->class == token_sym ? ntokens++ : nnterms++;
+
   if (s->class == token_sym
       && sym->tag[0] == '"'
       && !sym->is_alias)
@@ -742,7 +742,7 @@ symbol_pack (symbol *sym)
 }
 
 static void
-complain_code_redeclared (int num, symbol *first, symbol *second)
+complain_code_redeclared (int num, const symbol *first, const symbol *second)
 {
   symbols_sort (&first, &second);
   complain (&second->location, complaint,
@@ -758,13 +758,11 @@ complain_code_redeclared (int num, symbol *first, symbol *second)
 `-------------------------------------------------*/
 
 static void
-symbol_translation (symbol *sym)
+symbol_translation (const symbol *sym)
 {
-  /* Nonterminal? */
+  if (sym->content->class == token_sym && !sym->is_alias)
-  if (sym->content->class == token_sym
-      && !sym->is_alias)
     {
-      /* A token which translation has already been set?*/
+      /* A token whose translation has already been set? */
       if (token_translations[sym->content->code]
           != undeftoken->content->number)
         complain_code_redeclared
@@ -969,7 +967,6 @@ dummy_symbol_get (location loc)
   assure (len < sizeof buf);
   symbol *sym = symbol_get (buf, loc);
   sym->content->class = nterm_sym;
-  sym->content->number = nnterms++;
   return sym;
 }
 
@@ -1002,7 +999,7 @@ symbol_cmp (void const *a, void const *b)
 }
 
 /* Store in *SORTED an array of pointers to the symbols contained in
-   TABLE, sorted (alphabetically) by tag. */
+   TABLE, sorted by order of appearance (i.e., by location). */
 
 static void
 table_sort (struct hash_table *table, symbol ***sorted)

src/symtab.h

@@ -227,7 +227,7 @@ void symbol_precedence_set (symbol *sym, int prec, assoc a, location loc);
 void symbol_class_set (symbol *sym, symbol_class class, location loc,
                        bool declaring);
 
-/** Set the \c code associated with \c sym.  */
+/** Set the token \c code of \c sym, specified by the user at \c loc.  */
 void symbol_code_set (symbol *sym, int code, location loc);
 
 

tests/calc.at

@@ -915,6 +915,12 @@ AT_BISON_OPTION_PUSHDEFS([$1])
 
 AT_DATA_CALC_Y([$1])
 AT_FULL_COMPILE(AT_JAVA_IF([[Calc]], [[calc]]), AT_DEFINES_IF([[lex], [main]], [[], []]), [$2], [-Wno-deprecated])
+
+AT_YACC_IF(
+  [# No direct calls to malloc/free.
+  AT_CHECK([[$EGREP '(malloc|free) *\(' calc.[ch] | $EGREP -v 'INFRINGES ON USER NAME SPACE']],
+           [1])])
+
 AT_PUSH_IF([AT_JAVA_IF(
  [# Verify that this is a push parser.
   AT_CHECK_JAVA_GREP([[Calc.java]],

tests/input.at

@@ -135,6 +135,10 @@ input.y:9.1-10.0: error: missing '%}' at end of file
 AT_CLEANUP
 
 
+## ------------------------ ##
+## Invalid inputs with {}.  ##
+## ------------------------ ##
+
 AT_SETUP([Invalid inputs with {}])
 
 # We used to SEGV here.  See
@@ -816,6 +820,33 @@ input.y:3.8-10: note: previous declaration
 AT_CLEANUP
 
 
+## ---------------- ##
+## EOF redeclared.  ##
+## ---------------- ##
+
+AT_SETUP([EOF redeclared])
+
+# We used to crash when redefining a token after having defined EOF.
+# See https://lists.gnu.org/r/bug-bison/2020-08/msg00008.html.
+
+AT_DATA([[input.y]],
+[[%token FOO BAR FOO 0
+%%
+input: %empty
+]])
+
+AT_BISON_CHECK([-fcaret input.y], [0], [],
+[[input.y:1.16-18: warning: symbol FOO redeclared [-Wother]
+    1 | %token FOO BAR FOO 0
+      |                ^~~
+input.y:1.8-10: note: previous declaration
+    1 | %token FOO BAR FOO 0
+      |        ^~~
+]])
+
+AT_CLEANUP
+
+
 ## --------------------------- ##
 ## Symbol class redefinition.  ##
 ## --------------------------- ##
@@ -1364,11 +1395,6 @@ AT_CLEANUP
 AT_SETUP([Torturing the Scanner])
 
 AT_BISON_OPTION_PUSHDEFS
-AT_DATA([input.y], [])
-AT_BISON_CHECK([input.y], [1], [],
-[[input.y:1.1: error: unexpected end of file
-]])
-
 
 AT_DATA([input.y],
 [{}
@@ -2475,6 +2501,99 @@ input.y:5.19: error: invalid character after \-escape: \001
 AT_CLEANUP
 
 
+## ------------------------ ##
+## Unexpected end of file.  ##
+## ------------------------ ##
+
+AT_SETUP([[Unexpected end of file]])
+
+
+AT_DATA([input.y], [])
+AT_BISON_CHECK([-fcaret input.y], [1], [],
+[[input.y:1.1: error: unexpected end of file
+]])
+
+
+AT_DATA_NO_FINAL_EOL([char.y],
+[[%token FOO ']])
+
+AT_BISON_CHECK([-fcaret char.y], [1], [],
+[[char.y:1.12: error: missing "'" at end of file
+    1 | %token FOO '
+      |            ^
+char.y:1.12: error: empty character literal
+    1 | %token FOO '
+      |            ^
+]])
+
+
+AT_DATA_NO_FINAL_EOL([escape-in-char.y],
+[[%token FOO '\]])
+
+AT_BISON_CHECK([-fcaret escape-in-char.y], [1], [],
+[[escape-in-char.y:1.12-13: error: missing '?\'' at end of file
+    1 | %token FOO '\
+      |            ^~
+escape-in-char.y:1.14: error: unexpected end of file
+    1 | %token FOO '\
+      |              ^
+]])
+
+
+AT_DATA_NO_FINAL_EOL([string.y],
+[[%token FOO "]])
+
+AT_BISON_CHECK([-fcaret string.y], [1], [],
+[[string.y:1.12: error: missing '"' at end of file
+    1 | %token FOO "
+      |            ^
+string.y:1.13: error: unexpected end of file
+    1 | %token FOO "
+      |             ^
+]])
+
+
+AT_DATA_NO_FINAL_EOL([escape-in-string.y],
+[[%token FOO "\]])
+
+AT_BISON_CHECK([-fcaret escape-in-string.y], [1], [],
+[[escape-in-string.y:1.12-13: error: missing '?"' at end of file
+    1 | %token FOO "\
+      |            ^~
+escape-in-string.y:1.14: error: unexpected end of file
+    1 | %token FOO "\
+      |              ^
+]])
+
+
+AT_DATA_NO_FINAL_EOL([tstring.y],
+[[%token FOO _("]])
+
+AT_BISON_CHECK([-fcaret tstring.y], [1], [],
+[[tstring.y:1.12-14: error: missing '")' at end of file
+    1 | %token FOO _("
+      |            ^~~
+tstring.y:1.15: error: unexpected end of file
+    1 | %token FOO _("
+      |               ^
+]])
+
+
+AT_DATA_NO_FINAL_EOL([escape-in-tstring.y],
+[[%token FOO _("\]])
+
+AT_BISON_CHECK([-fcaret escape-in-tstring.y], [1], [],
+[[escape-in-tstring.y:1.12-15: error: missing '?")' at end of file
+    1 | %token FOO _("\
+      |            ^~~~
+escape-in-tstring.y:1.16: error: unexpected end of file
+    1 | %token FOO _("\
+      |                ^
+]])
+
+AT_CLEANUP
+
+
 ## ------------------------- ##
 ## LAC: Errors for %define.  ##
 ## ------------------------- ##

tests/local.at

@@ -390,6 +390,7 @@ AT_LOCATION_TYPE_SPAN_IF(
 
 
 AT_GLR_IF([AT_KEYWORDS([glr])])
+AT_PUSH_IF([AT_KEYWORDS([push])])
 ])# _AT_BISON_OPTION_PUSHDEFS
 
 

tests/output.at

@@ -759,7 +759,7 @@ AT_TEST([x1],
          ])
 
 # Check the CPP guard and Doxyen comments.
-AT_CHECK([sed -ne 's/#line [0-9]\+ "/#line "/p;/INCLUDED/p;/\\file/{p;n;p;}' out/include/ast/loc.hh], [],
+AT_CHECK([[sed -ne 's/#line [0-9][0-9]* "/#line "/p;/INCLUDED/p;/\\file/{p;n;p;}' out/include/ast/loc.hh]], [],
 [[ ** \file bar/include/ast/loc.hh
  ** Define the x1::location class.
 #ifndef YY_YY_BAR_INCLUDE_AST_LOC_HH_INCLUDED
@@ -771,7 +771,7 @@ AT_CHECK([sed -ne 's/#line [0-9]\+ "/#line "/p;/INCLUDED/p;/\\file/{p;n;p;}' out
 #endif // !YY_YY_BAR_INCLUDE_AST_LOC_HH_INCLUDED
 ]])
 
-AT_CHECK([sed -ne 's/^#line [0-9]\+ "/#line "/p;/INCLUDED/p;/\\file/{p;n;p;}' out/x1.hh], [],
+AT_CHECK([[sed -ne 's/^#line [0-9][0-9]* "/#line "/p;/INCLUDED/p;/\\file/{p;n;p;}' out/x1.hh]], [],
 [[ ** \file bar/x1.hh
  ** Define the x1::parser class.
 #ifndef YY_YY_BAR_X1_HH_INCLUDED