Merge 3.7.6 into master

* maint:
  maint: post-release administrivia
  version 3.7.6
  yacc: fix push parser
  tables: fix again the handling of useless tokens

.prev-version

@@ -1 +1 @@
-3.7.4
+3.7.6

NEWS

@@ -97,6 +97,21 @@ GNU Bison NEWS
   Users may define `YYLOCATION_PRINT` to cover other cases.
 
 
+* Noteworthy changes in release 3.7.6 (2021-03-08) [stable]
+
+** Bug fixes
+
+*** Reused Push Parsers
+
+  When a push-parser state structure is used for multiple parses, it was
+  possible for some state to leak from one run into the following one.
+
+*** Fix Table Generation
+
+  In some very rare conditions, when there are many useless tokens, it was
+  possible to generate incorrect parsers.
+
+
 * Noteworthy changes in release 3.7.5 (2021-01-24) [stable]
 
 ** Bug fixes
@@ -380,7 +395,7 @@ GNU Bison NEWS
   parse errors, since `yynerrs` was also reset.  This can be especially
   troublesome when used in autocompletion, since a parser with error
   recovery would suggest (irrelevant) expected tokens even if there were
-  failure.
+  failures.
 
   Now the parser state can be examined when parsing is finished.  The parser
   state is reset when starting a new parse.

data/skeletons/yacc.c

@@ -1650,14 +1650,16 @@ yyparse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)]])]
 
   switch (yyps->yynew)
     {
-    case 2:
-      yypstate_clear (yyps);
-      goto case_0;
-
-    case_0:
     case 0:
       yyn = yypact[yystate];
       goto yyread_pushed_token;
+
+    case 2:
+      yypstate_clear (yyps);
+      break;
+
+    default:
+      break;
     }]])[
 
   YYDPRINTF ((stderr, "Starting parse\n"));

src/tables.c

@@ -180,21 +180,26 @@ pos_set_set (int pos)
   int bitno = pos - pos_set_base;
   if (bitno < 0)
     {
+      // Need more room on the left.
+      // DELTA is positive.  Run 'pos_set >> delta'.
       const int delta = pos_set_base - pos;
       const int old_size = bitset_size (pos_set);
       const int new_size = old_size + delta;
       bitset_resize (pos_set, new_size);
-      // Shift all the bits by DELTA.
+      // Right-shift all the bits by DELTA.  Be sure to reset the new
+      // bits on the left.
+      //
       // FIXME: add bitset_assign, and bitset_shift?
-      for (int i = new_size - 1; delta <= i ; --i)
-        if (bitset_test (pos_set, i))
-          bitset_set (pos_set, i + delta);
+      for (int i = new_size - 1; 0 <= i ; --i)
+        if (delta <= i && bitset_test (pos_set, i - delta))
+          bitset_set (pos_set, i);
         else
-          bitset_reset (pos_set, i + delta);
+          bitset_reset (pos_set, i);
       pos_set_base = pos;
       bitno = 0;
     }
   else if (bitset_size (pos_set) <= bitno)
+    // Need more room on the right.
     bitset_resize (pos_set, bitno + 1);
   bitset_set (pos_set, bitno);
 }

tests/push.at

@@ -25,7 +25,7 @@ AT_BANNER([[Push Parsing Tests]])
 AT_SETUP([[Memory Leak for Early Deletion]])
 
 # Requires Valgrind.
-AT_BISON_OPTION_PUSHDEFS
+AT_BISON_OPTION_PUSHDEFS([%define api.push-pull push])
 AT_DATA_GRAMMAR([[input.y]],
 [[
 %{
@@ -144,7 +144,7 @@ AT_CLEANUP
 
 AT_SETUP([[Unsupported Skeletons]])
 
-AT_BISON_OPTION_PUSHDEFS
+AT_BISON_OPTION_PUSHDEFS([%define api.push-pull push])
 AT_DATA([[input.y]],
 [[%glr-parser
 %define api.push-pull push
@@ -158,3 +158,126 @@ AT_BISON_CHECK([[input.y]], [[1]], [],
 ]])
 
 AT_CLEANUP
+
+
+## -------------- ##
+## Pstate reuse.  ##
+## -------------- ##
+
+AT_SETUP([[Pstate reuse]])
+
+# Make sure that when a single pstate is used for multiple successive
+# parses, no state from a previous run leaks into the following one.
+#
+# See https://lists.gnu.org/r/bug-bison/2021-03/msg00000.html.
+
+AT_BISON_OPTION_PUSHDEFS([%define api.push-pull push])
+AT_DATA_GRAMMAR([[input.y]],
+[[%code top {
+  #include <stdlib.h>
+  #include <string.h>
+
+  static char *string_concat (char *a, char *b);
+  ]AT_YYERROR_DECLARE[
+}
+
+%define parse.trace
+%define api.pure full
+%define api.push-pull push
+%expect 0
+
+%union {
+  char *sval;
+};
+%destructor { free ($$); } <sval>
+%printer { fprintf (yyo, "%s", $$); } <sval>
+
+%token <sval> RAW
+%token EOL
+
+%type <sval> text
+
+%%
+
+line
+  : text EOL  { printf ("text: %s\n", $1); free ($1); YYACCEPT; };
+
+text
+  : RAW       { $$ = $1; }
+  | text RAW  { $$ = string_concat ($1, $2); }
+  ;
+
+%%
+]AT_YYERROR_DEFINE[
+
+static char *
+string_concat (char *a, char *b)
+{
+  size_t la = strlen (a);
+  size_t lb = strlen (b);
+  char *res = YY_CAST (char *, malloc (la + lb + 1));
+  strcpy (res, a);
+  strcpy (res + la, b);
+  free (a);
+  free (b);
+  return res;
+}
+
+static int
+push (yypstate *ps, yytoken_kind_t kind, const char *str)
+{
+  YYSTYPE lval;
+  lval.sval = str ? strdup (str) :  YY_NULLPTR;
+  switch (yypush_parse (ps, kind, &lval))
+    {
+    case 0:
+      return 0;
+    case YYPUSH_MORE:
+      // parsing incomplete, but valid; parser not reset
+      return 0;
+    case 1:
+      // YYABORT or syntax invalid; parser is reset
+      fprintf (stderr, "invalid input, but no error was thrown\n");
+      return 1;
+    case 2:
+      // memory exhaustion; parser is reset
+      fprintf (stderr, "memory exhaustion during yypush_parse\n");
+      return 1;
+    }
+  return 1;
+}
+
+int
+main (void)
+{
+  yydebug = !!getenv ("YYDEBUG");
+  yypstate *ps = yypstate_new ();
+
+#define PUSH(Kind, Val)                         \
+  do {                                          \
+    if (push (ps, Kind, Val))                   \
+      return 1;                                 \
+  } while (0)
+
+  PUSH (RAW, "te");
+  PUSH (RAW, "xt");
+  PUSH (EOL,  YY_NULLPTR);
+
+  PUSH (RAW, "te");
+  PUSH (RAW, "xt");
+  PUSH (EOL,  YY_NULLPTR);
+
+  yypstate_delete (ps);
+
+  return 0;
+}
+]])
+
+AT_FULL_COMPILE([input])
+AT_CHECK([./input], 0,
+[[text: text
+text: text
+]])
+
+AT_BISON_OPTION_POPDEFS
+AT_CLEANUP