yacc.c: push: initialize the pstate variables in pstate_new

Currently pstate_new does not set up its variables, this task is left
to yypush_parse.  This was probably to share more code with usual pull
parsers, where these (local) variables are indeed initialized by
yyparse.

But as a consequence yyexpected_tokens crashes at the very beginning
of the parse, since, for instance, the stacks are not even set up.
See https://lists.gnu.org/r/bison-patches/2020-03/msg00001.html.

The fix could have very simple, but the documentation actually makes
it very clear that we can reuse a pstate for several parses:

    After yypush_parse returns a status other than YYPUSH_MORE, the
    parser instance yyps may be reused for a new parse.

so we need to restore the parser to its pristine state so that (i) it
is ready to run the next parse, (ii) it properly supports
yyexpected_tokens for the next run.

* data/skeletons/yacc.c (b4_initialize_parser_state_variables): New,
extracted from the top of yyparse/yypush_parse.
(yypstate_clear): New.
(yypstate_new): Use it when push parsers are enabled.
Define after the yyps macros so that we can use the same code as the
regular pull parsers.
(yyparse): Use it when push parsers are _not_ enabled.

* examples/c/bistromathic/bistromathic.test: Check the completion on
the beginning of the line.

TODO

@@ -95,6 +95,9 @@ See also the item "$undefined" below.
 Consider deprecating impure push parsers.  They add a lot of complexity, for
 a bad feature.
 
+It's not clear to me that yyerror_range really needs to be in pstate,
+instead of just being a local variable.
+
 * Bison 3.7
 ** Unit rules / Injection rules (Akim Demaille)
 Maybe we could expand unit rules (or "injections", see

data/skeletons/yacc.c

@@ -195,6 +195,9 @@ m4_define([b4_declare_parser_state_variables],
        Refer to the stacks through separate pointers, to allow yyoverflow
        to reallocate them elsewhere.  */
 
+    /* Their size.  */
+    YYPTRDIFF_T yystacksize;
+
     /* The state stack.  */
     yy_state_t yyssa[YYINITDEPTH];
     yy_state_t *yyss;
@@ -211,15 +214,34 @@ m4_define([b4_declare_parser_state_variables],
     YYLTYPE *yylsp;
 
     /* The locations where the error started and ended.  */
-    YYLTYPE yyerror_range[3];]])[
+    YYLTYPE yyerror_range[3];]])[]b4_lac_if([[
-
-    YYPTRDIFF_T yystacksize;]b4_lac_if([[
 
     yy_state_t yyesa@{]b4_percent_define_get([[parse.lac.es-capacity-initial]])[@};
     yy_state_t *yyes;
     YYPTRDIFF_T yyes_capacity;]])])
 
 
+# b4_initialize_parser_state_variables
+# ------------------------------------
+# Initialize these variables.
+m4_define([b4_initialize_parser_state_variables],
+[[  yynerrs = 0;
+  yystate = 0;
+  yyerrstatus = 0;
+
+  yystacksize = YYINITDEPTH;
+  yyssp = yyss = yyssa;
+  yyvsp = yyvs = yyvsa;]b4_locations_if([[
+  yylsp = yyls = yylsa;]])[]b4_lac_if([[
+
+  yyes = yyesa;
+  yyes_capacity = ]b4_percent_define_get([[parse.lac.es-capacity-initial]])[;
+  if (YYMAXDEPTH < yyes_capacity)
+    yyes_capacity = YYMAXDEPTH;]])[
+]])
+
+
+
 # _b4_declare_yyparse_push
 # ------------------------
 # Declaration of yyparse (and dependencies) when using the push parser
@@ -328,6 +350,7 @@ m4_if(b4_api_prefix, [yy], [],
 #define yypush_parse    ]b4_prefix[push_parse]b4_pull_if([[
 #define yypull_parse    ]b4_prefix[pull_parse]])[
 #define yypstate_new    ]b4_prefix[pstate_new
+#define yypstate_clear  ]b4_prefix[pstate_clear
 #define yypstate_delete ]b4_prefix[pstate_delete
 #define yypstate        ]b4_prefix[pstate]])[
 #define yylex           ]b4_prefix[lex
@@ -1382,8 +1405,7 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
 ]b4_push_if([[
 struct yypstate
   {]b4_declare_parser_state_variables[
-    /* Used to determine if this is the first time this instance has
+    /* Whether this instance has not started parsing yet.  */
-       been used.  */
     int yynew;
   };]b4_pure_if([], [[
 
@@ -1425,40 +1447,7 @@ yypull_parse (yypstate *yyps]b4_user_formals[)
   return yystatus;
 }]])[
 
-/* Initialize the parser data structure.  */
+]b4_pure_if([[#define ]b4_prefix[nerrs yyps->]b4_prefix[nerrs]])[
-yypstate *
-yypstate_new (void)
-{
-  yypstate *yyps;]b4_pure_if([], [[
-  if (yypstate_allocated)
-    return YY_NULLPTR;]])[
-  yyps = YY_CAST (yypstate *, malloc (sizeof *yyps));
-  if (!yyps)
-    return YY_NULLPTR;
-  yyps->yynew = 1;]b4_pure_if([], [[
-  yypstate_allocated = 1;]])[
-  return yyps;
-}
-
-void
-yypstate_delete (yypstate *yyps)
-{
-  if (yyps)
-    {
-#ifndef yyoverflow
-      /* If the stack was reallocated but the parse did not complete, then the
-         stack still needs to be freed.  */
-      if (!yyps->yynew && yyps->yyss != yyps->yyssa)
-        YYSTACK_FREE (yyps->yyss);
-#endif]b4_lac_if([[
-      if (!yyps->yynew && yyps->yyes != yyps->yyesa)
-        YYSTACK_FREE (yyps->yyes);]])[
-      free (yyps);]b4_pure_if([], [[
-      yypstate_allocated = 0;]])[
-    }
-}
-]b4_pure_if([[
-#define ]b4_prefix[nerrs yyps->]b4_prefix[nerrs]])[
 #define yystate yyps->yystate
 #define yyerrstatus yyps->yyerrstatus
 #define yyssa yyps->yyssa
@@ -1475,6 +1464,50 @@ yypstate_delete (yypstate *yyps)
 #define yyesa yyps->yyesa
 #define yyes yyps->yyes
 #define yyes_capacity yyps->yyes_capacity]])[
+
+/* Initialize the parser data structure.  */
+void
+yypstate_clear (yypstate *yyps)
+{
+]b4_initialize_parser_state_variables[
+  /* Initialize the state stack, in case yyexpected_tokens is called
+     before the first call to yyparse. */
+  *yyssp = 0;
+}
+
+/* Initialize the parser data structure.  */
+yypstate *
+yypstate_new (void)
+{
+  yypstate *yyps;]b4_pure_if([], [[
+  if (yypstate_allocated)
+    return YY_NULLPTR;]])[
+  yyps = YY_CAST (yypstate *, malloc (sizeof *yyps));
+  if (!yyps)
+    return YY_NULLPTR;
+  yyps->yynew = 1;]b4_pure_if([], [[
+  yypstate_allocated = 1;]])[
+  yypstate_clear (yyps);
+  return yyps;
+}
+
+void
+yypstate_delete (yypstate *yyps)
+{
+  if (yyps)
+    {
+#ifndef yyoverflow
+      /* If the stack was reallocated but the parse did not complete, then the
+         stack still needs to be freed.  */
+      if (!yyps->yynew && yyss != yyssa)
+        YYSTACK_FREE (yyss);
+#endif]b4_lac_if([[
+      if (!yyps->yynew && yyes != yyesa)
+        YYSTACK_FREE (yyes);]])[
+      free (yyps);]b4_pure_if([], [[
+      yypstate_allocated = 0;]])[
+    }
+}
 ]])[
 
 ]b4_push_if([[
@@ -1522,29 +1555,17 @@ yyparse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)]])[
 
   /* The number of symbols on the RHS of the reduced rule.
      Keep to zero when no symbol should be popped.  */
-  int yylen = 0;]b4_push_if([[
+  int yylen = 0;
-
+]b4_push_if([[
   if (!yyps->yynew)
     {
       yyn = yypact[yystate];
       goto yyread_pushed_token;
-    }]])[
+    }]], [
-
+b4_initialize_parser_state_variables])[
-  yyssp = yyss = yyssa;
-  yyvsp = yyvs = yyvsa;]b4_locations_if([[
-  yylsp = yyls = yylsa;]])[
-  yystacksize = YYINITDEPTH;]b4_lac_if([[
-
-  yyes = yyesa;
-  yyes_capacity = ]b4_percent_define_get([[parse.lac.es-capacity-initial]])[;
-  if (YYMAXDEPTH < yyes_capacity)
-    yyes_capacity = YYMAXDEPTH;]])[
 
   YYDPRINTF ((stderr, "Starting parse\n"));
 
-  yystate = 0;
-  yyerrstatus = 0;
-  yynerrs = 0;
   yychar = YYEMPTY; /* Cause a token to be read.  */
 ]m4_ifdef([b4_initial_action], [
 b4_dollar_pushdef([m4_define([b4_dollar_dollar_used])yylval], [], [],
@@ -2033,6 +2054,7 @@ yyreturn:
 #endif]b4_lac_if([[
   if (yyes != yyesa)
     YYSTACK_FREE (yyes);]])b4_push_if([[
+  yypstate_clear (yyps);
   yyps->yynew = 1;
 
 

examples/c/bistromathic/bistromathic.test

@@ -102,3 +102,13 @@ sed -e 's/\\t/	/g' >input <<EOF
 EOF
 run 0 '> (atan ( ''
 > err: 1.9: syntax error: expected - or ( or double precision number or function or variable before end of file'
+
+# Check the completion at the very beginning.
+sed -e 's/\\t/	/g' >input <<EOF
+e\t\t
+EOF
+run 0 '> e
+end of file  exit         exp          ''
+> e
+0
+> err: '

examples/c/bistromathic/local.mk

@@ -23,7 +23,7 @@ check_PROGRAMS += %D%/bistromathic
 TESTS += %D%/bistromathic.test
 EXTRA_DIST += %D%/bistromathic.test
 nodist_%C%_bistromathic_SOURCES = %D%/parse.y %D%/parse.h
-%D%/calc.c: $(dependencies)
+%D%/parse.c: $(dependencies)
 
 # Don't use gnulib's system headers.
 %C%_bistromathic_CPPFLAGS = -I$(top_srcdir)/%D% -I$(top_builddir)/%D%