yacc.c: yypstate_expected_tokens

In push parsers, when asking for the list of expected tokens at some
point, it makes no sense to build a yyparse_context_t: the yypstate
alone suffices (the only difference being the lookahead).  Instead of
forcing the user to build a useless shell around yypstate, let's offer
yypstate_expected_tokens.

See https://lists.gnu.org/r/bison-patches/2020-03/msg00025.html.

* data/skeletons/yacc.c (yypstate): Declare earlier, so that we can
use it for...
(yypstate_expected_tokens): this new function, when in push parsers.
Adjust dependencies.
* examples/c/bistromathic/parse.y: Simplify: use
yypstate_expected_tokens.
Style fixes.
Reduce scopes (reported by Joel E. Denny).

NEWS

@@ -75,6 +75,16 @@ GNU Bison NEWS
   the translated symbol (i.e., it returns '_("variable")' rather that
   '"variable"').
 
+*** List of expected tokens (yacc.c)
+
+  At any point during parsing (including even before submitting the first
+  token), push parsers may now invoke yypstate_expected_tokens to get the
+  list of possible tokens.  This feature can be used to propose
+  autocompletion (see below the "bistromathic" example).
+
+  It makes little sense to use this feature without enabling LAC (lookahead
+  correction).
+
 *** Modernize display of explanatory statements in diagnostics
 
   Since Bison 2.7, output was indented four spaces for explanatory
@@ -106,9 +116,9 @@ GNU Bison NEWS
 
   A new C example, bistromathic, is a fully featured interactive calculator
   using many Bison features: pure interface, push parser, autocompletion
-  based on the current parser state (using yyexpected_tokens), location
+  based on the current parser state (using yypstate_expected_tokens),
-  tracking, internationalized custom error messages, lookahead-correction,
+  location tracking, internationalized custom error messages, lookahead
-  rich debug traces, etc.
+  correction, rich debug traces, etc.
 
 * Noteworthy changes in release 3.5.3 (2020-03-08) [stable]
 

data/skeletons/yacc.c

@@ -856,7 +856,17 @@ int yydebug;
 
 #ifndef YYMAXDEPTH
 # define YYMAXDEPTH ]b4_stack_depth_max[
-#endif]b4_lac_if([[
+#endif]b4_push_if([[
+/* Parser data structure.  */
+struct yypstate
+  {]b4_declare_parser_state_variables[
+    /* Whether this instance has not started parsing yet.  */
+    int yynew;
+  };]b4_pure_if([], [[
+
+/* Whether the only allowed instance of yypstate is allocated.  */
+static char yypstate_allocated = 0;]])])[
+]b4_lac_if([[
 
 /* Given a state stack such that *YYBOTTOM is its bottom, such that
    *YYTOP is either its top or is YYTOP_EMPTY to indicate an empty
@@ -1108,23 +1118,28 @@ yy_lac (yy_state_t *yyesa, yy_state_t **yyes,
 }]])[
 
 ]b4_parse_error_case([simple], [],
-[[typedef struct
+[[/* Context of a parse error.  */
-{
+typedef struct
-  yy_state_t *yyssp;
+{]b4_push_if([[
-  int yytoken;]b4_locations_if([[
+  yypstate* yyps;]], [[
-  YYLTYPE *yylloc;]])[]b4_lac_if([[
+  yy_state_t *yyssp;]b4_lac_if([[
   yy_state_t *yyesa;
   yy_state_t **yyes;
-  YYPTRDIFF_T *yyes_capacity;]])[
+  YYPTRDIFF_T *yyes_capacity;]])])[
+  int yytoken;]b4_locations_if([[
+  YYLTYPE *yylloc;]])[
 } yyparse_context_t;
 
 /* Put in YYARG at most YYARGN of the expected tokens given the
    current YYCTX, and return the number of tokens stored in YYARG.  If
    YYARG is null, return the number of expected tokens (guaranteed to
-   be less than YYNTOKENS).  */
+   be less than YYNTOKENS).  */]b4_push_if([[
+static int
+yypstate_expected_tokens (yypstate *yyps,
+                          int yyarg[], int yyargn)]], [[
 static int
 yyexpected_tokens (const yyparse_context_t *yyctx,
-                   int yyarg[], int yyargn)
+                   int yyarg[], int yyargn)]])[
 {
   /* Actual size of YYARG. */
   int yycount = 0;
@@ -1134,8 +1149,9 @@ yyexpected_tokens (const yyparse_context_t *yyctx,
     if (yyx != YYTERROR && yyx != YYUNDEFTOK)
       {
         {
-          int yy_lac_status = yy_lac (yyctx->yyesa, yyctx->yyes, yyctx->yyes_capacity,
+          int yy_lac_status
-                                      yyctx->yyssp, yyx);
+            = yy_lac (]b4_push_if([[yyps->yyesa, &yyps->yyes, &yyps->yyes_capacity, yyps->yyssp, yyx]],
+                                  [[yyctx->yyesa, yyctx->yyes, yyctx->yyes_capacity, yyctx->yyssp, yyx]])[);
           if (yy_lac_status == 2)
             return -2;
           if (yy_lac_status == 1)
@@ -1148,7 +1164,7 @@ yyexpected_tokens (const yyparse_context_t *yyctx,
         else
           yyarg[yycount++] = yyx;
       }]],
-[[  int yyn = yypact[+*yyctx->yyssp];
+[[  int yyn = yypact@{+*]b4_push_if([yyps], [yyctx])[->yyssp@};
   if (!yypact_value_is_default (yyn))
     {
       /* Start YYX at -YYN if negative to avoid negative indexes in
@@ -1174,6 +1190,15 @@ yyexpected_tokens (const yyparse_context_t *yyctx,
   return yycount;
 }
 
+]b4_push_if([[
+/* Similar to the previous function.  */
+static int
+yyexpected_tokens (const yyparse_context_t *yyctx,
+                   int yyarg[], int yyargn)
+{
+  return yypstate_expected_tokens (yyctx->yyps, yyarg, yyargn);
+}]])[
+
 static int
 yysyntax_error_arguments (const yyparse_context_t *yyctx,
                           int yyarg[], int yyargn)
@@ -1430,14 +1455,7 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
 
 ]b4_pure_if([], [b4_declare_scanner_communication_variables])[
 
-]b4_push_if([[
+]b4_push_if([b4_pull_if([[
-struct yypstate
-  {]b4_declare_parser_state_variables[
-    /* Whether this instance has not started parsing yet.  */
-    int yynew;
-  };]b4_pure_if([], [[
-
-static char yypstate_allocated = 0;]])b4_pull_if([[
 
 int
 yyparse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)
@@ -1873,7 +1891,7 @@ yyerrlab:
          [custom],
 [[      {
         yyparse_context_t yyctx
-          = {yyssp, yytoken]b4_locations_if([[, &yylloc]])[]b4_lac_if([[, yyesa, &yyes, &yyes_capacity]])[};]b4_lac_if([[
+          = {]b4_push_if([[yyps]], [[yyssp]b4_lac_if([[, yyesa, &yyes, &yyes_capacity]])])[, yytoken]b4_locations_if([[, &yylloc]])[};]b4_lac_if([[
         if (yychar != YYEMPTY)
           YY_LAC_ESTABLISH;]])[
         if (yyreport_syntax_error (&yyctx]m4_ifset([b4_parse_param],
@@ -1883,9 +1901,9 @@ yyerrlab:
          [simple],
 [[      yyerror (]b4_yyerror_args[YY_("syntax error"));]],
 [[      {
-        char const *yymsgp = YY_("syntax error");
         yyparse_context_t yyctx
-          = {yyssp, yytoken]b4_locations_if([[, &yylloc]])[]b4_lac_if([[, yyesa, &yyes, &yyes_capacity]])[};
+          = {]b4_push_if([[yyps]], [[yyssp]b4_lac_if([[, yyesa, &yyes, &yyes_capacity]])])[, yytoken]b4_locations_if([[, &yylloc]])[};
+        char const *yymsgp = YY_("syntax error");
         int yysyntax_error_status;]b4_lac_if([[
         if (yychar != YYEMPTY)
           YY_LAC_ESTABLISH;]])[

examples/c/bistromathic/parse.y

@@ -332,24 +332,23 @@ int
 expected_tokens (const char *input,
                  int *tokens, int ntokens)
 {
-  YYDPRINTF ((stderr, "expected_tokens(\"%s\")", input));
+  YYDPRINTF ((stderr, "expected_tokens (\"%s\")", input));
 
   // Parse the current state of the line.
-  YYLTYPE lloc;
   yypstate *ps = yypstate_new ();
   int status = 0;
   do {
+    YYLTYPE lloc;
     YYSTYPE lval;
     int token = yylex (&input, &lval, &lloc);
+    // Don't let the parse know when we reach the end of input.
     if (!token)
       break;
     status = yypush_parse (ps, token, &lval, &lloc);
   } while (status == YYPUSH_MORE);
 
   // Then query for the accepted tokens at this point.
-  yyparse_context_t yyctx
+  int res = yypstate_expected_tokens (ps, tokens, ntokens);
-    = {ps->yyssp, YYEMPTY, &lloc, ps->yyesa, &ps->yyes, &ps->yyes_capacity};
-  int res = yyexpected_tokens (&yyctx, tokens, ntokens);
   yypstate_delete (ps);
   return res;
 }
@@ -362,7 +361,7 @@ expected_tokens (const char *input,
 char **
 completion (const char *text, int start, int end)
 {
-  YYDPRINTF ((stderr, "completion(\"%.*s[%.*s]%s\")\n",
+  YYDPRINTF ((stderr, "completion (\"%.*s[%.*s]%s\")\n",
               start, rl_line_buffer,
               end - start, rl_line_buffer + start,
               rl_line_buffer + end));
@@ -414,7 +413,7 @@ completion (const char *text, int start, int end)
 
   if (yydebug)
     {
-      fprintf (stderr, "completion(\"%.*s[%.*s]%s\") = ",
+      fprintf (stderr, "completion (\"%.*s[%.*s]%s\") = ",
                start, rl_line_buffer,
                end - start, rl_line_buffer + start,
                rl_line_buffer + end);
@@ -452,7 +451,7 @@ void init_readline (void)
 int main (int argc, char const* argv[])
 {
   // Enable parse traces on option -p.
-  if (argc == 2 && strcmp(argv[1], "-p") == 0)
+  if (argc == 2 && strcmp (argv[1], "-p") == 0)
     yydebug = 1;
   init_table ();
   init_readline ();