lalr1.cc: rename lex_symbol as api.token.constructor

* data/bison.m4 (b4_lex_symbol_if): Rename as...
(b4_token_ctor_if): this.
Depend upon api.token.constructor.
* data/c++.m4, data/lalr1.cc: Adjust.
* doc/bison.texi: Fix all the occurrences of lex_symbol.
* etc/bench.pl.in: Adjust.
* examples/variant.yy: Likewise.

* tests/local.at (AT_BISON_OPTION_PUSHDEFS, AT_BISON_OPTION_POPDEFS):
Handle AT_TOKEN_CTOR_IF.
* tests/c++.at: Adjust to using api.token.constructor and AT_TOKEN_CTOR_IF.
Simplify the test of both build call styles.
(AT_CHECK_VARIANTS): Rename as...
(AT_TEST): this.
And undef when done.

data/bison.m4

@@ -796,10 +796,10 @@ m4_percent_define_default([[api.token.prefix]], [[]])
 
 # b4_parse_assert_if([IF-ASSERTIONS-ARE-USED], [IF-NOT])
 # b4_parse_trace_if([IF-DEBUG-TRACES-ARE-ENABLED], [IF-NOT])
-# b4_lex_symbol_if([IF-YYLEX-RETURNS-A-COMPLETE-SYMBOL], [IF-NOT])
+# b4_token_ctor_if([IF-YYLEX-RETURNS-A-TOKEN], [IF-NOT])
 # b4_variant_if([IF-VARIANT-ARE-USED], [IF-NOT])
 # ----------------------------------------------
-b4_percent_define_if_define([lex_symbol])
+b4_percent_define_if_define([token_ctor], [api.token.constructor])
 b4_percent_define_if_define([locations])     # Whether locations are tracked.
 b4_percent_define_if_define([parse.assert])
 b4_percent_define_if_define([parse.trace])

data/c++.m4

@@ -300,7 +300,7 @@ m4_define([b4_public_types_define],
   {
     return type;
   }
-]b4_lex_symbol_if([[
+]b4_token_ctor_if([[
   inline
   ]b4_parser_class_name[::token_type
   ]b4_parser_class_name[::symbol_type::token () const
@@ -335,7 +335,7 @@ m4_define([b4_symbol_constructor_define], [])
 m4_define([b4_yytranslate_define],
 [[  // Symbol number corresponding to token number t.
   ]b4_parser_class_name[::token_number_type
-  ]b4_parser_class_name[::yytranslate_ (]b4_lex_symbol_if([token_type],
+  ]b4_parser_class_name[::yytranslate_ (]b4_token_ctor_if([token_type],
                                                           [int])[ t)
   {
     static

data/lalr1.cc

@@ -239,7 +239,7 @@ b4_location_define])])])[
 #endif // ]b4_api_PREFIX[DEBUG
 
     /// Convert a scanner token number \a t to a symbol number.
-    static inline token_number_type yytranslate_ (]b4_lex_symbol_if([token_type], [int])[ t);
+    static inline token_number_type yytranslate_ (]b4_token_ctor_if([token_type], [int])[ t);
 
 #if ]b4_api_PREFIX[DEBUG
     /// \brief Display a symbol type, value and location.
@@ -320,7 +320,7 @@ b4_location_define])])])[
 ]b4_parse_param_vars[
   };
 
-]b4_lex_symbol_if([b4_yytranslate_define
+]b4_token_ctor_if([b4_yytranslate_define
 b4_public_types_define])[
 ]b4_namespace_close[
 
@@ -500,7 +500,7 @@ m4_if(b4_prefix, [yy], [],
   | Symbol types.  |
   `---------------*/
 
-]b4_lex_symbol_if([], [b4_public_types_define])[
+]b4_token_ctor_if([], [b4_public_types_define])[
 
   // stack_symbol_type.
   ]b4_parser_class_name[::stack_symbol_type::stack_symbol_type ()
@@ -726,7 +726,7 @@ b4_dollar_popdef])[]dnl
         YYCDEBUG << "Reading a token: ";
         try
           {
-]b4_lex_symbol_if(
+]b4_token_ctor_if(
 [            yyla = b4_function_call([yylex], [symbol_type],
                                      m4_ifdef([b4_lex_param], b4_lex_param));],
 [            yyla.type = yytranslate_ (b4_function_call([yylex], [int],
@@ -1138,7 +1138,7 @@ b4_error_verbose_if([state_type yystate, int yytoken],
   }
 #endif // ]b4_api_PREFIX[DEBUG
 
-]b4_lex_symbol_if([], [b4_yytranslate_define])[
+]b4_token_ctor_if([], [b4_yytranslate_define])[
 ]b4_namespace_close[
 ]b4_epilogue[]dnl
 m4_divert_pop(0)

doc/bison.texi

@@ -5547,6 +5547,30 @@ More user feedback will help to stabilize it.)
 
 
 
+@c ================================================== api.token.constructor
+@item api.token.constructor
+@findex %define api.token.constructor
+
+@itemize @bullet
+@item Language(s):
+C++
+
+@item Purpose:
+When variant-based semantic values are enabled (@pxref{C++ Variants}),
+request that symbols be handled as a whole (type, value, and possibly
+location) in the scanner.  @xref{Complete Symbols}, for details.
+
+@item Accepted Values:
+Boolean.
+
+@item Default Value:
+@code{false}
+@item History:
+introduced in Bison 2.8
+@end itemize
+@c api.token.constructor
+
+
 @c ================================================== api.token.prefix
 @item api.token.prefix
 @findex %define api.token.prefix
@@ -5587,28 +5611,6 @@ introduced in Bison 2.8
 @c api.token.prefix
 
 
-@c ================================================== lex_symbol
-@item lex_symbol
-@findex %define lex_symbol
-
-@itemize @bullet
-@item Language(s):
-C++
-
-@item Purpose:
-When variant-based semantic values are enabled (@pxref{C++ Variants}),
-request that symbols be handled as a whole (type, value, and possibly
-location) in the scanner.  @xref{Complete Symbols}, for details.
-
-@item Accepted Values:
-Boolean.
-
-@item Default Value:
-@code{false}
-@end itemize
-@c lex_symbol
-
-
 @c ================================================== lr.default-reduction
 
 @item lr.default-reduction
@@ -10194,7 +10196,8 @@ or
 @node Complete Symbols
 @subsubsection Complete Symbols
 
-If you specified both @code{%define variant} and @code{%define lex_symbol},
+If you specified both @code{%define variant} and
+@code{%define api.token.constructor},
 the @code{parser} class also defines the class @code{parser::symbol_type}
 which defines a @emph{complete} symbol, aggregating its type (i.e., the
 traditional value returned by @code{yylex}), its semantic value (i.e., the
@@ -10456,18 +10459,18 @@ the grammar for.
 @end example
 
 @noindent
+@findex %define api.token.constructor
 @findex %define variant
-@findex %define lex_symbol
 This example will use genuine C++ objects as semantic values, therefore, we
 require the variant-based interface.  To make sure we properly use it, we
 enable assertions.  To fully benefit from type-safety and more natural
-definition of ``symbol'', we enable @code{lex_symbol}.
+definition of ``symbol'', we enable @code{api.token.constructor}.
 
 @comment file: calc++-parser.yy
 @example
-%define variant
+%define api.token.constructor
 %define parse.assert
-%define lex_symbol
+%define variant
 @end example
 
 @noindent

etc/bench.pl.in

@@ -579,7 +579,7 @@ sub generate_grammar_list ($$@)
   my ($base, $max, @directive) = @_;
   my $directives = directives ($base, @directive);
   my $variant = grep { /%define "?variant"?/ } @directive;
-  my $lex_symbol = grep { /%define "?lex_symbol"?/ } @directive;
+  my $token_ctor = grep { /%define "?api.token.constructor"?/ } @directive;
   my $out = new IO::File ">$base.y"
     or die;
   print $out <<EOF;
@@ -601,12 +601,12 @@ $directives
 
 #define STAGE_MAX    ($max * 10) // max = $max
 
-#define USE_LEX_SYMBOL $lex_symbol
+#define USE_TOKEN_CTOR $token_ctor
 #define USE_VARIANTS $variant
 
   // Prototype of the yylex function providing subsequent tokens.
   static
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
   yy::parser::symbol_type yylex();
 #else
   yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
@@ -678,7 +678,7 @@ EOF
 #
 
 static
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
 yy::parser::symbol_type yylex()
 #else
 yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
@@ -691,7 +691,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
   ++stage;
   if (stage == STAGE_MAX)
     {
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
       return yy::parser::make_END_OF_FILE (location_type ());
 #else
       *yylloc = location_type ();
@@ -700,7 +700,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
     }
   else if (stage % 2)
     {
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
       return yy::parser::make_NUMBER (stage, location_type ());
 #else
 # if defined ONE_STAGE_BUILD
@@ -716,7 +716,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
     }
   else
     {
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
       return yy::parser::make_TEXT ("A string.", location_type ());
 #else
 # if defined ONE_STAGE_BUILD
@@ -914,7 +914,7 @@ sub bench_variant_parser ()
             [
               %d variant
               &
-              [ #d ONE_STAGE_BUILD | %d lex_symbol ]
+              [ #d ONE_STAGE_BUILD | %d api.token.constructor ]
             ]
          )
     );

examples/variant.yy

@@ -18,9 +18,9 @@
 %debug
 %skeleton "lalr1.cc"
 %defines
+%define api.token.constructor
 %define parse.assert
 %define variant
-%define lex_symbol
 %locations
 
 %code requires // *.hh

tests/c++.at

@@ -22,12 +22,13 @@ AT_BANNER([[C++ Features.]])
 ## Variants.  ##
 ## ---------- ##
 
-# AT_CHECK_VARIANTS([DIRECTIVES])
+# AT_TEST([DIRECTIVES])
-# -------------------------------
+# ---------------------
 # Check the support of variants in C++, with the additional DIRECTIVES.
-m4_define([AT_CHECK_VARIANTS],
+m4_pushdef([AT_TEST],
 [AT_SETUP([Variants $1])
 
+AT_BISON_OPTION_PUSHDEFS([%skeleton "lalr1.cc" $1])
 # Store strings and integers in a list of strings.
 AT_DATA_GRAMMAR([list.yy],
 [[%debug
@@ -50,13 +51,13 @@ typedef std::list<std::string> strings_type;
 #include <iostream>
 #include <sstream>
 
-  static
+  namespace yy
-#if defined USE_LEX_SYMBOL
+  {
-  yy::parser::symbol_type yylex ();
+    static]AT_TOKEN_CTOR_IF([[
-#else
+    parser::symbol_type yylex ()]], [[
-  yy::parser::token_type yylex (yy::parser::semantic_type* yylval,
+    parser::token_type yylex (parser::semantic_type* yylval,
-                                yy::parser::location_type* yylloc);
+                              parser::location_type* yylloc)]])[;
-#endif
+  }
 
   // Printing a list of strings (for %printer).
   // Koening look up will look into std, since that's an std::list.
@@ -118,66 +119,51 @@ item:
 ;
 %%
 
-#define STAGE_MAX 5
+#ifdef TWO_STAGE_BUILD
-static
+# define BUILD(Type, Value) build<Type> () = Value
-#if defined USE_LEX_SYMBOL
-yy::parser::symbol_type yylex ()
 #else
-yy::parser::token_type yylex (yy::parser::semantic_type* yylval,
+# define BUILD(Type, Value) build (Value)
-                              yy::parser::location_type* yylloc)
 #endif
-{
-#ifndef USE_LEX_SYMBOL
-  typedef yy::parser::token token;
-#endif
-  typedef yy::parser::location_type location_type;
-  static int stage = -1;
-  ++stage;
-  if (stage == STAGE_MAX)
-    {
-#if defined USE_LEX_SYMBOL
-      return yy::parser::make_END_OF_FILE (location_type ());
-#else
-      *yylloc = location_type ();
-      return token::END_OF_FILE;
-#endif
-    }
-  else if (stage % 2)
-    {
-#if defined USE_LEX_SYMBOL
-      return yy::parser::make_NUMBER (stage, location_type ());
-#else
-# if defined ONE_STAGE_BUILD
-      yylval->build (stage);
-# else
-      yylval->build<int>() = stage;
-# endif
-      *yylloc = location_type ();
-      return token::NUMBER;
-#endif
-    }
-  else
-    {
-#if defined USE_LEX_SYMBOL
-      return yy::parser::make_TEXT (string_cast (stage), location_type ());
-#else
-# if defined ONE_STAGE_BUILD
-      yylval->build (string_cast (stage));
-# else
-      yylval->build<std::string>() = string_cast (stage);
-# endif
-      *yylloc = location_type ();
-      return token::TEXT;
-#endif
-    }
-  abort ();
-}
 
-void
+#define STAGE_MAX 5
-yy::parser::error (const yy::parser::location_type&,
+namespace yy
-                   const std::string& message)
 {
-  std::cerr << message << std::endl;
+  static]AT_TOKEN_CTOR_IF([[
+  parser::symbol_type yylex ()]], [[
+  parser::token_type yylex (parser::semantic_type* yylval,
+                            parser::location_type* yylloc)]])[
+  {
+    typedef parser::location_type location;
+    static int stage = -1;
+    ++stage;
+    if (stage == STAGE_MAX)
+      {]AT_TOKEN_CTOR_IF([[
+        return parser::make_END_OF_FILE (location ());]], [[
+        *yylloc = location ();
+        return parser::token::END_OF_FILE;]])[
+      }
+    else if (stage % 2)
+      {]AT_TOKEN_CTOR_IF([[
+        return parser::make_NUMBER (stage, location ());]], [[
+        yylval->BUILD (int, stage);
+        *yylloc = location ();
+        return parser::token::NUMBER;]])[
+      }
+    else
+      {]AT_TOKEN_CTOR_IF([[
+        return parser::make_TEXT (string_cast (stage), location ());]], [[
+        yylval->BUILD (std::string, string_cast (stage));
+        *yylloc = location ();
+        return parser::token::TEXT;]])[
+      }
+    abort ();
+  }
+
+  void
+  parser::error (const parser::location_type&, const std::string& message)
+  {
+    std::cerr << message << std::endl;
+  }
 }
 
 int
@@ -195,14 +181,17 @@ AT_CHECK([./list], 0,
          [(0, 1, 2, 4)
 ])
 
+AT_BISON_OPTION_POPDEFS
 AT_CLEANUP
 ])
 
-AT_CHECK_VARIANTS([])
+AT_TEST([])
-AT_CHECK_VARIANTS([%define parse.assert])
+AT_TEST([%define parse.assert])
-AT_CHECK_VARIANTS([[%define parse.assert %code {\n#define ONE_STAGE_BUILD\n}]])
+AT_TEST([[%define parse.assert %code {\n#define TWO_STAGE_BUILD\n}]])
-AT_CHECK_VARIANTS([[%define parse.assert %define lex_symbol %code {\n#define USE_LEX_SYMBOL\n}]])
+AT_TEST([[%define parse.assert %define api.token.constructor]])
-AT_CHECK_VARIANTS([[%define parse.assert %define lex_symbol %code {\n#define USE_LEX_SYMBOL\n} %define api.token.prefix "TOK_"]])
+AT_TEST([[%define parse.assert %define api.token.constructor %define api.token.prefix "TOK_"]])
+
+m4_popdef([AT_TEST])
 
 
 ## ----------------------- ##

tests/local.at

@@ -152,6 +152,8 @@ m4_pushdef([AT_NAME_PREFIX],
 [m4_bmatch([$3], [\(%define api\.prefix\|%name-prefix\) ".*"],
            [m4_bregexp([$3], [\(%define api\.prefix\|%name-prefix\) "\([^""]*\)"], [\2])],
            [yy])])
+m4_pushdef([AT_TOKEN_CTOR_IF],
+[m4_bmatch([$3], [%define api.token.constructor], [$1], [$2])])
 m4_pushdef([AT_TOKEN_PREFIX],
 [m4_bmatch([$3], [%define api.token.prefix ".*"],
            [m4_bregexp([$3], [%define api.token.prefix "\(.*\)"], [\1])])])
@@ -241,6 +243,8 @@ m4_popdef([AT_YYERROR_SEES_LOC_IF])
 m4_popdef([AT_YYERROR_ARG_LOC_IF])
 m4_popdef([AT_API_PREFIX])
 m4_popdef([AT_API_prefix])
+m4_popdef([AT_TOKEN_PREFIX])
+m4_popdef([AT_TOKEN_CTOR_IF])
 m4_popdef([AT_NAME_PREFIX])
 m4_popdef([AT_GLR_OR_PARAM_IF])
 m4_popdef([AT_PURE_AND_LOC_IF])