lalr1.cc: rename lex_symbol as api.token.constructor

* data/bison.m4 (b4_lex_symbol_if): Rename as... (b4_token_ctor_if): this. Depend upon api.token.constructor. * data/c++.m4, data/lalr1.cc: Adjust. * doc/bison.texi: Fix all the occurrences of lex_symbol. * etc/bench.pl.in: Adjust. * examples/variant.yy: Likewise. * tests/local.at (AT_BISON_OPTION_PUSHDEFS, AT_BISON_OPTION_POPDEFS): Handle AT_TOKEN_CTOR_IF. * tests/c++.at: Adjust to using api.token.constructor and AT_TOKEN_CTOR_IF. Simplify the test of both build call styles. (AT_CHECK_VARIANTS): Rename as... (AT_TEST): this. And undef when done.
2026-03-09 20:33:03 +00:00 · 2012-11-01 17:54:13 +01:00
parent 0b3287025d
commit e36ec1f41f
8 changed files with 112 additions and 116 deletions
--- a/data/bison.m4
+++ b/data/bison.m4
@@ -796,10 +796,10 @@ m4_percent_define_default([[api.token.prefix]], [[]])

 # b4_parse_assert_if([IF-ASSERTIONS-ARE-USED], [IF-NOT])
 # b4_parse_trace_if([IF-DEBUG-TRACES-ARE-ENABLED], [IF-NOT])
-# b4_lex_symbol_if([IF-YYLEX-RETURNS-A-COMPLETE-SYMBOL], [IF-NOT])
+# b4_token_ctor_if([IF-YYLEX-RETURNS-A-TOKEN], [IF-NOT])
 # b4_variant_if([IF-VARIANT-ARE-USED], [IF-NOT])
 # ----------------------------------------------
-b4_percent_define_if_define([lex_symbol])
+b4_percent_define_if_define([token_ctor], [api.token.constructor])
 b4_percent_define_if_define([locations])     # Whether locations are tracked.
 b4_percent_define_if_define([parse.assert])
 b4_percent_define_if_define([parse.trace])
--- a/data/c++.m4
+++ b/data/c++.m4
@@ -300,7 +300,7 @@ m4_define([b4_public_types_define],
  {
    return type;
  }
-]b4_lex_symbol_if([[
+]b4_token_ctor_if([[
  inline
  ]b4_parser_class_name[::token_type
  ]b4_parser_class_name[::symbol_type::token () const
@@ -335,7 +335,7 @@ m4_define([b4_symbol_constructor_define], [])
 m4_define([b4_yytranslate_define],
 [[  // Symbol number corresponding to token number t.
  ]b4_parser_class_name[::token_number_type
-  ]b4_parser_class_name[::yytranslate_ (]b4_lex_symbol_if([token_type],
+  ]b4_parser_class_name[::yytranslate_ (]b4_token_ctor_if([token_type],
                                                          [int])[ t)
  {
    static
--- a/data/lalr1.cc
+++ b/data/lalr1.cc
@@ -239,7 +239,7 @@ b4_location_define])])])[
 #endif // ]b4_api_PREFIX[DEBUG

    /// Convert a scanner token number \a t to a symbol number.
-    static inline token_number_type yytranslate_ (]b4_lex_symbol_if([token_type], [int])[ t);
+    static inline token_number_type yytranslate_ (]b4_token_ctor_if([token_type], [int])[ t);

 #if ]b4_api_PREFIX[DEBUG
    /// \brief Display a symbol type, value and location.
@@ -320,7 +320,7 @@ b4_location_define])])])[
 ]b4_parse_param_vars[
  };

-]b4_lex_symbol_if([b4_yytranslate_define
+]b4_token_ctor_if([b4_yytranslate_define
 b4_public_types_define])[
 ]b4_namespace_close[

@@ -500,7 +500,7 @@ m4_if(b4_prefix, [yy], [],
  | Symbol types.  |
  `---------------*/

-]b4_lex_symbol_if([], [b4_public_types_define])[
+]b4_token_ctor_if([], [b4_public_types_define])[

  // stack_symbol_type.
  ]b4_parser_class_name[::stack_symbol_type::stack_symbol_type ()
@@ -726,7 +726,7 @@ b4_dollar_popdef])[]dnl
        YYCDEBUG << "Reading a token: ";
        try
          {
-]b4_lex_symbol_if(
+]b4_token_ctor_if(
 [            yyla = b4_function_call([yylex], [symbol_type],
                                     m4_ifdef([b4_lex_param], b4_lex_param));],
 [            yyla.type = yytranslate_ (b4_function_call([yylex], [int],
@@ -1138,7 +1138,7 @@ b4_error_verbose_if([state_type yystate, int yytoken],
  }
 #endif // ]b4_api_PREFIX[DEBUG

-]b4_lex_symbol_if([], [b4_yytranslate_define])[
+]b4_token_ctor_if([], [b4_yytranslate_define])[
 ]b4_namespace_close[
 ]b4_epilogue[]dnl
 m4_divert_pop(0)
--- a/doc/bison.texi
+++ b/doc/bison.texi
@@ -5547,6 +5547,30 @@ More user feedback will help to stabilize it.)



+@c ================================================== api.token.constructor
+@item api.token.constructor
+@findex %define api.token.constructor
+
+@itemize @bullet
+@item Language(s):
+C++
+
+@item Purpose:
+When variant-based semantic values are enabled (@pxref{C++ Variants}),
+request that symbols be handled as a whole (type, value, and possibly
+location) in the scanner.  @xref{Complete Symbols}, for details.
+
+@item Accepted Values:
+Boolean.
+
+@item Default Value:
+@code{false}
+@item History:
+introduced in Bison 2.8
+@end itemize
+@c api.token.constructor
+
+
@c ================================================== api.token.prefix
@item api.token.prefix
@findex %define api.token.prefix
@@ -5587,28 +5611,6 @@ introduced in Bison 2.8
@c api.token.prefix


-@c ================================================== lex_symbol
-@item lex_symbol
-@findex %define lex_symbol
-
-@itemize @bullet
-@item Language(s):
-C++
-
-@item Purpose:
-When variant-based semantic values are enabled (@pxref{C++ Variants}),
-request that symbols be handled as a whole (type, value, and possibly
-location) in the scanner.  @xref{Complete Symbols}, for details.
-
-@item Accepted Values:
-Boolean.
-
-@item Default Value:
-@code{false}
-@end itemize
-@c lex_symbol
-
-
@c ================================================== lr.default-reduction

@item lr.default-reduction
@@ -10194,7 +10196,8 @@ or
@node Complete Symbols
@subsubsection Complete Symbols

-If you specified both @code{%define variant} and @code{%define lex_symbol},
+If you specified both @code{%define variant} and
+@code{%define api.token.constructor},
 the @code{parser} class also defines the class @code{parser::symbol_type}
 which defines a @emph{complete} symbol, aggregating its type (i.e., the
 traditional value returned by @code{yylex}), its semantic value (i.e., the
@@ -10456,18 +10459,18 @@ the grammar for.
@end example

@noindent
+@findex %define api.token.constructor
@findex %define variant
-@findex %define lex_symbol
 This example will use genuine C++ objects as semantic values, therefore, we
 require the variant-based interface.  To make sure we properly use it, we
 enable assertions.  To fully benefit from type-safety and more natural
-definition of ``symbol'', we enable @code{lex_symbol}.
+definition of ``symbol'', we enable @code{api.token.constructor}.

@comment file: calc++-parser.yy
@example
-%define variant
+%define api.token.constructor
 %define parse.assert
-%define lex_symbol
+%define variant
@end example

@noindent
--- a/etc/bench.pl.in
+++ b/etc/bench.pl.in
@@ -579,7 +579,7 @@ sub generate_grammar_list ($$@)
  my ($base, $max, @directive) = @_;
  my $directives = directives ($base, @directive);
  my $variant = grep { /%define "?variant"?/ } @directive;
-  my $lex_symbol = grep { /%define "?lex_symbol"?/ } @directive;
+  my $token_ctor = grep { /%define "?api.token.constructor"?/ } @directive;
  my $out = new IO::File ">$base.y"
    or die;
  print $out <<EOF;
@@ -601,12 +601,12 @@ $directives

 #define STAGE_MAX    ($max * 10) // max = $max

-#define USE_LEX_SYMBOL $lex_symbol
+#define USE_TOKEN_CTOR $token_ctor
 #define USE_VARIANTS $variant

  // Prototype of the yylex function providing subsequent tokens.
  static
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
  yy::parser::symbol_type yylex();
 #else
  yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
@@ -678,7 +678,7 @@ EOF
 #

 static
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
 yy::parser::symbol_type yylex()
 #else
 yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
@@ -691,7 +691,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
  ++stage;
  if (stage == STAGE_MAX)
    {
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
      return yy::parser::make_END_OF_FILE (location_type ());
 #else
      *yylloc = location_type ();
@@ -700,7 +700,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
    }
  else if (stage % 2)
    {
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
      return yy::parser::make_NUMBER (stage, location_type ());
 #else
 # if defined ONE_STAGE_BUILD
@@ -716,7 +716,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval,
    }
  else
    {
-#if USE_LEX_SYMBOL
+#if USE_TOKEN_CTOR
      return yy::parser::make_TEXT ("A string.", location_type ());
 #else
 # if defined ONE_STAGE_BUILD
@@ -914,7 +914,7 @@ sub bench_variant_parser ()
            [
              %d variant
              &
-              [ #d ONE_STAGE_BUILD | %d lex_symbol ]
+              [ #d ONE_STAGE_BUILD | %d api.token.constructor ]
            ]
         )
    );
--- a/examples/variant.yy
+++ b/examples/variant.yy
@@ -18,9 +18,9 @@
 %debug
 %skeleton "lalr1.cc"
 %defines
+%define api.token.constructor
 %define parse.assert
 %define variant
-%define lex_symbol
 %locations

 %code requires // *.hh
--- a/tests/c++.at
+++ b/tests/c++.at
@@ -22,12 +22,13 @@ AT_BANNER([[C++ Features.]])
 ## Variants.  ##
 ## ---------- ##

-# AT_CHECK_VARIANTS([DIRECTIVES])
-# -------------------------------
+# AT_TEST([DIRECTIVES])
+# ---------------------
 # Check the support of variants in C++, with the additional DIRECTIVES.
-m4_define([AT_CHECK_VARIANTS],
+m4_pushdef([AT_TEST],
 [AT_SETUP([Variants $1])

+AT_BISON_OPTION_PUSHDEFS([%skeleton "lalr1.cc" $1])
 # Store strings and integers in a list of strings.
 AT_DATA_GRAMMAR([list.yy],
 [[%debug
@@ -50,13 +51,13 @@ typedef std::list<std::string> strings_type;
 #include <iostream>
 #include <sstream>

-  static
-#if defined USE_LEX_SYMBOL
-  yy::parser::symbol_type yylex ();
-#else
-  yy::parser::token_type yylex (yy::parser::semantic_type* yylval,
-                                yy::parser::location_type* yylloc);
-#endif
+  namespace yy
+  {
+    static]AT_TOKEN_CTOR_IF([[
+    parser::symbol_type yylex ()]], [[
+    parser::token_type yylex (parser::semantic_type* yylval,
+                              parser::location_type* yylloc)]])[;
+  }

  // Printing a list of strings (for %printer).
  // Koening look up will look into std, since that's an std::list.
@@ -118,66 +119,51 @@ item:
 ;
 %%

-#define STAGE_MAX 5
-static
-#if defined USE_LEX_SYMBOL
-yy::parser::symbol_type yylex ()
+#ifdef TWO_STAGE_BUILD
+# define BUILD(Type, Value) build<Type> () = Value
 #else
-yy::parser::token_type yylex (yy::parser::semantic_type* yylval,
-                              yy::parser::location_type* yylloc)
+# define BUILD(Type, Value) build (Value)
 #endif
-{
-#ifndef USE_LEX_SYMBOL
-  typedef yy::parser::token token;
-#endif
-  typedef yy::parser::location_type location_type;
-  static int stage = -1;
-  ++stage;
-  if (stage == STAGE_MAX)
-    {
-#if defined USE_LEX_SYMBOL
-      return yy::parser::make_END_OF_FILE (location_type ());
-#else
-      *yylloc = location_type ();
-      return token::END_OF_FILE;
-#endif
-    }
-  else if (stage % 2)
-    {
-#if defined USE_LEX_SYMBOL
-      return yy::parser::make_NUMBER (stage, location_type ());
-#else
-# if defined ONE_STAGE_BUILD
-      yylval->build (stage);
-# else
-      yylval->build<int>() = stage;
-# endif
-      *yylloc = location_type ();
-      return token::NUMBER;
-#endif
-    }
-  else
-    {
-#if defined USE_LEX_SYMBOL
-      return yy::parser::make_TEXT (string_cast (stage), location_type ());
-#else
-# if defined ONE_STAGE_BUILD
-      yylval->build (string_cast (stage));
-# else
-      yylval->build<std::string>() = string_cast (stage);
-# endif
-      *yylloc = location_type ();
-      return token::TEXT;
-#endif
-    }
-  abort ();
-}

-void
-yy::parser::error (const yy::parser::location_type&,
-                   const std::string& message)
+#define STAGE_MAX 5
+namespace yy
 {
-  std::cerr << message << std::endl;
+  static]AT_TOKEN_CTOR_IF([[
+  parser::symbol_type yylex ()]], [[
+  parser::token_type yylex (parser::semantic_type* yylval,
+                            parser::location_type* yylloc)]])[
+  {
+    typedef parser::location_type location;
+    static int stage = -1;
+    ++stage;
+    if (stage == STAGE_MAX)
+      {]AT_TOKEN_CTOR_IF([[
+        return parser::make_END_OF_FILE (location ());]], [[
+        *yylloc = location ();
+        return parser::token::END_OF_FILE;]])[
+      }
+    else if (stage % 2)
+      {]AT_TOKEN_CTOR_IF([[
+        return parser::make_NUMBER (stage, location ());]], [[
+        yylval->BUILD (int, stage);
+        *yylloc = location ();
+        return parser::token::NUMBER;]])[
+      }
+    else
+      {]AT_TOKEN_CTOR_IF([[
+        return parser::make_TEXT (string_cast (stage), location ());]], [[
+        yylval->BUILD (std::string, string_cast (stage));
+        *yylloc = location ();
+        return parser::token::TEXT;]])[
+      }
+    abort ();
+  }
+
+  void
+  parser::error (const parser::location_type&, const std::string& message)
+  {
+    std::cerr << message << std::endl;
+  }
 }

 int
@@ -195,14 +181,17 @@ AT_CHECK([./list], 0,
         [(0, 1, 2, 4)
 ])

+AT_BISON_OPTION_POPDEFS
 AT_CLEANUP
 ])

-AT_CHECK_VARIANTS([])
-AT_CHECK_VARIANTS([%define parse.assert])
-AT_CHECK_VARIANTS([[%define parse.assert %code {\n#define ONE_STAGE_BUILD\n}]])
-AT_CHECK_VARIANTS([[%define parse.assert %define lex_symbol %code {\n#define USE_LEX_SYMBOL\n}]])
-AT_CHECK_VARIANTS([[%define parse.assert %define lex_symbol %code {\n#define USE_LEX_SYMBOL\n} %define api.token.prefix "TOK_"]])
+AT_TEST([])
+AT_TEST([%define parse.assert])
+AT_TEST([[%define parse.assert %code {\n#define TWO_STAGE_BUILD\n}]])
+AT_TEST([[%define parse.assert %define api.token.constructor]])
+AT_TEST([[%define parse.assert %define api.token.constructor %define api.token.prefix "TOK_"]])
+
+m4_popdef([AT_TEST])


 ## ----------------------- ##
--- a/tests/local.at
+++ b/tests/local.at
@@ -152,6 +152,8 @@ m4_pushdef([AT_NAME_PREFIX],
 [m4_bmatch([$3], [\(%define api\.prefix\|%name-prefix\) ".*"],
           [m4_bregexp([$3], [\(%define api\.prefix\|%name-prefix\) "\([^""]*\)"], [\2])],
           [yy])])
+m4_pushdef([AT_TOKEN_CTOR_IF],
+[m4_bmatch([$3], [%define api.token.constructor], [$1], [$2])])
 m4_pushdef([AT_TOKEN_PREFIX],
 [m4_bmatch([$3], [%define api.token.prefix ".*"],
           [m4_bregexp([$3], [%define api.token.prefix "\(.*\)"], [\1])])])
@@ -241,6 +243,8 @@ m4_popdef([AT_YYERROR_SEES_LOC_IF])
 m4_popdef([AT_YYERROR_ARG_LOC_IF])
 m4_popdef([AT_API_PREFIX])
 m4_popdef([AT_API_prefix])
+m4_popdef([AT_TOKEN_PREFIX])
+m4_popdef([AT_TOKEN_CTOR_IF])
 m4_popdef([AT_NAME_PREFIX])
 m4_popdef([AT_GLR_OR_PARAM_IF])
 m4_popdef([AT_PURE_AND_LOC_IF])