api.value.type: implement proper support, check, and document

* data/c.m4 (b4_symbol_type_register, b4_type_define_tag)
(b4_symbol_value_union, b4_value_type_setup_union)
(b4_value_type_setup_variant, b4_value_type_setup):
New.
(b4_value_type_define): Use it to set up properly the type.
Handle the various possible values of api.value.type.
* data/c++.m4 (b4_value_type_declare): Likewise.
* data/lalr1.cc (b4_value_type_setup_variant): Redefine.

* tests/types.at: New.
Exercise all the C/C++ skeletons with different types of
api.value.type values.
* tests/local.mk, tests/testsuite.at: Use it.

* doc/bison.texi (%define Summary): Document api.value.type.
* NEWS: Advertise it, together with api.token.constructor.

NEWS

@@ -252,6 +252,77 @@ GNU Bison NEWS
   use these prefixed token names, although the grammar itself still
   uses the short names (as in the sample rule given above).
 
+** Variable api.value.type
+
+  This new %define variable supersedes the #define macro YYSTYPE.  The use
+  of YYSTYPE is discouraged.  In particular, #defining YYSTYPE *and* either
+  using %union or %defining api.value.type results in undefined behavior.
+
+  Either define api.value.type, or use "%union":
+
+    %union
+    {
+      int ival;
+      char *sval;
+    }
+    %token <ival> INT "integer"
+    %token <sval> STRING "string"
+    %printer { fprintf (yyo, "%d", $$); } <ival>
+    %destructor { free ($$); } <sval>
+
+    /* In yylex().  */
+    yylval.ival = 42; return INT;
+    yylval.sval = "42"; return STRING;
+
+  The %define variable api.value.type supports several special values.  The
+  value "union" means that the user provides genuine types, not union member
+  names such as "ival" and "sval" above.
+
+    %define api.value.type "union"
+    %token <int> INT "integer"
+    %token <char *> STRING "string"
+    %printer { fprintf (yyo, "%d", $$); } <int>
+    %destructor { free ($$); } <char *>
+
+    /* In yylex().  */
+    yylval.INT = 42; return INT;
+    yylval.STRING = "42"; return STRING;
+
+  The value "variant" is somewhat equivalent, but for C++ special provision
+  is made to allow classes to be used (more about this below).
+
+    %define api.value.type "variant"
+    %token <int> INT "integer"
+    %token <std::string> STRING "string"
+
+  Any other name is a user type to use.  This is where YYSTYPE used to be
+  used.
+
+    %code requires
+    {
+      struct my_value
+      {
+        enum
+        {
+          is_int, is_string
+        } kind;
+        union
+        {
+          int ival;
+          char *sval;
+        } u;
+      };
+    }
+    %define api.value.type "struct my_value"
+    %token <u.ival> INT "integer"
+    %token <u.sval> STRING "string"
+    %printer { fprintf (yyo, "%d", $$); } <u.ival>
+    %destructor { free ($$); } <u.sval>
+
+    /* In yylex().  */
+    yylval.u.ival = 42; return INT;
+    yylval.u.sval = "42"; return STRING;
+
 ** Variable parse.error
 
   This variable controls the verbosity of error messages.  The use of the
@@ -2536,7 +2607,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
  LocalWords:  Wprecedence Rassoul Wempty Paolo Bonzini parser's Michiel loc
  LocalWords:  redeclaration sval fcaret reentrant XSLT xsl Wmaybe yyvsp Tedi
  LocalWords:  pragmas noreturn untyped Rozenman unexpanded Wojciech Polak
- LocalWords:  Alexandre MERCHANTABILITY
+ LocalWords:  Alexandre MERCHANTABILITY yytype
 
 Local Variables:
 mode: outline

data/bison.m4

@@ -361,6 +361,7 @@ b4_define_flag_if([yacc])               # Whether POSIX Yacc is emulated.
 #   Whether the symbol has an id.
 # - id: string
 #   If has_id, the id.  Guaranteed to be usable as a C identifier.
+#   Prefixed by api.token.prefix if defined.
 # - tag: string.
 #   A representat of the symbol.  Can be 'foo', 'foo.id', '"foo"' etc.
 # - user_number: integer
@@ -371,9 +372,13 @@ b4_define_flag_if([yacc])               # Whether POSIX Yacc is emulated.
 #   The internalized number (used after yytranslate).
 # - has_type: 0, 1
 #   Whether has a semantic value.
+# - type_tag: string
+#   When api.value.type=union, the generated name for the union member.
+#   yytype_INT etc. for symbols that has_id, otherwise yytype_1 etc.
 # - type
 #   If it has a semantic value, its type tag, or, if variant are used,
 #   its type.
+#   In the case of api.value.type=union, type is the real type (e.g. int).
 # - has_printer: 0, 1
 # - printer: string
 # - printer_file: string
@@ -962,3 +967,10 @@ b4_percent_define_ifdef([api.prefix],
                 [['%s' and '%s' cannot be used together]],
                 [%name-prefix],
                 [%define api.prefix])])])
+
+b4_percent_define_ifdef([api.value.type],
+[m4_ifdef([b4_union_members],
+[b4_complain_at(b4_percent_define_get_loc([api.value.type]),
+                [['%s' and '%s' cannot be used together]],
+                [%union],
+                [%define api.value.type])])])

data/c++.m4

@@ -118,15 +118,16 @@ m4_define([b4_token_enums],
 # ---------------------
 # Declare semantic_type.
 m4_define([b4_value_type_declare],
+[b4_value_type_setup[]dnl
 [    /// Symbol semantic values.
-m4_ifdef([b4_union_members],
+]m4_bmatch(b4_percent_define_get([api.value.type]),
-[    union semantic_type
+[^%union\|union$],
+[[    union semantic_type
     {
-b4_user_union_members
+]b4_user_union_members[
-    };],
+    };]],
-[m4_if(b4_tag_seen_flag, 0,
+[^$], [],
-[[    typedef int semantic_type;]],
+[[    typedef ]b4_percent_define_get([api.value.type])[ semantic_type;]])])
-[[    typedef ]b4_api_PREFIX[STYPE semantic_type;]])])])
 
 
 # b4_public_types_declare

data/c.m4

@@ -492,28 +492,127 @@ b4_locations_if([, yylocationp])[]b4_user_args[);
 }]dnl
 ])
 
+
+## ---------------- ##
+## api.value.type.  ##
+## ---------------- ##
+
+
+# ---------------------- #
+# api.value.type=union.  #
+# ---------------------- #
+
+# b4_symbol_type_register(SYMBOL-NUM)
+# -----------------------------------
+# Symbol SYMBOL-NUM has a type (for variant) instead of a type-tag.
+# Extend the definition of %union's body with a field of that type,
+# and extend the symbol's "type" field to point to the field name,
+# instead of the type name.
+m4_define([b4_symbol_type_register],
+[m4_define([b4_symbol($1, type_tag)],
+           [b4_symbol_if([$1], [has_id],
+                         [b4_symbol([$1], [id])],
+                         [yytype_[]b4_symbol([$1], [number])])])dnl
+m4_append([b4_user_union_members],
+m4_expand([
+  b4_symbol_tag_comment([$1])dnl
+  b4_symbol([$1], [type]) b4_symbol([$1], [type_tag]);]))
+])
+
+
+# b4_type_define_tag(SYMBOL1-NUM, ...)
+# ------------------------------------
+# For the batch of symbols SYMBOL1-NUM... (which all have the same
+# type), enhance the %union definition for each of them, and set
+# there "type" field to the field tag name, instead of the type name.
+m4_define([b4_type_define_tag],
+[b4_symbol_if([$1], [has_type],
+              [m4_map([b4_symbol_type_register], [$@])])
+])
+
+
+# b4_symbol_value_union(VAL, [TYPE])
+# ----------------------------------
+# Same of b4_symbol_value, but when api.value.type=union.
+m4_define([b4_symbol_value_union],
+[m4_ifval([$2],
+          [(*($2*)(&$1))],
+          [$1])])
+])
+
+
+# b4_value_type_setup_union
+# -------------------------
+# Setup support for api.value.type=union.  Symbols are defined with a
+# type instead of a union member name: build the corresponding union,
+# and give the symbols their tag.
+m4_define([b4_value_type_setup_union],
+[m4_define([b4_union_members])
+b4_type_foreach([b4_type_define_tag])
+m4_copy_force([b4_symbol_value_union], [b4_symbol_value])
+])
+
+
+# ---------------- #
+# api.value.type.  #
+# ---------------- #
+
+
+# b4_value_type_setup_variant
+# ---------------------------
+# Setup support for api.value.type=variant.  By default, fail, specialized
+# by other skeletons.
+m4_define([b4_value_type_setup_variant],
+[b4_complain_at(b4_percent_define_get_loc([api.value.type]),
+                [['%s' does not support '%s']],
+                [b4_skeleton],
+                [%define api.value.type variant])])
+
+
+# b4_value_type_setup
+# -------------------
+# Check if api.value.type is properly defined, and possibly prepare
+# its use.
+m4_define([b4_value_type_setup],
+[b4_percent_define_default([[api.value.type]],
+[m4_ifdef([b4_union_members], [%union],
+          [m4_if(b4_tag_seen_flag, 0, [int],
+                 [])])])dnl
+m4_case(b4_percent_define_get([api.value.type]),
+   [union],   [b4_value_type_setup_union],
+   [variant], [b4_value_type_setup_variant])])
+
+
+
 ## -------------- ##
 ## Declarations.  ##
 ## -------------- ##
 
+
 # b4_value_type_define
 # --------------------
 m4_define([b4_value_type_define],
-[[/* Value type.  */
+[b4_value_type_setup[]dnl
-#if ! defined ]b4_api_PREFIX[STYPE && ! defined ]b4_api_PREFIX[STYPE_IS_DECLARED
+/* Value type.  */
-]m4_ifdef([b4_union_members],
+m4_bmatch(b4_percent_define_get([api.value.type]),
-[[typedef union ]b4_union_name[ ]b4_api_PREFIX[STYPE;
+[^%?union$],
+[[#if ! defined ]b4_api_PREFIX[STYPE && ! defined ]b4_api_PREFIX[STYPE_IS_DECLARED
+typedef union ]b4_union_name[ ]b4_api_PREFIX[STYPE;
 union ]b4_union_name[
 {
 ]b4_user_union_members[
 };
-# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1]],
+# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1
-[m4_if(b4_tag_seen_flag, 0,
-[[typedef int ]b4_api_PREFIX[STYPE;
-# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1]])])[
 # define ]b4_api_PREFIX[STYPE_IS_DECLARED 1
 #endif
-]])
+]],
+[^$], [],
+[[#if ! defined ]b4_api_PREFIX[STYPE && ! defined ]b4_api_PREFIX[STYPE_IS_DECLARED
+typedef ]b4_percent_define_get([api.value.type])[ ]b4_api_PREFIX[STYPE;
+# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1
+# define ]b4_api_PREFIX[STYPE_IS_DECLARED 1
+#endif
+]])])
 
 
 # b4_location_type_define

data/lalr1.cc

@@ -17,6 +17,8 @@
 
 m4_include(b4_pkgdatadir/[c++.m4])
 
+# api.value.type=variant is valid.
+m4_define([b4_value_type_setup_variant])
 
 # b4_integral_parser_table_declare(TABLE-NAME, CONTENT, COMMENT)
 # --------------------------------------------------------------
@@ -42,9 +44,11 @@ m4_define([b4_integral_parser_table_define],
 # b4_symbol_value_template(VAL, [TYPE])
 # -------------------------------------
 # Same as b4_symbol_value, but used in a template method.  It makes
-# a difference when using variants.
+# a difference when using variants.  Note that b4_value_type_setup_union
+# overrides b4_symbol_value, so we must override it again.
 m4_copy([b4_symbol_value], [b4_symbol_value_template])
-
+m4_append([b4_value_type_setup_union],
+          [m4_copy_force([b4_symbol_value_union], [b4_symbol_value_template])])
 
 # b4_lhs_value([TYPE])
 # --------------------

doc/bison.texi

@@ -5586,6 +5586,7 @@ Summary,,%skeleton}).
 Unaccepted @var{variable}s produce an error.
 Some of the accepted @var{variable}s are described below.
 
+@c ================================================== api.namespace
 @deffn Directive {%define api.namespace} @{@var{namespace}@}
 @itemize
 @item Languages(s): C++
@@ -5812,14 +5813,89 @@ introduced in Bison 2.8
 @deffn Directive {%define api.value.type} @var{type}
 @itemize @bullet
 @item Language(s):
-C++
+all
 
 @item Purpose:
-Request variant-based semantic values.
+The type for semantic values.
+
+@item Accepted Values:
+@table @asis
+@item @code{""}
+This grammar has no semantic value at all.  This is not properly supported
+yet.
+@item @code{%union} (C, C++)
+The type is defined thanks to the @code{%union} directive.  You don't have
+to define @code{api.value.type} in that case, using @code{%union} suffices.
+@xref{Union Decl, ,The Collection of Value Types}.
+For instance:
+@example
+%define api.value.type "%union"
+%union
+@{
+  int ival;
+  char *sval;
+@}
+%token <ival> INT "integer"
+%token <sval> STR "string"
+@end example
+
+@item @code{union} (C, C++)
+The symbols are defined with type names, from which Bison will generate a
+@code{union}.  For instance:
+@example
+%define api.value.type "union"
+%token <int> INT "integer"
+%token <char *> STR "string"
+@end example
+This feature needs user feedback to stabilize.  Note that most C++ objects
+cannot be stored in a @code{union}.
+
+@item @code{variant} (C++)
+This is similar to @code{union}, but special storage techniques are used to
+allow any kind of C++ object to be used. For instance:
+@example
+%define api.value.type "variant"
+%token <int> INT "integer"
+%token <std::string> STR "string"
+@end example
+This feature needs user feedback to stabilize.
 @xref{C++ Variants}.
 
+@item any other identifier
+Use this name as semantic value.
+@example
+%code requires
+@{
+  struct my_value
+  @{
+    enum
+    @{
+      is_int, is_str
+    @} kind;
+    union
+    @{
+      int ival;
+      char *sval;
+    @} u;
+  @};
+@}
+%define api.value.type "struct my_value"
+%token <u.ival> INT "integer"
+%token <u.sval> STR "string"
+@end example
+@end table
+
 @item Default Value:
-FIXME:
+@itemize @minus
+@item
+@code{%union} if @code{%union} is used, otherwise @dots{}
+@item
+@code{int} if type tags are used (i.e., @samp{%token <@var{type}>@dots{}} or
+@samp{%token <@var{type}>@dots{}} is used), otherwise @dots{}
+@item
+@code{""}
+@end itemize
+
 @item History:
 introduced in Bison 2.8.  Was introduced for Java only in 2.3b as
 @code{stype}.

tests/local.mk

@@ -63,7 +63,8 @@ TESTSUITE_AT =                                  \
   tests/sets.at                                 \
   tests/skeletons.at                            \
   tests/synclines.at                            \
-  tests/torture.at
+  tests/torture.at                              \
+  tests/types.at
 
 TESTSUITE = $(top_srcdir)/tests/testsuite
 

tests/testsuite.at

@@ -35,6 +35,9 @@ m4_include([sets.at])
 # Testing grammar reduction.
 m4_include([reduce.at])
 
+# Testing conflicts detection and resolution.
+m4_include([conflicts.at])
+
 # Testing that #lines are correct.
 m4_include([synclines.at])
 
@@ -44,8 +47,8 @@ m4_include([headers.at])
 # Testing that user actions are properly performed.
 m4_include([actions.at])
 
-# Testing conflicts detection and resolution.
+# Testing semantic types support.
-m4_include([conflicts.at])
+m4_include([types.at])
 
 # Fulling testing (compilation and execution of the parser) on calc.
 m4_include([calc.at])

tests/types.at

@@ -0,0 +1,171 @@
+# Value type.                                     -*- Autotest -*-
+
+# Copyright (C) 2013 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+AT_BANNER([[Value type tests.]])
+
+
+## ----------------------------------- ##
+## %union vs. %define api.value.type.  ##
+## ----------------------------------- ##
+
+AT_SETUP([[%union vs. %define api.value.type]])
+
+AT_DATA([[input.y]],
+[[%union { int ival; }
+%define api.value.type "%union"
+%%
+exp: %empty;
+]])
+
+AT_BISON_CHECK([[input.y]], [[1]], [[]],
+[[input.y:2.9-22: error: '%union' and '%define api.value.type' cannot be used together
+]])
+
+AT_CLEANUP
+
+## ---------------- ##
+## api.value.type.  ##
+## ---------------- ##
+
+# AT_TEST($1: BISON-DIRECTIVES,
+#         $2: MORE-BISON-DIRECTIVES,
+#         $3: PARSER-ACTION,
+#         $4: INPUT, $5: SCANNER-ACTION,
+#         $6: RESULT)
+# --------------------------------------
+# Compile the grammar and check the expected result.
+# BISON-DIRECTIVES are passed to AT_SETUP, contrary to MORE-BISON-DIRECTIVES.
+m4_pushdef([AT_TEST],
+[
+AT_SETUP([$1])
+AT_KEYWORDS([api.value.type])
+AT_BISON_OPTION_PUSHDEFS([$1 $2])
+AT_DATA_GRAMMAR([test.y],
+[[%debug
+
+%code
+{
+# include <stdio.h>
+# include <stdlib.h>
+]AT_YYERROR_DECLARE[
+]AT_YYLEX_DECLARE[
+}
+
+]$1[
+]$2[
+
+%%
+
+start: $3;
+
+%%
+]AT_YYERROR_DEFINE[
+]AT_YYLEX_DEFINE([$4], [$5])[
+]AT_MAIN_DEFINE[
+]])
+
+AT_FULL_COMPILE([[test]])
+AT_PARSER_CHECK([./test], 0, [$6
+], [stderr])
+AT_BISON_OPTION_POPDEFS
+AT_CLEANUP
+])
+
+m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], [glr.cc]],
+ [# A built-in type.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type double],
+          [],
+          ['1' '2' { printf ("%2.1f\n", $1 + $2); }],
+          ["12"],
+          [AT_VAL = (res - '0') / 10.0],
+          [0.3])
+
+  # A user defined struct.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type "struct foo"],
+          [%code requires { struct foo { float fval; int ival; }; }],
+          ['1' '2'
+             { printf ("%d %2.1f\n", $1.ival + $2.ival, $1.fval + $2.fval); }],
+          ["12"],
+          [AT_VAL.ival = (res - '0') * 10;
+           AT_VAL.fval = (res - '0') / 10.f],
+          [30 0.3])
+
+  # A user defined union.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type "union foo"],
+          [%code requires { union foo { float fval; int ival; }; }],
+          ['1' '2' { printf ("%d %2.1f\n", $1.ival, $2.fval); }],
+          ["12"],
+          [if (res == '1')
+             AT_VAL.ival = 10;
+           else
+             AT_VAL.fval = .2f],
+          [10 0.2])
+
+  # A %union.
+  AT_TEST([%skeleton "]b4_skel["
+           %union { float fval; int ival; };],
+          [%token <ival> '1';
+           %token <fval> '2';],
+          ['1' '2' { printf ("%d %2.1f\n", $1, $2); }],
+          ["12"],
+          [if (res == '1')
+             AT_VAL.ival = 10;
+           else
+             AT_VAL.fval = 0.2f],
+          [10 0.2])
+
+  # A Bison-defined union.
+  # The tokens names are not available directly in C++, we use their
+  # user number to keep it simple between C and C++.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type union],
+          [%token <int> ONE 101;
+           %token <float> TWO 102 THREE 103;
+           %printer { ]AT_SKEL_CC_IF([[yyoutput << $$]],
+                                     [[fprintf (yyo, "%d", $$)]])[; } <int>
+           %printer { ]AT_SKEL_CC_IF([[yyoutput << $$]],
+                                     [[fprintf (yyo, "%f", $$)]])[; } <float>
+          ],
+          [ONE TWO THREE { printf ("%d %2.1f %2.1f\n", $1, $2, $3); }],
+          [{ 101, 102, 103, EOF }],
+          [if (res == 101)
+             AT_VAL.ONE = 10;
+           else if (res == 102)
+             AT_VAL.TWO = .2f;
+           else if (res == 103)
+             AT_VAL.THREE = 3.3f],
+          [10 0.2 3.3])
+
+  # A Bison-defined variant, for lalr1.cc only.
+  m4_if(b4_skel, [lalr1.cc], [
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type variant],
+          [%token <int> '1';
+           %token <std::string> '2';],
+          ['1' '2' { std::cout << $1 << ", " << $2 << std::endl; }],
+          ["12"],
+          [if (res == '1')
+             AT_VAL.build(10);
+           else
+             AT_VAL.build<std::string>("two");],
+          [10, two])])
+])
+
+m4_popdef([AT_TEST])