output: use the token list to define the yytokentype

There are currently two systems used to pass information about tokens to
m4: the original one, and another, which is used for instance for
printers and destructors, variants etc.  Move to using only the latter.

* data/bison.m4 (b4_symbol_map, b4_token_visible_if)
(b4_token_has_definition, b4_any_token_visible_if, b4_token_format): New.
* data/c++.m4, data/c.m4, data/glr.c, data/java.m4: Adjust to use them.

data/bison.m4

@@ -326,6 +326,13 @@ b4_define_flag_if([yacc])               # Whether POSIX Yacc is emulated.
 ## Symbols.  ##
 ## --------- ##
 
+# In order to unify the handling of the various aspects of symbols
+# (tag, type_name, whether terminal, etc.), bison.exe defines one
+# macro per (token, field), where field can has_id, id, etc.: see
+# src/output.c:prepare_symbols_definitions().
+#
+# The following macros provide access to these values.
+
 # b4_symbol_(NUM, FIELD)
 # ----------------------
 # Recover a FIELD about symbol #NUM.  Thanks to m4_indir, fails if
@@ -406,6 +413,45 @@ m4_define([b4_symbol_case_],
 m4_define([b4_symbol_foreach],
           [m4_map([$1], m4_defn([b4_symbol_numbers]))])
 
+# b4_symbol_map(MACRO)
+# --------------------
+# Return a list (possibly empty elements) of MACRO invoked for each
+# SYMBOL-NUM.
+m4_define([b4_symbol_map],
+[m4_map_args_sep([$1(], [)], [,], b4_symbol_numbers)])
+
+
+# b4_token_visible_if(NUM, IF-TRUE, IF-FALSE)
+# -------------------------------------------
+# Whether NUM denotes a token that has an exported definition (i.e.,
+# shows in enum yytokentype).
+m4_define([b4_token_visible_if],
+[b4_symbol_if([$1], [is_token],
+              [b4_symbol_if([$1], [has_id], [$2], [$3])],
+              [$3])])
+
+# b4_token_has_definition(NUM)
+# ----------------------------
+# 1 if NUM is visible, nothing otherwise.
+m4_define([b4_token_has_definition],
+[b4_token_visible_if([$1], [1])])
+
+# b4_any_token_visible_if([IF-TRUE], [IF-FALSE])
+# ----------------------------------------------
+# Whether there is a token that needs to be defined.
+m4_define([b4_any_token_visible_if],
+[m4_ifval(b4_symbol_foreach([b4_token_has_definition]),
+          [$1], [$2])])
+
+
+# b4_token_format(FORMAT, NUM)
+# ----------------------------
+m4_define([b4_token_format],
+[b4_token_visible_if([$2],
+[m4_quote(m4_format([$1],
+                     [b4_symbol([$2], [id])],
+                     [b4_symbol([$2], [user_number])]))])])
+
 
 ## ------- ##
 ## Types.  ##

data/c++.m4

@@ -86,16 +86,16 @@ m4_bpatsubst(m4_dquote(m4_bpatsubst(m4_dquote(b4_namespace_ref[ ]),
              [::\([^][:]\|:[^:]\)*], [} ])[} // ]b4_namespace_ref])])
 
 
-# b4_token_enums(LIST-OF-PAIRS-TOKEN-NAME-TOKEN-NUMBER)
+# b4_token_enums
-# -----------------------------------------------------
+# --------------
 # Output the definition of the tokens as enums.
 m4_define([b4_token_enums],
-[/* Tokens.  */
+[[enum yytokentype
-   enum yytokentype {
+      {
-m4_map_sep([     b4_token_enum], [,
+        ]m4_join([,
-],
+        ],
-           [$@])
+                 b4_symbol_map([b4_token_enum]))[
-   };
+      };]dnl
 ])
 
 
@@ -143,7 +143,7 @@ m4_define([b4_public_types_declare],
     /// Tokens.
     struct token
     {
-      ]b4_token_enums(b4_tokens)[
+      ]b4_token_enums[
     };
 
     /// Token type.

data/c.m4

@@ -250,57 +250,54 @@ static const b4_int_type_for([$2]) yy$1[[]] =
 ## Assigning token numbers.  ##
 ## ------------------------- ##
 
-# b4_token_define(TOKEN-NAME, TOKEN-NUMBER)
+# b4_token_define(TOKEN-NUM)
-# -----------------------------------------
+# --------------------------
 # Output the definition of this token as #define.
 m4_define([b4_token_define],
-[#define b4_percent_define_get([api.tokens.prefix])$1 $2
+[b4_token_format([#define %s %s], [$1])])
-])
 
-
+# b4_token_defines
-# b4_token_defines(LIST-OF-PAIRS-TOKEN-NAME-TOKEN-NUMBER)
+# ----------------
-# -------------------------------------------------------
+# Output the definition of the tokens.
-# Output the definition of the tokens (if there are) as #defines.
 m4_define([b4_token_defines],
-[m4_if([$#$1], [1], [],
+[b4_any_token_visible_if([/* Tokens.  */
-[/* Tokens.  */
+m4_join([
-m4_map([b4_token_define], [$@])])
+], b4_symbol_map([b4_token_define]))
-])
+])])
 
 
-# b4_token_enum(TOKEN-NAME, TOKEN-NUMBER)
+# b4_token_enum(TOKEN-NUM)
-# ---------------------------------------
+# ------------------------
 # Output the definition of this token as an enum.
 m4_define([b4_token_enum],
-[b4_percent_define_get([api.tokens.prefix])$1 = $2])
+[b4_token_format([%s = %s], [$1])])
 
 
-# b4_token_enums(LIST-OF-PAIRS-TOKEN-NAME-TOKEN-NUMBER)
+# b4_token_enums
-# -----------------------------------------------------
+# --------------
 # Output the definition of the tokens (if there are) as enums.
 m4_define([b4_token_enums],
-[m4_if([$#$1], [1], [],
+[b4_any_token_visible_if([[/* Tokens.  */
-[[/* Tokens.  */
 #ifndef ]b4_api_PREFIX[TOKENTYPE
 # define ]b4_api_PREFIX[TOKENTYPE
-   /* Put the tokens into the symbol table, so that GDB and other debuggers
+  /* Put the tokens into the symbol table, so that GDB and other debuggers
-      know about them.  */
+     know about them.  */
-   enum ]b4_api_prefix[tokentype {
+  enum ]b4_api_prefix[tokentype
-]m4_map_sep([     b4_token_enum], [,
+  {
-],
+    ]m4_join([,
-           [$@])
+    ],
-   };[
+             b4_symbol_map([b4_token_enum]))[
+  };
 #endif
 ]])])
 
 
-# b4_token_enums_defines(LIST-OF-PAIRS-TOKEN-NAME-TOKEN-NUMBER)
+# b4_token_enums_defines
-# -------------------------------------------------------------
+# ----------------------
-# Output the definition of the tokens (if there are any) as enums and, if POSIX
+# Output the definition of the tokens (if there are any) as enums and,
-# Yacc is enabled, as #defines.
+# if POSIX Yacc is enabled, as #defines.
 m4_define([b4_token_enums_defines],
-[b4_token_enums($@)b4_yacc_if([b4_token_defines($@)], [])
+[b4_token_enums[]b4_yacc_if([b4_token_defines])])
-])
 
 
 ## ----------------- ##

data/java.m4

@@ -142,23 +142,20 @@ m4_define([b4_integral_parser_table_define],
 ## Assigning token numbers.  ##
 ## ------------------------- ##
 
-# b4_token_enum(TOKEN-NAME, TOKEN-NUMBER)
+# b4_token_enum(TOKEN-NUM)
-# ---------------------------------------
+# ------------------------
 # Output the definition of this token as an enum.
 m4_define([b4_token_enum],
-[    /** Token number, to be returned by the scanner.  */
+[b4_token_format([    /** Token number, to be returned by the scanner.  */
-    static final int b4_percent_define_get([api.tokens.prefix])$1 = $2;
+    static final int %s = %s;
-])
+], [$1])])
 
-
+# b4_token_enums
-# b4_token_enums(LIST-OF-PAIRS-TOKEN-NAME-TOKEN-NUMBER)
+# --------------
-# -----------------------------------------------------
 # Output the definition of the tokens (if there are) as enums.
 m4_define([b4_token_enums],
-[m4_if([$#$1], [1], [],
+[b4_any_token_visible_if([/* Tokens.  */
-[/* Tokens.  */
+b4_symbol_foreach([b4_token_enum])])])
-m4_map([b4_token_enum], [$@])])
-])
 
 # b4-case(ID, CODE)
 # -----------------