api.token.raw: implement

Bison used to feature %raw, documented as follows:

    @item %raw
    The output file @file{@var{name}.h} normally defines the tokens with
    Yacc-compatible token numbers.  If this option is specified, the
    internal Bison numbers are used instead.  (Yacc-compatible numbers start
    at 257 except for single character tokens; Bison assigns token numbers
    sequentially for all tokens starting at 3.)

Unfortunately, as far as I can tell, it never worked: token numbers
are indeed changed in the generated tables (from external token number
to internal), yet the code was still applying the mapping from
external token numbers to internal token numbers.

This commit reintroduces the feature as it was expected to be.

* data/skeletons/bison.m4 (b4_token_format): When api.token.raw is
enabled, use the internal token number.
* data/skeletons/yacc.c (yytranslate): Don't emit if api.token.raw is
enabled.
(YYTRANSLATE): Adjust.
This commit is contained in:
Akim Demaille
2019-08-24 08:09:58 -05:00
parent d94d83e10b
commit 9861bcc540
2 changed files with 6 additions and 3 deletions

View File

@@ -535,7 +535,7 @@ m4_define([b4_token_format],
[b4_token_visible_if([$2],
[m4_quote(m4_format([$1],
[b4_symbol([$2], [id])],
[b4_symbol([$2], [user_number])]))])])
[b4_symbol([$2], b4_api_token_raw_if([[number]], [[user_number]]))]))])])
## ------- ##
@@ -1002,6 +1002,7 @@ m4_define([b4_percent_code_ifdef],
# b4_parse_trace_if([IF-DEBUG-TRACES-ARE-ENABLED], [IF-NOT])
# b4_token_ctor_if([IF-YYLEX-RETURNS-A-TOKEN], [IF-NOT])
# ----------------------------------------------------------
b4_percent_define_if_define([api.token.raw])
b4_percent_define_if_define([token_ctor], [api.token.constructor])
b4_percent_define_if_define([locations]) # Whether locations are tracked.
b4_percent_define_if_define([parse.assert])

View File

@@ -608,7 +608,9 @@ union yyalloc
/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
as returned by yylex, with out-of-bounds checking. */
#define YYTRANSLATE(YYX) \
]b4_api_token_raw_if(dnl
[[#define YYTRANSLATE(YYX) (YYX)]],
[[#define YYTRANSLATE(YYX) \
((unsigned) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM
@@ -616,7 +618,7 @@ union yyalloc
static const ]b4_int_type_for([b4_translate])[ yytranslate[] =
{
]b4_translate[
};
};]])[
#if ]b4_api_PREFIX[DEBUG
]b4_integral_parser_table_define([rline], [b4_rline],