version 3.7.2

* NEWS: Record release date.

.prev-version

@@ -1 +1 @@
-3.5.1
+3.7.1

.travis.yml

@@ -29,12 +29,13 @@ jobs:
         clone: true
       dist: bionic
       script:
-        - sudo apt-get install -qq autoconf automake autopoint flex gettext graphviz help2man m4 texinfo
+        - sudo apt-get install -qq autoconf automake autopoint flex gettext gperf graphviz help2man m4 texinfo
         - autoconf --version
         - automake --version
         - autopoint --version
         - dot -V
         - gettext --version
+        - gperf --version
         - help2man --version
         - makeinfo --version
         - m4 --version
@@ -46,9 +47,10 @@ jobs:
         - ./bootstrap
         - ./configure --enable-gcc-warnings || { cat config.log && false; }
         - make -j2
-        - make -j2 dist
+        - make -j2 dist-xz
         # Can help understanding why we get "dirty" tarballs.
         - git status
+        - git diff
         - dist=$(echo bison*.xz)
 
         # Unfortunately we cannot deterministically know the name of the tarball without the full
@@ -67,10 +69,122 @@ jobs:
     ## First.  ##
     ## ------- ##
 
-    # Start with three completely different environments, to get
-    # errors asap.
+    # Start with three completely different environments, to get errors asap.
 
-    - name: "GCC 9 -O3"
+    - name: "GCC 10 -O3"
+      stage: check
+      os: linux
+      dist: bionic
+      addons: &gcc10
+        apt:
+          sources:
+            # See https://github.com/travis-ci/apt-source-safelist/issues/410.
+            - sourceline: 'ppa:ubuntu-toolchain-r/test'
+          packages: g++-10
+      env:
+        - CC=gcc-10
+        - CXX=g++-10
+        - CONFIGUREFLAGS='CPPFLAGS=-DNDEBUG CFLAGS=-O3 CXXFLAGS=-O3'
+
+    # ASAN is time consuming, and we timeout the 50min granted by
+    # Travis if we run all the tests in one go.  Run in two parts.
+    - name: "Clang 10 libc++ and ASAN part 1"
+      stage: check
+      os: linux
+      dist: bionic
+      addons: &clang10
+        apt:
+          sources:
+            # See https://github.com/travis-ci/apt-source-safelist/issues/410.
+            - sourceline: 'ppa:ubuntu-toolchain-r/test'
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+          packages:
+            - clang-10
+            - libc++-10-dev
+            - libc++abi-10-dev
+      env:
+        # Do not use ASAN with ubuntu's libc++: https://bugs.llvm.org/show_bug.cgi?id=17379
+        - CC='clang-10 -fsanitize=address'
+        - CXX='clang++-10 -fsanitize=address -stdlib=libc++'
+        - PART=1
+
+    - name: "Clang 10 libc++ and ASAN part 2"
+      stage: check
+      os: linux
+      dist: bionic
+      addons: *clang10
+      env:
+        # Do not use ASAN with ubuntu's libc++: https://bugs.llvm.org/show_bug.cgi?id=17379
+        - CC='clang-10 -fsanitize=address'
+        - CXX='clang++-10 -fsanitize=address -stdlib=libc++'
+        - PART=2
+
+    # See https://github.com/simd-everywhere/simde/blob/master/.travis.yml
+    # and https://software.intel.com/content/www/us/en/develop/documentation/get-started-with-intel-oneapi-render-linux/top/configure-your-system.html.
+    - name: "ICC"
+      stage: check
+      os: linux
+      compiler: icc
+      env:
+        - CC=icc
+        - CXX=icpc
+      install:
+        - source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      addons:
+        apt:
+          sources:
+          - sourceline: 'deb https://apt.repos.intel.com/oneapi all main'
+            key_url: 'https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB'
+          packages:
+          - intel-oneapi-icc
+
+    - name: "ARM64: GCC 10 -O2 part 1"
+      stage: check
+      os: linux
+      arch: arm64
+      dist: bionic
+      addons: *gcc10
+      env:
+        - CC=gcc-10
+        - CXX=g++-10
+        - CONFIGUREFLAGS='CPPFLAGS=-DNDEBUG CFLAGS=-O2 CXXFLAGS=-O2'
+        - PART=1
+
+    - name: "PPC64le: GCC 10 -O2 part 1"
+      stage: check
+      os: linux
+      arch: ppc64le
+      dist: bionic
+      addons:
+        apt:
+          sources:
+            # See https://github.com/travis-ci/apt-source-safelist/issues/410.
+            - sourceline: 'ppa:ubuntu-toolchain-r/test'
+          packages: g++-10
+      env:
+        - CC=gcc-10
+        - CXX=g++-10
+        - CONFIGUREFLAGS='CFLAGS=-O2 CXXFLAGS=-O2'
+        - PART=1
+
+    - name: "s390x: GCC 10 -O2 part 1"
+      stage: check
+      os: linux
+      arch: s390x
+      dist: bionic
+      addons: *gcc10
+      env:
+        - CC=gcc-10
+        - CXX=g++-10
+        - CONFIGUREFLAGS='CFLAGS=-O2 CXXFLAGS=-O2'
+        - PART=1
+
+    ## ----- ##
+    ## GCC.  ##
+    ## ----- ##
+
+    - name: "GCC 9 with sanitizers part 1"
       stage: check
       os: linux
       dist: bionic
@@ -81,74 +195,8 @@ jobs:
             - sourceline: 'ppa:ubuntu-toolchain-r/test'
           packages: g++-9
       env:
-        - CC=gcc-9
-        - CXX=g++-9
-        - CONFIGUREFLAGS='CPPFLAGS=-DNDEBUG CFLAGS=-O3 CXXFLAGS=-O3'
-
-    # ASAN is time consuming, and we timeout the 50min granted by
-    # Travis if we run all the tests in one go.  Run in two parts.
-    - name: "Clang 9 libc++ and ASAN part 1"
-      stage: check
-      os: linux
-      dist: bionic
-      addons: &clang9
-        apt:
-          sources:
-            # See https://github.com/travis-ci/apt-source-safelist/issues/410.
-            - sourceline: 'ppa:ubuntu-toolchain-r/test'
-            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main'
-              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
-          packages:
-            - clang-9
-            - libc++-9-dev
-            - libc++abi-9-dev
-      env:
-        # Do not use ASAN with ubuntu's libc++: https://bugs.llvm.org/show_bug.cgi?id=17379
-        - CC='clang-9 -fsanitize=address'
-        - CXX='clang++-9 -fsanitize=address -stdlib=libc++'
-        - PART=1
-
-    - name: "Clang 9 libc++ and ASAN part 2"
-      stage: check
-      os: linux
-      dist: bionic
-      addons: *clang9
-      env:
-        # Do not use ASAN with ubuntu's libc++: https://bugs.llvm.org/show_bug.cgi?id=17379
-        - CC='clang-9 -fsanitize=address'
-        - CXX='clang++-9 -fsanitize=address -stdlib=libc++'
-        - PART=2
-
-    - name: "ICC"
-      stage: check
-      os: linux
-      compiler: icc
-      env:
-        - CC=icc
-        - CXX=icpc
-      install:
-        - source /opt/intel/inteloneapi/compiler/latest/env/vars.sh
-      addons:
-        apt:
-          sources:
-          - sourceline: 'deb https://apt.repos.intel.com/oneapi all main'
-            key_url: 'https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB'
-          packages:
-          - intel-oneapi-icc
-
-    ## ----- ##
-    ## GCC.  ##
-    ## ----- ##
-
-    - name: "GCC 8 with sanitizers part 1"
-      os: linux
-      dist: bionic
-      addons:
-        apt:
-          packages: g++-8
-      env:
-        - CC='gcc-8 -fsanitize=undefined,address -fno-omit-frame-pointer'
-        - CXX='g++-8 -fsanitize=undefined,address -fno-omit-frame-pointer'
+        - CC='gcc-9 -fsanitize=undefined,address -fno-omit-frame-pointer'
+        - CXX='g++-9 -fsanitize=undefined,address -fno-omit-frame-pointer'
         - CONFIGUREFLAGS='CFLAGS=-O1 CXXFLAGS=-O1'
         - PART=1
 
@@ -250,7 +298,27 @@ jobs:
     ## Clang.  ##
     ## ------- ##
 
-    - name: "Clang 8 -O3"
+    - name: "Clang 9 -O3"
+      stage: check
+      os: linux
+      dist: bionic
+      addons:
+        apt:
+          sources:
+            # See https://github.com/travis-ci/apt-source-safelist/issues/410.
+            - sourceline: 'ppa:ubuntu-toolchain-r/test'
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+          packages:
+            - clang-9
+            - libc++-9-dev
+            - libc++abi-9-dev
+      env:
+        - CC='clang-9'
+        - CXX='clang++-9 -stdlib=libc++'
+        - CONFIGUREFLAGS='CPPFLAGS=-DNDEBUG CFLAGS=-O3 CXXFLAGS=-O3'
+
+    - name: "Clang 8"
       stage: check
       os: linux
       dist: bionic
@@ -263,7 +331,6 @@ jobs:
       env:
         - CC=clang-8
         - CXX='clang++-8 -stdlib=libc++'
-        - CONFIGUREFLAGS='CPPFLAGS=-DNDEBUG CFLAGS=-O3 CXXFLAGS=-O3'
 
     - name: "Clang 7"
       stage: check
@@ -437,11 +504,11 @@ script:
   - sudo apt-get install -qq doxygen flex m4
   # Install and activate dmd.
   - mkdir -p ~/dlang && wget https://dlang.org/install.sh -O ~/dlang/install.sh
-  - source $(source ~/dlang/install.sh dmd -a)
+  - source $(source ~/dlang/install.sh dmd -a) || true
 
   - $CC --version
   - $CXX --version
-  - dmd --version
+  - dmd --version || true
   - doxygen --version
   - flex --version
   - ld --version

Makefile.am

@@ -42,7 +42,7 @@ dependencies = $(BISON_IN) $(src_bison_SOURCES) $(dist_skeletons_DATA)
 BISON = $(top_builddir)/tests/bison
 BISON_IN = $(top_srcdir)/tests/bison.in
 YACC = $(BISON) -o y.tab.c
-AM_YFLAGS_WITH_LINES = --defines -Werror -Wall --report=all
+AM_YFLAGS_WITH_LINES = --defines -Werror -Wall,dangling-alias --report=all
 AM_YFLAGS = $(AM_YFLAGS_WITH_LINES) --no-lines
 
 # Initialization before completion by local.mk's.
@@ -127,3 +127,15 @@ gen-ChangeLog:
 	    --since=$(gen_start_date) > $$cl.tmp &&	\
 	  mv -f $$cl.tmp $$cl;				\
 	fi
+
+# Useful to debug.
+.c.i:
+	$(CC) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -o $@ -E $<
+
+.PHONY: codespell
+codespell:
+	$(AM_V_GEN) cd $(srcdir) \
+	&& codespell \
+	     --regex "[\\w\\-'’\`]+\+*" \
+	     --ignore-words-list "ba,circularly,cloneable,copyable,define'd,dout,froms,iff,ifset,od,ois" \
+	     $$(git ls-files data doc lib src tests)

README

@@ -1,29 +1,52 @@
-This package contains the GNU Bison parser generator.
+GNU Bison is a general-purpose parser generator that converts an annotated
+context-free grammar into a deterministic LR or generalized LR (GLR) parser
+employing LALR(1) parser tables.  Bison can also generate IELR(1) or
+canonical LR(1) parser tables.  Once you are proficient with Bison, you can
+use it to develop a wide range of language parsers, from those used in
+simple desk calculators to complex programming languages.
+
+Bison is upward compatible with Yacc: all properly-written Yacc grammars
+work with Bison with no change.  Anyone familiar with Yacc should be able to
+use Bison with little trouble.  You need to be fluent in C, C++ or Java
+programming in order to use Bison.
+
+Bison and the parsers it generates are portable, they do not require any
+specific compilers.
+
+GNU Bison's home page is https://gnu.org/software/bison/.
 
 # Installation
 ## Build from git
-Here are basic installation instructions for a repository checkout:
+The [README-hacking.md file](README-hacking.md) is about building, modifying
+and checking Bison.  See its "Working from the Repository" section to build
+Bison from the git repo.  Roughly, run:
 
     $ git submodule update --init
     $ ./bootstrap
 
 then proceed with the usual `configure && make` steps.
 
-The file README-hacking.md is about building, modifying and checking Bison.
-
 ## Build from tarball
-See the file INSTALL for generic compilation and installation instructions.
+See the [INSTALL file](INSTALL] for generic compilation and installation
+instructions.
 
 Bison requires GNU m4 1.4.6 or later.  See
 https://ftp.gnu.org/gnu/m4/m4-1.4.6.tar.gz.
 
+## Running a non installed bison
+Once you ran `make`, you might want to toy with this fresh bison before
+installing it.  In that case, do not use `src/bison`: it would use the
+*installed* files (skeletons, etc.), not the local ones.  Use `tests/bison`.
+
 ## Colored diagnostics
 As an experimental feature, diagnostics are now colored, controlled by the
 `--color` and `--style` options.
 
-To use them, install the libtextstyle library before configuring Bison.  It
-is available from https://alpha.gnu.org/gnu/gettext/, for instance
-https://alpha.gnu.org/gnu/gettext/libtextstyle-0.8.tar.gz.
+To use them, install the libtextstyle library, 0.20.5 or newer, before
+configuring Bison.  It is available from https://alpha.gnu.org/gnu/gettext/,
+for instance https://alpha.gnu.org/gnu/gettext/libtextstyle-0.20.5.tar.gz,
+or as part of Gettext 0.21 or newer, for instance
+https://ftp.gnu.org/gnu/gettext/gettext-0.21.tar.gz.
 
 The option --color supports the following arguments:
 - always, yes: Enable colors.
@@ -40,6 +63,13 @@ To customize the styles, create a CSS file, say `bison-bw.css`, similar to
 then invoke bison with `--style=bison-bw.css`, or set the `BISON_STYLE`
 environment variable to `bison-bw.css`.
 
+In some diagnostics, bison uses libtextstyle to emit special escapes to
+generate clickable hyperlinks.  The environment variable
+`NO_TERM_HYPERLINKS` can be used to suppress them.  This may be useful for
+terminal emulators which produce garbage output when they receive the escape
+sequence for a hyperlink. Currently (as of 2020), this affects some versions
+of emacs, guake, konsole, lxterminal, rxvt, yakuake.
+
 ## Relocatability
 If you pass `--enable-relocatable` to `configure`, Bison is relocatable.
 
@@ -80,23 +110,9 @@ that the range specifies every single year in that closed interval.
 
 <!--
 
-Copyright (C) 1992, 1998-1999, 2003-2005, 2008-2015, 2018-2020 Free
-Software Foundation, Inc.
-
-This file is part of GNU bison, the GNU Compiler Compiler.
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+LocalWords:  parsers ngettext Texinfo pdf html YYYY ZZZZ ispell american md
+LocalWords:  MERCHANTABILITY GLR LALR IELR submodule init README src bw
+LocalWords:  Relocatability symlinks symlink
 
 Local Variables:
 mode: markdown
@@ -104,7 +120,16 @@ fill-column: 76
 ispell-dictionary: "american"
 End:
 
-LocalWords:  parsers ngettext Texinfo pdf html YYYY ZZZZ ispell american
-LocalWords:  MERCHANTABILITY
+Copyright (C) 1992, 1998-1999, 2003-2005, 2008-2015, 2018-2020 Free
+Software Foundation, Inc.
+
+This file is part of GNU bison, the GNU Compiler Compiler.
+
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
 -->

README-hacking.md

@@ -5,44 +5,8 @@ Everything related to the development of Bison is on Savannah:
 http://savannah.gnu.org/projects/bison/.
 
 
-# Administrivia
-
-## If you incorporate a change from somebody on the net:
-First, if it is a large change, you must make sure they have signed the
-appropriate paperwork.  Second, be sure to add their name and email address
-to THANKS.
-
-## If a change fixes a test, mention the test in the commit message.
-
-## Bug reports
-If somebody reports a new bug, mention his name in the commit message and in
-the test case you write.  Put him into THANKS.
-
-The correct response to most actual bugs is to write a new test case which
-demonstrates the bug.  Then fix the bug, re-run the test suite, and check
-everything in.
-
-
-# Hacking
-
-## Visible changes
-Which include serious bug fixes, must be mentioned in NEWS.
-
-## Translations
-Only user visible strings are to be translated: error messages, bits of the
-.output file etc.  This excludes impossible error messages (comparable to
-assert/abort), and all the --trace output which is meant for the maintainers
-only.
-
-## Horizontal tabs
-Do not add horizontal tab characters to any file in Bison's repository
-except where required.  For example, do not use tabs to format C code.
-However, make files, ChangeLog, and some regular expressions require tabs.
-Also, test cases might need to contain tabs to check that Bison properly
-processes tabs in its input.
-
-
-# Working from the repository
+Working from the Repository
+===========================
 
 These notes intend to help people working on the checked-out sources.  These
 requirements do not apply when building from a distribution tarball.
@@ -59,6 +23,7 @@ tools we depend upon, including:
 - Automake <http://www.gnu.org/software/automake/>
 - Flex <http://www.gnu.org/software/flex/>
 - Gettext <http://www.gnu.org/software/gettext/>
+- Gperf <http://www.gnu.org/software/gperf/>
 - Graphviz <http://www.graphviz.org>
 - Gzip <http://www.gnu.org/software/gzip/>
 - Help2man <http://www.gnu.org/software/help2man/>
@@ -75,7 +40,7 @@ above packages depends on your system.  The following shell command should
 work for Debian-based systems such as Ubuntu:
 
     sudo apt-get install \
-      autoconf automake autopoint flex graphviz help2man texinfo valgrind
+      autoconf automake autopoint flex gperf graphviz help2man texinfo valgrind
 
 Bison is written using Bison grammars, so there are bootstrapping issues.
 The bootstrap script attempts to discover when the C code generated from the
@@ -185,7 +150,231 @@ These files don't change very often in Autoconf, so it should be relatively
 straight-forward to examine the differences in order to decide whether to
 update.
 
-# Test Suite
+## Troubleshooting
+
+Bison is self-hosted: its parser is generated by Bison.  We don't force
+ourselves to use the previous release of Bison, we use the current git
+master for several reasons:
+- dogfooding: let Bison be its first user
+- monitoring: seeing the diff on the generated parsers with git is very
+  helpful, as it allows to see easily the impact of the changes on a real
+  case parser.
+
+If you are unlucky the generated files, src/parse-gram.[ch], may be older
+than their source, src/parse-gram.y.  And your current version of Bison
+might not be able to grok it.  In that case, first refresh the generated
+files:
+
+    $ touch src/parse-gram.[ch]
+
+Then proceed.
+
+In case you wrecked your current copy of the parser, get back the previous
+version, compile bison, then force it to recreate the files:
+
+    $ git checkout HEAD^ src/parse-gram.[ch]
+    $ make -C _build
+    $ touch src/parse-gram.y
+    $ make -C _build
+
+
+Administrivia
+=============
+
+## If you incorporate a change from somebody on the net:
+First, if it is a large change, you must make sure they have signed the
+appropriate paperwork.  Second, be sure to add their name and email address
+to THANKS.
+
+## If a change fixes a test, mention the test in the commit message.
+
+## Bug reports
+If somebody reports a new bug, mention his name in the commit message and in
+the test case you write.  Put him into THANKS.
+
+The correct response to most actual bugs is to write a new test case which
+demonstrates the bug.  Then fix the bug, re-run the test suite, and check
+everything in.
+
+
+
+Hacking
+=======
+
+## Visible Changes
+Which include serious bug fixes, must be mentioned in NEWS.
+
+## Translations
+Only user visible strings are to be translated: error messages, bits of the
+.output file etc.  This excludes impossible error messages (comparable to
+assert/abort), and all the --trace output which is meant for the maintainers
+only.
+
+## Vocabulary
+- "lookahead", not "look-ahead".
+- "midrule", not "mid-rule".
+- "nonterminal", not "variable" or "non-terminal" or "non terminal".
+  Abbreviated as "nterm".
+- "shift/reduce" and "reduce/reduce", not "shift-reduce" or "shift reduce",
+  etc.
+
+## Syntax Highlighting
+It's quite nice to be in C++ mode when editing lalr1.cc for instance.
+However tools such as Emacs will be fooled by the fact that braces and
+parens do not nest, as in `[[}]]`.  As a consequence you might be misguided
+by its visual pairing to parens.  The m4-mode is safer.  Unfortunately the
+m4-mode is also fooled by `#` which is sees as a comment, stops pairing with
+parens/brackets that are inside...
+
+## Implementation Notes
+There are several places with interesting details about the implementation:
+- [Understanding C parsers generated by GNU
+Bison](https://www.cs.uic.edu/~spopuri/cparser.html) by Satya Kiran Popuri,
+is a wonderful piece of work that explains the implementation of Bison,
+- [src/gram.h](src/gram.h) documents the way the grammar is represented
+- [src/tables.h](src/tables.h) documents the generated tables
+- [data/README.md](data/README.md) contains details about the m4 implementation
+
+## Coding Style
+Do not add horizontal tab characters to any file in Bison's repository
+except where required.  For example, do not use tabs to format C code.
+However, make files, ChangeLog, and some regular expressions require tabs.
+Also, test cases might need to contain tabs to check that Bison properly
+processes tabs in its input.
+
+Prefer `res` as the name of the local variable that will be "return"ed by
+the function.
+
+In writing arithmetic comparisons, use "<" and "<=" rather than ">" and ">="
+(http://www.gelato.unsw.edu.au/archives/git/0505/4507.html).
+
+### Bison
+Follow the GNU Coding Standards.
+
+Don't reinvent the wheel: we use gnulib, which features many components.
+Actually, Bison has legacy code that we should replace with gnulib modules
+(e.g., many ad hoc implementations of lists).
+
+#### Includes
+The `#include` directives follow an order:
+- first section for *.c files is `<config.h>`.  Don't include it in header
+  files
+- then, for *.c files, the corresponding *.h file
+- then possibly the `"system.h"` header
+- then the system headers.
+  Consider headers from `lib/` like system headers (i.e., `#include
+  <verify.h>`, not `#include "verify.h"`).
+- then headers from src/ with double quotes (`#include "getargs.h"`).
+
+Keep headers sorted alphabetically in each section.
+
+See also the [Header
+files](https://www.gnu.org/software/gnulib/manual/html_node/Header-files.html)
+and the [Implementation
+files](https://www.gnu.org/software/gnulib/manual/html_node/Implementation-files.html#Implementation-files)
+nodes of the gnulib documentation.
+
+Some source files are in the build tree (e.g., `src/scan-gram.c` made from
+`src/scan-gram.l`).  For them to find the headers from `src/`, we actually
+use `#include "src/getargs.h"` instead of `#include "getargs.h"`---that
+saves us from additional `-I` flags.
+
+### Skeletons
+We try to use the "typical" coding style for each language.
+
+#### CPP
+We indent the CPP directives this way:
+
+```
+#if FOO
+# if BAR
+#  define BAZ
+# endif
+#endif
+```
+
+Don't indent with leading spaces in the skeletons (it's OK in the grammar
+files though, e.g., in `%code {...}` blocks).
+
+On occasions, use `cppi -c` to see where we stand.  We don't aim at full
+correctness: depending `-d`, some bits can be in the *.c file, or the *.h
+file within the double-inclusion cpp-guards.  In that case, favor the case
+of the *.h file, but don't waste time on this.
+
+Don't hesitate to leave a comment on the `#endif` (e.g., `#endif /* FOO
+*/`), especially for long blocks.
+
+There is no consistency on `! defined` vs. `!defined`.  The day gnulib
+decides, we'll follow them.
+
+#### C/C++
+Follow the GNU Coding Standards.
+
+The `glr.c` skeleton was implemented with `camlCase`.  We are migrating it
+to `snake_case`.  Because we are gradually standardizing the code, it is
+currently inconsistent.
+
+Use `YYFOO` and `yyfoo` for entities that are exposed to the user.  They are
+part of our contract with the users wrt backward compatibility.
+
+Use `YY_FOO` and `yy_foo` for private matters.  Users should not use them,
+we are free to change them without fear of backward compatibility issues.
+
+Use `*_t` for types, especially for `yy*_t` in which case we shouldn't worry
+about the C standard introducing such a name.
+
+#### C++
+Follow the [C++ Core
+Guidelines](http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines).
+The [Google ones](https://google.github.io/styleguide/cppguide.html) may be
+interesting too.
+
+Our enumerators, such as the kinds (symbol and token kinds), should be lower
+case, but it was too late to follow that track for token kinds, and symbol
+kind enumerators are made to be consistent with them.
+
+Use `*_type` for type aliases.  Use `foo_get()` and `foo_set(v)` for
+accessors, or simply `foo()` and `foo(v)`.
+
+Use the `yy` prefix for private stuff, but there's no need for it in the
+public API.  The `yy` prefix is already taken care of via the namespace.
+
+#### Java
+We follow the [Java Code
+Conventions](https://www.oracle.com/technetwork/java/codeconventions-150003.pdf)
+and [Google Java Style
+Guide](https://google.github.io/styleguide/javaguide.html).  Unfortunately
+at some point some GNU Coding Style was installed in Java, but it's an
+error.  So we should for instance stop putting spaces in function calls.
+Because we are standardizing the code, it is currently inconsistent.
+
+Use a 2-space indentation (Google) rather than 4 (Oracle).
+
+Don't use the "yy" prefix for public members: "getExpectedTokens", not
+"yyexpectedTokens" or "yygetExpectedTokens".
+
+## Commit Messages
+Imitate the style we use.  Use `git log` to get sources of inspiration.
+
+If the changes have a small impact on Bison's generated parser, embed these
+changes in the commit itself.  If the impact is large, first push all the
+changes except those about src/parse-gram.[ch], and then another commit
+named "regen" which is only about them.
+
+## Debugging
+Bison supports tracing of its various steps, via the `--trace` option.
+Since it is not meant for the end user, it is not displayed by `bison
+--help`, nor is it documented in the manual.  Instead, run `bison
+--trace=help`.
+
+## Documentation
+Use `@option` for options and options with their argument if they have no
+space (e.g., `@option{-Dfoo=bar}`).  However, use `@samp` elsewhere (e.g.,
+`@samp{-I foo}`).
+
+
+Test Suite
+==========
 
 ## make check
 Consume without moderation.  It is composed of two kinds of tests: the
@@ -193,9 +382,9 @@ examples, and the main test suite.
 
 ### The Examples
 In examples/, there is a number of ready-to-use examples (see
-examples/README.md).  These examples have small test suites run by `make
-check`.  The test results are in local `*.log` files (e.g.,
-`$build/examples/c/calc/calc.log`).
+[examples/README.md](examples/README.md)).  These examples have small test
+suites run by `make check`.  The test results are in local `*.log` files
+(e.g., `$build/examples/c/calc/calc.log`).
 
 ### The Main Test Suite
 The main test suite, in tests/, is written on top of GNU Autotest, which is
@@ -255,7 +444,7 @@ added the `[-Wother]` part to all the warnings).  Part of the update can be
 done with a crude tool: `build-aux/update-test`.
 
 Once you ran the test suite, and therefore have many `testsuite.log` files,
-run, from the source tree:
+run `make update-tests`.  Or, by hand, from the *source* tree:
 
     $ ./build-aux/update-test $build/tests/testsuite.dir/*/testsuite.log
 
@@ -265,6 +454,37 @@ again, since some early failures may stop latter tests from being run.  Yet
 at some point, you'll have to fix remaining issues by hand...
 
 
+## Running Java parsers
+Use the `javaexec.sh` script.  For instance to run the parser of test case
+504:
+
+    $ sh ./_build/javaexec.sh -cp ./_build/tests/testsuite.dir/504 Calc
+
+## Using Sanitizers
+Address sanitizer (ASAN) and undefined-behavior sanitizer (UBSAN) are very
+useful.  Here's one way to set ASAN up with GCC 10 on Mac Ports
+
+1. Configure with
+
+    $ ./configure -C --enable-gcc-warnings \
+        CPPFLAGS='-isystem /opt/local/include' \
+        CC='gcc-mp-10 -fsanitize=address' \
+        CFLAGS='-ggdb' \
+        CXX='g++-mp-10.0 -fsanitize=address' \
+        CXXFLAGS='-ggdb' \
+        LDFLAGS='-L/opt/local/lib'
+
+2. Compile
+
+3. Generate debug symbols:
+
+    $ dsymutil src/bison
+
+4. Run the tests with leak detection enabled
+   (`ASAN_OPTIONS=detect_leaks=1`).  E.g. for counterexamples:
+
+    $ make check-local TESTSUITEFLAGS='-j5 -k cex' ASAN_OPTIONS=detect_leaks=1
+
 ## make maintainer-check-valgrind
 This target uses valgrind both to check bison, and the generated parsers.
 
@@ -339,171 +559,13 @@ re-run the tests, run:
     ./configure -C CC='gcc-mp-8 -fsanitize=undefined' CFLAGS='-ggdb'
     make check
 
-# Release Procedure
-This section needs to be updated to take into account features from gnulib.
-In particular, be sure to read README-release.
 
-## Update the submodules.  See above.
 
-## Update maintainer tools, such as Autoconf.  See above.
+Release Procedure
+=================
 
-## Try to get the *.pot files to the Translation Project at least one
-week before a stable release, to give them time to translate them.  Before
-generating the *.pot files, make sure that po/POTFILES.in and
-runtime-po/POTFILES.in list all files with translatable strings.  This
-helps: `grep -l '\<_(' *`.
-
-## Tests
-See above.
-
-## Update the foreign files
-Running `./bootstrap` in the top level should update them all for you.  This
-covers PO files too.  Sometimes a PO file contains problems that causes it
-to be rejected by recent Gettext releases; please report these to the
-Translation Project.
-
-## Update README
-Make sure the information in README is current.  Most notably, make sure it
-recommends a version of GNU M4 that is compatible with the latest Bison
-sources.
-
-## Check copyright years.
-We update years in copyright statements throughout Bison once at the start
-of every year by running `make update-copyright`.  However, before a
-release, it's good to verify that it's actually been run.  Besides the
-copyright statement for each Bison file, check the copyright statements that
-the skeletons insert into generated parsers, and check all occurrences of
-`PACKAGE_COPYRIGHT_YEAR` in configure.ac.
-
-## Update NEWS, commit and tag.
-See do-release-commit-and-tag in README-release.  For a while, we used beta
-names such as `2.6_rc1`.  Now that we use gnulib in the release procedure,
-we must use `2.5.90`, which has the additional benefit of being properly
-sorted in `git tag -l`.
-
-## make alpha, beta, or stable
-See README-release.
-
-## Upload
-There are two ways to upload the tarballs to the GNU servers: using gnupload
-(from gnulib), or by hand.  Obviously prefer the former.  But in either
-case, be sure to read the following paragraph.
-
-### Setup
-You need `gnupg`.
-
-Make sure your public key has been uploaded at least to keys.gnupg.net.  You
-can upload it with:
-
-  gpg --keyserver keys.gnupg.net --send-keys F125BDF3
-
-where F125BDF3 should be replaced with your key ID.
-
-### Using gnupload
-You need `ncftp`.
-
-At the end `make stable` (or alpha/beta) will display the procedure to run.
-Just copy and paste it in your shell.
-
-### By hand
-
-The generic GNU upload procedure is at
-http://www.gnu.org/prep/maintain/maintain.html#Automated-FTP-Uploads.
-
-Follow the instructions there to register your information so you're permitted
-to upload.
-
-Here's a brief reminder of how to roll the tarballs and upload them:
-
-### make distcheck
-### gpg -b bison-2.3b.tar.gz
-### In a file named `bison-2.3b.tar.gz.directive`, type:
-
-    version: 1.1
-    directory: bison
-    filename: bison-2.3b.tar.gz
-
-### gpg --clearsign bison-2.3b.tar.gz.directive
-### ftp ftp-upload.gnu.org # Log in as anonymous.
-### cd /incoming/alpha # cd /incoming/ftp for full release.
-### put bison-2.3b.tar.gz # This can take a while.
-### put bison-2.3b.tar.gz.sig
-### put bison-2.3b.tar.gz.directive.asc
-### Repeat all these steps for bison-2.3b.tar.xz.
-
-## Update Bison manual on www.gnu.org.
-
-The instructions below are obsolete, and left in case one would like to run
-the commands by hand.  Today, one just needs to run
-
-    $ make web-manual-update
-
-See README-release.
-
-### You need a non-anonymous checkout of the web pages directory.
-
-    $ cvs -d YOUR_USERID@cvs.savannah.gnu.org:/web/bison checkout bison
-
-### Get familiar with the instructions for web page maintainers.
-http://www.gnu.org/server/standards/readme_index.html
-http://www.gnu.org/server/standards/README.software.html
-especially the note about symlinks.
-
-### Build the web pages.
-Assuming BISON_CHECKOUT refers to a checkout of the Bison dir, and
-BISON_WWW_CHECKOUT refers to the web directory created above, do:
-
-    $ cd $BISON_CHECKOUT/doc
-    $ make stamp-vti
-    $ ../build-aux/gendocs.sh -o "$BISON_WWW_CHECKOUT/manual" \
-      bison "Bison - GNU parser generator"
-    $ cd $BISON_WWW_CHECKOUT
-
-Verify that the result looks sane.
-
-### Commit the modified and the new files.
-
-### Remove old files.
-Find the files which have not been overwritten (because they belonged to
-sections that have been removed or renamed):
-
-     $ cd manual/html_node
-     $ ls -lt
-
-Remove these files and commit their removal to CVS.  For each of these
-files, add a line to the file .symlinks.  This will ensure that hyperlinks
-to the removed files will redirect to the entire manual; this is better than
-a 404 error.
-
-## Announce
-The "make release" command just created a template,
-`$HOME/announce-bison-X.Y`.  Otherwise, to generate it, run:
-
-    make RELEASE_TYPE=alpha gpg_key_ID=F125BDF3 announcement
-
-where alpha can be replaced by `beta` or `table` and F125BDF3 should be
-replaced with your key ID.
-
-Complete/fix the announcement file.  The generated list of recipients
-(info-gnu@gnu.org, bison-announce@gnu.org, bug-bison@gnu.org,
-help-bison@gnu.org, bison-patches@gnu.org, and
-coordinator@translationproject.org) is appropriate for a stable release or a
-"serious beta".  For any other release, drop at least info-gnu@gnu.org.  For
-an example of how to fill out the rest of the template, search the mailing
-list archives for the most recent release announcement.
-
-For a stable release, send the same announcement on the comp.compilers
-newsgroup by sending email to compilers@iecc.com.  Do not make any Cc as the
-moderator will throw away anything cross-posted or Cc'ed.  It really needs
-to be a separate message.
-
-## Prepare NEWS
-So that developers don't accidentally add new items to the old NEWS entry,
-create a new empty entry in line 3 (without the two leading spaces):
-
-  * Noteworthy changes in release ?.? (????-??-??) [?]
-
-Push these changes.
+See the [README-release file](README-release), created when the package is
+bootstrapped.
 
 <!--
 
@@ -536,9 +598,13 @@ LocalWords:  submodule init cd distcheck ChangeLog valgrind sigreturn sudo
 LocalWords:  UC gcc DGNULIB POSIXCHECK xml XSLT glr lalr README po runtime rc
 LocalWords:  gnupload gnupg gpg keyserver BDF ncftp filename clearsign cvs dir
 LocalWords:  symlinks vti html lt POSIX Cc'ed Graphviz Texinfo autoconf jN
-LocalWords:  automake autopoint graphviz texinfo PROG Wother parsers
-LocalWords:  TESTSUITEFLAGS deprec struct gnulib's getopt config ggdb
-LocalWords:  bitset fsanitize symlink CFLAGS MERCHANTABILITY ispell
-LocalWords:  american
+LocalWords:  automake autopoint graphviz texinfo PROG Wother parsers YYFOO
+LocalWords:  TESTSUITEFLAGS deprec struct gnulib's getopt config ggdb yyfoo
+LocalWords:  bitset fsanitize symlink CFLAGS MERCHANTABILITY ispell wrt YY
+LocalWords:  american Administrivia camlCase yy accessors namespace src hoc
+LocalWords:  getExpectedTokens yyexpectedTokens yygetExpectedTokens parens
+LocalWords:  regen dogfooding Autotest testsuite getargs CPP BAZ endif cppi
+LocalWords:  cpp javaexec cp Calc ASAN UBSAN CPPFLAGS isystem CXX cex Gperf
+LocalWords:  CXXFLAGS LDFLAGS dsymutil gperf
 
 -->

THANKS

@@ -4,8 +4,10 @@ it is today without the invaluable help of these people:
 Aaro Koskinen             [email protected]
 Аскар Сафин               [email protected]
 Adam Sampson              [email protected]
+Adrian Vogelsgesang       [email protected]
+Ahcheong Lee              [email protected]
 Airy Andre                [email protected]
-Akim Demaille             akim@lrde.epita.fr
+Akim Demaille             akim@gnu.org
 Albert Chin-A-Young       [email protected]
 Alexander Belopolsky      [email protected]
 Alexandre Duret-Lutz      [email protected]
@@ -18,6 +20,7 @@ Anthony Heading           [email protected]
 Antonio Silva Correia     [email protected]
 Arnold Robbins            [email protected]
 Art Haas                  [email protected]
+Arthur Schwarz            [email protected]
 Askar Safin               [email protected]
 Balázs Scheidler          [email protected]
 Baron Schwartz            [email protected]
@@ -42,6 +45,7 @@ Csaba Raduly              [email protected]
 Dagobert Michelsen        [email protected]
 Daniel Frużyński          [email protected]
 Daniel Galloway           [email protected]
+Daniela Becker            [email protected]
 Daniel Hagerty            [email protected]
 David Barto               [email protected]
 David J. MacKenzie        [email protected]
@@ -55,11 +59,14 @@ Derek M. Jones            [email protected]
 Di-an Jan                 [email protected]
 Dick Streefland           [email protected]
 Didier Godefroy           [email protected]
+Don Macpherson            [email protected]
+Dwight Guth               [email protected]
 Efi Fogel                 [email protected]
 Enrico Scholz             [email protected]
 Eric Blake                [email protected]
 Eric S. Raymond           [email protected]
 Étienne Renault           [email protected]
+Evan Lavelle              [email protected]
 Evan Nemerson             [email protected]
 Evgeny Stambulchik        [email protected]
 Fabrice Bauzac            [email protected]
@@ -78,6 +85,7 @@ Guido Trentalancia        [email protected]
 H. Merijn Brand           [email protected]
 Hans Åberg                [email protected]
 Horst Von Brand           [email protected]
+Jacob L. Mandelson        [email protected]
 Jan Nieuwenhuizen         [email protected]
 Jannick                   [email protected]
 Jeff Hammond              [email protected]
@@ -98,14 +106,17 @@ Keith Browne              [email protected]
 Ken Moffat                [email protected]
 Kiyoshi Kanazawa          [email protected]
 Lars Maier                [email protected]
+Lars Wendler              [email protected]
 László Várady             [email protected]
 Laurent Mascherpa         [email protected]
 Lie Yan                   [email protected]
+Maarten De Braekeleer     [email protected]
 Magnus Fromreide          [email protected]
 Marc Autret               [email protected]
 Marc Mendiola             [email protected]
 Marc Schönefeld           [email protected]
 Mark Boyall               [email protected]
+Martin Blais              [email protected]
 Martin Jacobs             [email protected]
 Martin Mokrejs            [email protected]
 Martin Nylin              [email protected]
@@ -151,11 +162,13 @@ Petr Machata              [email protected]
 Pho                       [email protected]
 Piotr Gackiewicz          [email protected]
 Piotr Marcińczyk          [email protected]
+Pramod Kumbhar            [email protected]
 Quentin Hocquet           [email protected]
 Quoc Peyrot               [email protected]
 R Blake                   [email protected]
 Raja R Harinath           [email protected]
 Ralf Wildenhues           [email protected]
+Ryan                      [email protected]
 Rich Wilson               [email protected]
 Richard Stallman          [email protected]
 Rici Lake                 [email protected]
@@ -174,6 +187,7 @@ Simon Sobisch             [email protected]
 Stefano Lattarini         [email protected]
 Stephen Cameron           [email protected]
 Steve Murphy              [email protected]
+Suhwan Song               [email protected]
 Sum Wu                    [email protected]
 Théophile Ranquet         [email protected]
 Thiru Ramakrishnan        [email protected]
@@ -207,6 +221,7 @@ Wolfram Wagner            [email protected]
 Wwp                       [email protected]
 xolodho                   [email protected]
 Yuichiro Kaneko           [email protected]
+Yuriy Solodkyy            [email protected]
 Zack Weinberg             [email protected]
 江 祖铭                    [email protected]
 長田偉伸                   [email protected]

TODO

@@ -1,121 +1,166 @@
-* Bison 3.6
+* Soon
+** gnulib
+Bruno notes:
+
+> I haven't looked deeply, but it strikes me that gnulib/lib/bitset/array.c
+> does not make use of the 'ffsl' function, nor or the 'integer_length_l'
+> function. Maybe because in Bison, all bitsets are so dense that it does
+> not give a performance advantage?
+
+** Cex
+*** Improve gnulib
+Don't do this (counterexample.c):
+
+// This is the fastest way to get the tail node from the gl_list API.
+gl_list_node_t
+list_get_end (gl_list_t list)
+{
+  gl_list_node_t sentinel = gl_list_add_last (list, NULL);
+  gl_list_node_t res = gl_list_previous_node (list, sentinel);
+  gl_list_remove_node (list, sentinel);
+  return res;
+}
+
+*** Ambiguous rewriting
+If the user is stupid enough to have equal rules, then the derivations are
+harder to read:
+
+    Reduce/reduce conflict on tokens $end, "+", "⊕":
+        2 exp: exp "+" exp .
+        3 exp: exp "+" exp .
+      Example                  exp "+" exp •
+      First derivation         exp ::=[ exp "+" exp • ]
+      Example                  exp "+" exp •
+      Second derivation        exp ::=[ exp "+" exp • ]
+
+Do we care about this?  In color, we use twice the same color here, but we
+could try to use the same color for the same rule.
+
+*** XML reports
+Show the counterexamples.  This is going to be really hard and/or painful.
+Unless we play it dumb (little structure).
+
+** Bistromathic
+- How about not evaluating incomplete lines when the text is not finished
+  (as shells do).
+
+** Questions
+*** Java
+- Should i18n be part of the Lexer?  Currently it's a static method of
+  Lexer.
+
+- is there a migration path that would allow to use TokenKinds in
+  yylex?
+
+- define the tokens as an enum too.
+
+- promote YYEOF rather than EOF.
+
+** YYerror
+https://git.savannah.gnu.org/gitweb/?p=gettext.git;a=blob;f=gettext-runtime/intl/plural.y;h=a712255af4f2f739c93336d4ff6556d932a426a5;hb=HEAD
+
+should be updated to not use YYERRCODE.  Returning an undef token is good
+enough.
+
+** Java
+*** calc.at
+Stop hard-coding "Calc".  Adjust local.at (look for FIXME).
+
+** A dev warning for b4_
+Maybe we should check for m4_ and b4_ leaking out of the m4 processing, as
+Autoconf does.  It would have caught over-quotation issues.
+
 ** doc
-I feel its ugly to use the GNU style to declare functions in the doc.  It
+I feel it's ugly to use the GNU style to declare functions in the doc.  It
 generates tons of white space in the page, and may contribute to bad page
 breaks.
 
-Also, we seem to teach YYPRINT very early on, although it should be
-considered deprecated: %printer is superior.
-
-** improve syntax errors (UTF-8, internationalization)
-Bison depends on the current locale.  For instance:
-
-%define parse.error verbose
-%code top {
- #include <stdio.h>
- #include <stdlib.h>
- void yyerror(const char* msg) { fprintf(stderr, "%s\n", msg); }
- int yylex() { return 0; }
-}
-%%
-exp: "↦" | "🎅🐃" | '\n'
-%%
-int main() { return yyparse(); }
-
-gives different results with/without LC_ALL=C.
-
-$ LC_ALL=C /opt/local/bin/bison /tmp/mangle.y -o ascii.c
-$ /opt/local/bin/bison /tmp/mangle.y -o utf8.c
-$ diff -u ascii.c utf8.c -I#line
---- ascii.c     2019-01-12 08:15:35.878010093 +0100
-+++ utf8.c      2019-01-12 08:15:38.856495929 +0100
-@@ -415,9 +415,8 @@
-    First, the terminals, then, starting at YYNTOKENS, nonterminals.  */
- static const char *const yytname[] =
- {
--  "$end", "error", "$undefined", "\"\\342\\206\\246\"",
--  "\"\\360\\237\\216\\205\\360\\237\\220\\203\"", "'\\n'", "$accept",
--  "exp", YY_NULLPTR
-+  "$end", "error", "$undefined", "\"↦\"", "\"🎅🐃\"", "'\\n'",
-+  "$accept", "exp", YY_NULLPTR
- };
- #endif
-
-$ gcc ascii.c -o ascii && ./ascii
-syntax error, unexpected $end, expecting "\342\206\246" or "\360\237\216\205\360\237\220\203" or '\n'
-$ gcc utf8.c -o utf8 && ./utf8
-syntax error, unexpected $end, expecting ↦ or 🎅🐃 or '\n'
-
-
-While at it, we should stop using "$end" by default, in favor of "end of
-file", or "end of input", whatever.  See how lalr1.java does that.
-
 ** consistency
-token vs terminal, variable vs non terminal.
-
-** Stop indentation in diagnostics
-Before Bison 2.7, we printed "flatly" the dependencies in long diagnostics:
-
-    input.y:2.7-12: %type redeclaration for exp
-    input.y:1.7-12: previous declaration
-
-In Bison 2.7, we indented them
-
-    input.y:2.7-12: error: %type redeclaration for exp
-    input.y:1.7-12:     previous declaration
-
-Later we quoted the source in the diagnostics, and today we have:
-
-    /tmp/foo.y:1.12-14: warning: symbol FOO redeclared [-Wother]
-        1 | %token FOO FOO
-          |            ^~~
-    /tmp/foo.y:1.8-10:      previous declaration
-        1 | %token FOO FOO
-          |        ^~~
-
-The indentation is no longer helping.  We should probably get rid of it, or
-maybe keep it only when -fno-caret. GCC displays this as a "note":
-
-    $ g++-mp-9 -Wall /tmp/foo.c -c
-    /tmp/foo.c:1:10: error: redefinition of 'int foo'
-        1 | int foo, foo;
-          |          ^~~
-    /tmp/foo.c:1:5: note: 'int foo' previously declared here
-        1 | int foo, foo;
-          |     ^~~
-
-Likewise for Clang, contrary to what I believed (because "note:" is written
-in black, so it doesn't show in my terminal :-)
-
-    $ clang++-mp-8.0 -Wall /tmp/foo.c -c
-    clang: warning: treating 'c' input as 'c++' when in C++ mode, this behavior is deprecated [-Wdeprecated]
-    /tmp/foo.c:1:10: error: redefinition of 'foo'
-    int foo, foo;
-             ^
-    /tmp/foo.c:1:5: note: previous definition is here
-    int foo, foo;
-        ^
-    1 error generated.
-
-See also the item "Complaint submessage indentation" below.
+token vs terminal.
 
 ** api.token.raw
-Maybe we should exhibit the YYUNDEFTOK token.  It could also be assigned a
-semantic value so that yyerror could be used to report invalid lexemes.
-See also the item "$undefined" below.
+The YYUNDEFTOK could be assigned a semantic value so that yyerror could be
+used to report invalid lexemes.
 
-** C++
-Move to int everywhere instead of unsigned?  stack_size, etc.  The parser
-itself uses int (for yylen for instance), yet stack is based on size_t.
+** push parsers
+Consider deprecating impure push parsers.  They add a lot of complexity, for
+a bad feature.  On the other hand, that would make it much harder to sit
+push parsers on top of pull parser.  Which is currently not relevant, since
+push parsers are measurably slower.
 
-Maybe locations should also move to ints.
+** %define parse.error formatted
+How about pushing Bistromathic's yyreport_syntax_error as another standard
+way to generate the error message, and leave to the user the task of
+providing the message formats?  Currently in bistro, it reads:
 
-Paul Eggert already covered most of this.  But before publishing these
-changes, we need to ask our C++ users if they agree with that change, or if
-we need some migration path.  Could be a %define variable, or simply
-%require "3.5".
+    const char *
+    error_format_string (int argc)
+    {
+      switch (argc)
+        {
+        default: /* Avoid compiler warnings. */
+        case 0: return _("%@: syntax error");
+        case 1: return _("%@: syntax error: unexpected %u");
+          // TRANSLATORS: '%@' is a location in a file, '%u' is an
+          // "unexpected token", and '%0e', '%1e'... are expected tokens
+          // at this point.
+          //
+          // For instance on the expression "1 + * 2", you'd get
+          //
+          // 1.5: syntax error: expected - or ( or number or function or variable before *
+        case 2: return _("%@: syntax error: expected %0e before %u");
+        case 3: return _("%@: syntax error: expected %0e or %1e before %u");
+        case 4: return _("%@: syntax error: expected %0e or %1e or %2e before %u");
+        case 5: return _("%@: syntax error: expected %0e or %1e or %2e or %3e before %u");
+        case 6: return _("%@: syntax error: expected %0e or %1e or %2e or %3e or %4e before %u");
+        case 7: return _("%@: syntax error: expected %0e or %1e or %2e or %3e or %4e or %5e before %u");
+        case 8: return _("%@: syntax error: expected %0e or %1e or %2e or %3e or %4e or %5e or %6e before %u");
+        }
+    }
+
+The message would have to be generated in a string, and pushed to yyerror.
+Which will be a pain in the neck in yacc.c.
+
+If we want to do that, we should think very carefully about the syntax of
+the format string.
+
+** yyclearin does not invoke the lookahead token's %destructor
+https://lists.gnu.org/r/bug-bison/2018-02/msg00000.html
+Rici:
+
+> Modifying yyclearin so that it calls yydestruct seems like the simplest
+> solution to this issue, but it is conceivable that such a change would
+> break programs which already perform some kind of workaround in order to
+> destruct the lookahead symbol. So it might be necessary to use some kind of
+> compatibility %define, or to create a new replacement macro with a
+> different name such as yydiscardin.
+>
+> At a minimum, the fact that yyclearin does not invoke the %destructor
+> should be highlighted in the documentation, since it is not at all obvious.
+
+** Issues in i18n
+
+Les catégories d'avertissements incluent :
+  conflicts-sr      conflits S/R (activé par défaut)
+  conflicts-rr      conflits R/R (activé par défaut)
+  dangling-alias    l'alias chaîne n'est pas attaché à un symbole
+  deprecated        construction obsolète
+  empty-rule        règle vide sans %empty
+  midrule-values    valeurs de règle intermédiaire non définies ou inutilisées
+  precedence        priorité et associativité inutiles
+  yacc              incompatibilités avec POSIX Yacc
+  other             tous les autres avertissements (activé par défaut)
+  all               tous les avertissements sauf « dangling-alias » et « yacc »
+  no-CATEGORY       désactiver les avertissements dans CATEGORIE
+  none              désactiver tous les avertissements
+  error[=CATEGORY]  traiter les avertissements comme des erreurs
+
+Line -1 and -3 should mention CATEGORIE, not CATEGORY.
+
+* Bison 3.8
+** Rewrite glr.cc
+Get rid of scaffolding in glr.c.
 
-* Bison 3.7
 ** Unit rules / Injection rules (Akim Demaille)
 Maybe we could expand unit rules (or "injections", see
 https://homepages.cwi.nl/~daybuild/daily-books/syntax/2-sdf/sdf.html), i.e.,
@@ -158,18 +203,292 @@ The "automaton" and "set" categories are not so useful.  We should probably
 introduce lr(0) and lalr, just the way we have ielr categories.  The
 "closure" function is too verbose, it should probably have its own category.
 
-"set" can still be used for summariring the important sets.  That would make
+"set" can still be used for summarizing the important sets.  That would make
 tests easy to maintain.
 
 *** complain.*
 Rename these guys as "diagnostics.*" (or "diagnose.*"), since that's the
-name they have in gcc, clang, etc.  Likewise for the complain_* series of
+name they have in GCC, clang, etc.  Likewise for the complain_* series of
 functions.
 
 *** ritem
 states/nstates, rules/nrules, ..., ritem/nritems
 Fix the latter.
 
+* D programming language
+There's a number of features that are missing, here sorted in _suggested_
+order of implementation.
+
+When copying code from other skeletons, keep the comments exactly as they
+are.  Keep the same variable names.  If you change the wording in one place,
+do it in the others too.  In other words: make sure to keep the
+maintenance *simple* by avoiding any gratuitous difference.
+
+** Rename the D example
+Move the current content of examples/d into examples/d/simple.
+
+** Create a second example
+Duplicate examples/d/simple into examples/d/calc.
+
+** Add location tracking to d/calc
+Look at the examples in the other languages to see how to do that.
+
+** yysymbol_name
+The SymbolKind is an enum.  For a given SymbolKind we want to get its string
+representation.  Currently it's a separate table in the parser that does
+that:
+
+  /* Symbol kinds.  */
+  public enum SymbolKind
+  {
+    S_YYEMPTY = -2,  /* No symbol.  */
+    S_YYEOF = 0,                   /* "end of file"  */
+    S_YYerror = 1,                 /* error  */
+    S_YYUNDEF = 2,                 /* "invalid token"  */
+    S_EQ = 3,                      /* "="  */
+    ...
+    S_input = 14,                  /* input  */
+    S_line = 15,                   /* line  */
+    S_exp = 16,                    /* exp  */
+  };
+
+  ...
+
+  /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+     First, the terminals, then, starting at \a yyntokens_, nonterminals.  */
+  private static immutable string[] yytname_ =
+  [
+  "\"end of file\"", "error", "\"invalid token\"", "\"=\"", "\"+\"",
+  "\"-\"", "\"*\"", "\"/\"", "\"(\"", "\")\"", "\"end of line\"",
+  "\"number\"", "UNARY", "$accept", "input", "line", "exp", null
+  ];
+
+  ...
+
+So to get a symbol kind, one runs `yytname_[yykind]`.
+
+Is there a way to attach this conversion to string to SymbolKind?  In Java
+for instance, we have:
+
+  public enum SymbolKind
+  {
+    S_YYEOF(0),                    /* "end of file"  */
+    S_YYerror(1),                  /* error  */
+    S_YYUNDEF(2),                  /* "invalid token"  */
+    ...
+    S_input(16),                   /* input  */
+    S_line(17),                    /* line  */
+    S_exp(18);                     /* exp  */
+
+    private final int yycode_;
+
+    SymbolKind (int n) {
+      this.yycode_ = n;
+    }
+    ...
+    /* YYNAMES_[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+       First, the terminals, then, starting at \a YYNTOKENS_, nonterminals.  */
+    private static final String[] yynames_ = yynames_init();
+    private static final String[] yynames_init()
+    {
+      return new String[]
+      {
+        i18n("end of file"), i18n("error"), i18n("invalid token"), "!", "+", "-", "*",
+        "/", "^", "(", ")", "=", i18n("end of line"), i18n("number"), "NEG",
+        "$accept", "input", "line", "exp", null
+      };
+    }
+
+    /* The user-facing name of this symbol.  */
+    public final String getName() {
+      return yynames_[yycode_];
+    }
+  };
+
+which allows to write more naturally `yykind.getName()` rather than
+`yytname_[yykind]`.  Is there something comparable in (idiomatic) D?
+
+** Change the return value of yylex
+Historically people were allowed to return any int from the scanner (which
+is convenient and allows `return '+'` from the scanner).  Akim tends to see
+this as an error, we should restrict the return values to TokenKind (not to
+be confused with SymbolKind).
+
+In the case of D, without the history, we have the choice to support or not
+`int`.  If we want to _keep_ `int`, is there a way, say via introspection,
+to support both signatures of yylex?  If we don't keep `int`, just move to
+TokenKind.
+
+** Documentation
+Write documentation about D support in doc/bison.texi.  Imitate the Java
+documentation.  You should be more succinct IMHO.
+
+** Complete Symbols
+The current interface from the scanner to the parser is somewhat clumsy: the
+token kind is returned by yylex, but the value and location are stored in
+the scanner.  This reflects the fact that the implementation of the parser
+uses three variables to deal with each parsed symbol: its kind, its value,
+its location.
+
+So today the scanner of examples/d/calc.d (no locations) looks like:
+
+    if (input.front.isNumber)
+      {
+        import std.conv : parse;
+        semanticVal_.ival = input.parse!int;
+        return TokenKind.NUM;
+      }
+
+and the generated parser:
+
+    /* Read a lookahead token.  */
+    if (yychar == TokenKind.YYEMPTY)
+    {
+      yychar = yylex ();
+      yylval = yylexer.semanticVal;
+    }
+
+The parser class should feature a `Symbol` type which binds together kind,
+value and location, and the scanner should be able to return an instance of
+that type.  Something like
+
+    if (input.front.isNumber)
+      {
+        import std.conv : parse;
+        return parser.Symbol (TokenKind.NUM, input.parse!int);
+      }
+
+** Token Constructors
+In the previous example it is possible to mix incorrectly kinds and values,
+and for instance:
+
+        return parser.Symbol (TokenKind.NUM, "Hello, World!\n");
+
+attaches a string value to NUM kind (wrong, of course).  When
+api.token.constructor is set, in C++, Bison generated "token constructors":
+parser.make_NUM. parser.make_PLUS, parser.make_STRING, etc.  The previous
+example becomes
+
+        return parser.make_NUM ("Hello, World!\n");
+
+which would easily be caught by the type checker.
+
+** Lookahead Correction
+Add support for LAC to the D skeleton.  It should not be too hard: look how
+this is done in lalr1.cc, and mock it.
+
+** Push Parser
+Add support for push parser.  Do not start a nice skeleton, just enhance the
+current one to support push parsers.  This is going to be a tougher nut to
+crack.
+
+First, you need to understand well how the push parser is expected to work.
+To this end:
+- read the doc
+- look at examples/c/pushcalc
+- create an example of a Java push parser.
+- have a look at the generated parser in Java, which has the advantage of
+  being already based on a parser object, instead of just a function.
+
+The C case is harder to read, but it may help too.  Keep in mind that
+because there's no object to maintain state, the C push parser uses some
+struct (yypstate) to preserve this state.  We don't need this in D, the
+parser object will suffice.
+
+I think working directly on the skeleton to add push-parser support is not
+the simplest path.  I suggest that you (1) transform a generated parser into
+a push parser by hand, and then (2) transform lalr1.d to generate such a
+parser.
+
+Use `git commit` frequently to make sure you keep track of your progress.
+
+*** (1.a) Prepare pull parser by hand
+Copy again one of the D examples into say examples/d/pushcalc.  Also
+check-in the generated parser to facilitate experimentation.
+
+- find local variables of yyparse should become members of the parser object
+  (so that we preserve state from one call to the next).
+
+- do it in your generated D parser.  We don't need an equivalent for
+  yypstate, because we already have it: that the parser object itself.
+
+- have your *pull*-parser (i.e., the good old yy::parser::parse()) work
+  properly this way.  Write and run tests.  That's one of the reasons I
+  suggest using examples/d/calc as a starting point: it already has tests,
+  you can/should add more.
+
+At this point you have a pull-parser which you prepared to turn into a
+push-parser.
+
+*** (1.b) Turn pull parser into push parser by hand
+
+- look again at how push parsers are implemented in Java/C to see what needs
+  to change in yyparse so that the control is inverted: parse() will
+  be *given* the tokens, instead of having to call yylex itself.  When I say
+  "look at C", I think your best option are (i) yacc.c (look for b4_push_if)
+  and (ii) examples/c/pushcalc.
+
+- rename parse() as push_parse(Symbol yyla) (or push_parse(TokenKind, Value,
+  Location)) that takes the symbol as argument.  That's the push parser we
+  are looking for.
+
+- define a new parse() function which has the same signature as the usual
+  pull-parser, that repeatedly calls the push_parse function.  Something
+  like this:
+
+int parse ()
+{
+ int status = 0;
+ do {
+  status = this->push_parse (yylex());
+ } while (status == YYPUSH_MORE);
+ return status;
+}
+
+- show me that parser, so that we can validate the approach.
+
+*** (2) Port that into the skeleton
+- once we agree on the API of the push parser, implement it into lalr1.d.
+  You will probaby need help on this regard, but imitation, again, should
+  help.
+
+- have example/d/pushcalc work properly and pass tests
+
+- add tests in the "real" test suite.  Do that in tests/calc.at.  I can
+  help.
+
+- document
+
+** GLR Parser
+This is very ambitious.  That's the final boss.  There are currently no
+"clean" implementation to get inspiration from.
+
+glr.c is very clean but:
+- is low-level C
+- is a different skeleton from yacc.c
+
+glr.cc is (currently) an ugly hack: a C++ shell around glr.c.  Valentin
+Tolmer is currently rewriting glr.cc to be clean C++, but he is not
+finished.  There will be a lot a common code between lalr1.cc and glr.cc, so
+eventually I would like them to be fused into a single skeleton, supporting
+both deterministic and generalized parsing.
+
+It would be great for D to also support this.
+
+The basic ideas of GLR are explained here:
+
+https://www.codeproject.com/Articles/5259825/GLR-Parsing-in-Csharp-How-to-Use-The-Most-Powerful
+
+* Better error messages
+The users are not provided with enough tools to forge their error messages.
+See for instance "Is there an option to change the message produced by
+YYERROR_VERBOSE?" by Simon Sobisch, on bison-help.
+
+See also
+https://www.cs.tufts.edu/~nr/cs257/archive/clinton-jefferey/lr-error-messages.pdf
+https://research.swtch.com/yyerror
+http://gallium.inria.fr/~fpottier/publis/fpottier-reachability-cc2016.pdf
+
 * Modernization
 Fix data/skeletons/yacc.c so that it defines YYPTRDIFF_T properly for modern
 and older C++ compilers.  Currently the code defaults to defining it to
@@ -177,12 +496,12 @@ and older C++ compilers.  Currently the code defaults to defining it to
 define it to the same type as the C ptrdiff_t type.
 
 * Completion
-Several features are not available in all the backends.
+Several features are not available in all the back-ends.
 
-- push parsers: glr.cc, lalr1.cc
-- ielr: C++ and Java
-- glr: Java
-- token constructors: Java and C
+- lac: D, Java (easy)
+- push parsers: glr.c, glr.cc, lalr1.cc (not very difficult)
+- token constructors: Java, C, D (a bit difficult)
+- glr: D, Java (super difficult)
 
 * Bugs
 ** Autotest has quotation issues
@@ -198,8 +517,8 @@ $ ./tests/testsuite -l | grep errors | sed q
 Besides yytoknum is wrong when api.token.raw is defined.
 
 ** Better design for diagnostics
-The current implementation of diagnostics is adhoc, it grew organically.  It
-works as a series of calls to several functions, with dependency of the
+The current implementation of diagnostics is ad hoc, it grew organically.
+It works as a series of calls to several functions, with dependency of the
 latter calls on the former.  For instance:
 
       complain (&sym->location,
@@ -247,7 +566,7 @@ opposite side we have some use of \l, which is graphviz-specific, in what
 should be generic code.
 
 Little effort seems to have been given to factoring these files and their
-rint{,-xml} counterpart. We would very much like to re-use the pretty format
+print{,-xml} counterpart. We would very much like to re-use the pretty format
 of states from .output for the graphs, etc.
 
 Since graphviz dies on medium-to-big grammars, maybe consider an other tool?
@@ -314,52 +633,7 @@ It would be a very nice source of inspiration for the other languages.
 
 Valentin Tolmer is working on this.
 
-** YYERRCODE
-Defined to 256, but not used, not documented.  Probably the token
-number for the error token, which POSIX wants to be 256, but which
-Bison might renumber if the user used number 256.  Keep fix and doc?
-Throw away?
-
-Also, why don't we output the token name of the error token in the
-output?  It is explicitly skipped:
-
-      /* Skip error token and tokens without identifier.  */
-      if (sym != errtoken && id)
-
-Of course there are issues with name spaces, but if we disable we have
-something which seems to be more simpler and more consistent instead
-of the special case YYERRCODE.
-
-   enum yytokentype {
-     error = 256,
-     // ...
-   };
-
-
-We could (should?) also treat the case of the undef_token, which is
-numbered 257 for yylex, and 2 internal.  Both appear for instance in
-toknum:
-
-  const unsigned short
-  parser::yytoken_number_[] =
-  {
-       0,   256,   257,   258,   259,   260,   261,   262,   263,   264,
-
-while here
-
-   enum yytokentype {
-     TOK_EOF = 0,
-     TOK_EQ = 258,
-
-so both 256 and 257 are "mysterious".
-
-  const char*
-  const parser::yytname_[] =
-  {
-  "\"end of command\"", "error", "$undefined", "\"=\"", "\"break\"",
-
-
-** yychar == yyempty_
+** yychar == YYEMPTY
 The code in yyerrlab reads:
 
       if (yychar <= YYEOF)
@@ -391,14 +665,6 @@ We need trustworthy benchmarks for Bison, for all our backends.  Akim has a
 few things scattered around; we need to put them in the repo, and make them
 more useful.
 
-** yysyntax_error
-The code bw glr.c and yacc.c is really alike, we can certainly factor
-some parts.
-
-This should be worked on when we also address the expected improvements for
-error generation (e.g., i18n).
-
-
 * Report
 
 ** Figures
@@ -425,17 +691,16 @@ undocumented ''features''.  Maybe an empty action ought to be
 presented too.  Shall we try to make a single grammar with all these
 features, or should we have several very small grammars?
 
-** --report=conflict-path
-Provide better assistance for understanding the conflicts by providing
-a sample text exhibiting the (LALR) ambiguity.  See the paper from
-DeRemer and Penello: they already provide the algorithm.
-
-** Statically check for potential ambiguities in GLR grammars
-See <http://www.lsv.fr/~schmitz/pub/expamb.pdf> for an approach.
-An Experimental Ambiguity Detection Tool ∗ Sylvain Schmitz
-LORIA, INRIA Nancy - Grand Est, Nancy, France
-
 * Extensions
+** More languages?
+Well, only if there is really some demand for it.
+
+*** PHP
+https://github.com/scfc/bison-php/blob/master/data/lalr1.php
+
+*** Python
+https://lists.gnu.org/r/bison-patches/2013-09/msg00000.html and following
+
 ** Multiple start symbols
 Would be very useful when parsing closely related languages.  The idea is to
 declare several start symbols, for instance
@@ -458,15 +723,6 @@ happen with yy_start: stmt | expr).  Then adjust the skeletons so that this
 initial token (YY_START_STMT, YY_START_EXPR) be shifted first in the
 corresponding parse function.
 
-** Better error messages
-The users are not provided with enough tools to forge their error messages.
-See for instance "Is there an option to change the message produced by
-YYERROR_VERBOSE?" by Simon Sobisch, on bison-help.
-
-See also
-https://www.cs.tufts.edu/~nr/cs257/archive/clinton-jefferey/lr-error-messages.pdf
-and https://research.swtch.com/yyerror.
-
 ** %include
 This is a popular demand.  We already made many changes in the parser that
 should make this reasonably easy to implement.
@@ -525,13 +781,6 @@ XML output for GNU Bison and gcc
 XML output for GNU Bison
    http://yaxx.sourceforge.net/
 
-** Counterexample generation
-https://lists.gnu.org/archive/html/bug-bison/2016-06/msg00000.html
-http://www.cs.cornell.edu/andru/papers/cupex/
-
-Andrew Myers and Vincent Imbimbo are working on this item, see
-https://github.com/akimd/bison/issues/12
-
 * Coding system independence
 Paul notes:
 
@@ -563,15 +812,6 @@ move to partial orders (sounds like series/parallel orders to me).
 
 This is a prerequisite for modules.
 
-* $undefined
-From Hans:
-- If the Bison generated parser experiences an undefined number in the
-character range, that character is written out in diagnostic messages, an
-addition to the $undefined value.
-
-Suggest: Change the name $undefined to undefined; looks better in outputs.
-
-
 * Pre and post actions.
 From: Florian Krohm <[email protected]>
 Subject: YYACT_EPILOGUE
@@ -604,28 +844,47 @@ to bison. If you're interested, I'll work on a patch.
 Equip the parser with a means to create the (visual) parse tree.
 
 
+-----
+
+# LocalWords:  Cex gnulib gl Bistromathic TokenKinds yylex enum YYEOF EOF
+# LocalWords:  YYerror gettext af hb YYERRCODE undef calc FIXME dev yyerror
+# LocalWords:  Autoconf YYUNDEFTOK lexemes parsers Bistromathic's yyreport
+# LocalWords:  const argc yacc yyclearin lookahead destructor Rici incluent
+# LocalWords:  yydestruct yydiscardin catégories d'avertissements sr activé
+# LocalWords:  conflits défaut rr l'alias chaîne n'est attaché un symbole
+# LocalWords:  obsolète règle vide midrule valeurs de intermédiaire ou avec
+# LocalWords:  définies inutilisées priorité associativité inutiles POSIX
+# LocalWords:  incompatibilités tous les autres avertissements sauf dans rp
+# LocalWords:  désactiver CATEGORIE traiter comme des erreurs glr Akim bool
+# LocalWords:  Demaille arith lalr goto struct pathlen nullable ntokens lr
+# LocalWords:  nterm bitsetv ielr ritem nstates nrules nritems yysymbol EQ
+# LocalWords:  SymbolKind YYEMPTY YYUNDEF YYTNAME NUM yyntokens yytname sed
+# LocalWords:  nonterminals yykind yycode YYNAMES yynames init getName conv
+# LocalWords:  TokenKind semanticVal ival yychar yylval yylexer Tolmer hoc
+# LocalWords:  Sobisch YYPTRDIFF ptrdiff Autotest YYPRINT toknum yytoknum
+# LocalWords:  sym Wother stderr FP fixits xgettext fdiagnostics Graphviz
+# LocalWords:  graphviz VCG bitset xml bw maint yytoken YYABORT deps
+# LocalWords:  YYACCEPT yytranslate nonnegative destructors yyerrlab repo
+# LocalWords:  backends stmt expr yy Mardle baz qux Vadim Maslow CPP cpp
+# LocalWords:  yydebug gcc UCHAR EBCDIC gung PDP NUL Pre Florian Krohm utf
+# LocalWords:  YYACT YYLLOC YYLSP yyval yyvsp yylen yyloc yylsp endif
+# LocalWords:  ispell american
+
 Local Variables:
 mode: outline
 coding: utf-8
 fill-column: 76
+ispell-dictionary: "american"
 End:
 
------
-
 Copyright (C) 2001-2004, 2006, 2008-2015, 2018-2020 Free Software
 Foundation, Inc.
 
 This file is part of Bison, the GNU Compiler Compiler.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.

bootstrap

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2019-01-04.17; # UTC
+scriptversion=2020-04-13.15; # UTC
 
 # Bootstrap this package from checked-out sources.
 
@@ -970,7 +970,7 @@ bootstrap_post_import_hook \
 # Uninitialized submodules are listed with an initial dash.
 if $use_git && git submodule | grep '^-' >/dev/null; then
   die "some git submodules are not initialized. "     \
-      "Run 'git submodule init' and bootstrap again."
+      "Run 'git submodule update --init' and bootstrap again."
 fi
 
 # Remove any dangling symlink matching "*.m4" or "*.[ch]" in some

bootstrap.conf

@@ -17,7 +17,7 @@
 
 # gnulib modules used by this package.
 gnulib_modules='
-  argmatch array-list assert assure
+  argmatch array-list assert assure attribute
   bitsetv
   calloc-posix close closeout config-h c-strcase
   configmake
@@ -30,6 +30,7 @@ gnulib_modules='
   javaexec-script
   ldexpl
   libtextstyle-optional
+  linked-list
   malloc-gnu
   mbfile mbswidth
   non-recursive-gnulib-prefix-hack
@@ -37,13 +38,17 @@ gnulib_modules='
   obstack-printf
   perror progname
   quote quotearg
+  rbtreehash-list
+  readline
   readme-release
   realloc-posix
   relocatable-prog relocatable-script
   rename
-  spawn-pipe stdbool stpcpy strdup-posix strerror strverscmp
+  spawn-pipe stdbool stpcpy stpncpy strdup-posix strerror strverscmp
+  sys_ioctl
+  termios
   timevar
-  unistd unistd-safer unlink unlocked-io
+  unicodeio unistd unistd-safer unlink unlocked-io
   update-copyright unsetenv verify
   warnings
   winsz-ioctl

build-aux/cross-options.pl

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
-# Written by Akimn Demaille.
+# Written by Akim Demaille.
 
 use warnings;
 use 5.005;

build-aux/update-test

@@ -73,6 +73,11 @@ def diff_to_re(match):
             is_diff = True
             to.append(l)
     if is_diff:
+        # Do not run s//SOMETHING/g (with an emty pattern), that won't
+        # work well...
+        if frm == []:
+            trace("no from for", match.group(1))
+            return
         frm = "\n".join(frm)
         to = "\n".join(to)
         subst[frm] = to
@@ -102,7 +107,7 @@ def update(at_file, logfile):
 def process(logfile):
     log = contents(logfile)
     # Look for the file to update.
-    m = re.search(r'^\d+\. ([-\w]+\.at):\d+: ', log, re.MULTILINE)
+    m = re.search(r'^\d+\. ([\-\+\w]+\.at):\d+: ', log, re.MULTILINE)
     if not m:
         trace("no diff found:", logfile)
         return
@@ -112,4 +117,6 @@ def process(logfile):
 
 for logfile in args.logs:
     trace("FILE:", logfile)
+    if os.path.isdir(logfile):
+        logfile = os.path.join(logfile, 'testsuite.log')
     process(logfile)

cfg.mk

@@ -51,7 +51,7 @@ gnulib_dir = $(srcdir)/gnulib
 bootstrap-tools = autoconf,automake,flex,gettext,gnulib
 
 announcement_Cc_ = \
-  [email protected], [email protected], bison-patches@gnu.org, \
+  [email protected], bison-announce@gnu.org, \
   [email protected]
 
 update-copyright: update-b4-copyright update-package-copyright-year
@@ -126,7 +126,7 @@ _sed_rm_comments_q = $(subst ','\'',$(_sed_remove_comments))
 
 _space_before_paren_exempt =? \\n\\$$
 _space_before_paren_exempt = \
-  (^ *\#|(LA)?LR\([01]\)|percent_(code|define)|b4_syncline|m4_(define|init)|symbol)
+  (^ *\#|(LA)?LR\([01]\)|percent_(code|define)|b4_syncline|m4_(define|init))
 # Ensure that there is a space before each open parenthesis in C code.
 sc_space_before_open_paren:
 	@if $(VC_LIST_EXCEPT) | grep -l '\.[ch]$$' > /dev/null; then	\
@@ -156,7 +156,8 @@ exclude = \
 $(call exclude,                                                                 \
   bindtextdomain=^lib/main.c$$                                                  \
   cast_of_argument_to_free=^src/muscle-tab.c$$                                  \
-  po_check=(^po/POTFILES.in|.md)$$		                                \
+  error_message_uppercase=etc/bench.pl.in$$                                     \
+  po_check=^tests|(^po/POTFILES.in|.md)$$                                       \
   preprocessor_indentation=^data/|^lib/|^src/parse-gram.[ch]$$                  \
   program_name=^lib/main.c$$                                                    \
   prohibit_always-defined_macros=^data/skeletons/yacc.c$$                       \
@@ -165,7 +166,7 @@ $(call exclude,
   prohibit_doubled_word=^tests/named-refs.at$$                                  \
   prohibit_magic_number_exit=^doc/bison.texi$$                                  \
   prohibit_magic_number_exit+=?|^tests/(conflicts|regression).at$$              \
-  prohibit_strcmp=^doc/bison\.texi|examples|tests/local\.at$$                   \
+  prohibit_strcmp=^doc/bison\.texi|examples|tests                               \
   prohibit_tab_based_indentation=install-icc.sh|\.(am|mk)$$|^\.git|tests/input.at|Makefile$$   \
   require_config_h=^(lib/yyerror|data/skeletons/(glr|yacc))\.c$$                \
   require_config_h_first=^(lib/yyerror|data/skeletons/(glr|yacc))\.c$$          \

configure.ac

@@ -36,10 +36,8 @@ AC_DEFINE_UNQUOTED([PACKAGE_COPYRIGHT_YEAR], [$PACKAGE_COPYRIGHT_YEAR],
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_MACRO_DIR([m4])
 
-# We use Automake 1.14's %D% and %C%.
-#
 # When we move to Automake 1.16, simplify examples/c/reccalc/local.mk.
-# Our CI runs on Xenial, which has only Automake 1.15.
+# Our CI build the packages on Bionic, which has only Automake 1.15.
 #
 # We want gnits strictness only when rolling a stable release.  For
 # release candidates, we use version strings like 2.4.3_rc1, but gnits
@@ -47,7 +45,7 @@ AC_CONFIG_MACRO_DIR([m4])
 # releases, we want to be able run make dist without being required to
 # add a bogus NEWS entry.  In that case, the version string
 # automatically contains a dash, which we also let disable gnits.
-AM_INIT_AUTOMAKE([1.14 dist-xz nostdinc
+AM_INIT_AUTOMAKE([1.15 dist-lzip dist-xz nostdinc
                  color-tests parallel-tests
                  silent-rules]
                  m4_bmatch(m4_defn([AC_PACKAGE_VERSION]), [[-_]],
@@ -62,6 +60,9 @@ AC_PROG_CXX
 # Gnulib (early checks).
 gl_EARLY
 
+# We want ostream_printf and hyperlink support.
+gl_LIBTEXTSTYLE_OPTIONAL([0.20.5])
+
 # Gnulib uses '#pragma GCC diagnostic push' to silence some
 # warnings, but older gcc doesn't support this.
 AC_CACHE_CHECK([whether pragma GCC diagnostic push works],
@@ -134,10 +135,12 @@ if test "$enable_gcc_warnings" = yes; then
     -fparse-all-comments -Wdocumentation
     -Wformat -Wimplicit-fallthrough -Wnull-dereference
     -Wno-sign-compare -Wno-tautological-constant-out-of-range-compare
-    -Wpointer-arith -Wshadow
+    -Wpointer-arith -Wshadow -Wstrict-aliasing
     -Wwrite-strings
     -wr188 -wr2259 -wr3179'
-  warn_c='-Wbad-function-cast -Wstrict-prototypes'
+  warn_c='-Wbad-function-cast
+    -Wmissing-prototypes
+    -Wstrict-prototypes'
   warn_cxx='-Wextra-semi -Wnoexcept -Wold-style-cast -Wundefined-func-template
     -Wweak-vtables'
   # Warnings for the test suite only.
@@ -261,9 +264,9 @@ if ! "$LEX_IS_FLEX" || test "X$LEX" = X:; then
   AC_MSG_WARN([bypassing lex because flex is required])
   LEX=:
 fi
-AM_CONDITIONAL([FLEX_WORKS], [$LEX_IS_FLEX])
+AM_CONDITIONAL([FLEX_WORKS], [$LEX_IS_FLEX && $FLEX_SUPPORTS_HEADER_OPT])
 AM_CONDITIONAL([FLEX_CXX_WORKS],
-  [$LEX_IS_FLEX && test $bison_cv_cxx_works = yes])
+  [$LEX_IS_FLEX && $FLEX_SUPPORTS_HEADER_OPT && test $bison_cv_cxx_works = yes])
 AC_PROG_YACC
 AC_PROG_RANLIB
 AC_PROG_GNU_M4
@@ -332,6 +335,13 @@ AC_MSG_RESULT([$suppfile])
 # Whether we cannot run the compiled bison.
 AM_CONDITIONAL([CROSS_COMPILING], [test "$cross_compiling" = yes])
 
+# Build bistromathic if we have the lib and both readline/readline.h
+# and readline/history.h.
+AM_CONDITIONAL([ENABLE_BISTROMATHIC],
+[test "$gl_cv_lib_readline" != no &&
+  test "$ac_cv_header_readline_readline_h" = yes &&
+  test "$ac_cv_header_readline_history_h" = yes])
+
 AM_MISSING_PROG([AUTOM4TE], [autom4te])
 # Needed by tests/atlocal.in.
 AC_SUBST([GCC])

data/README.md

@@ -1,6 +1,6 @@
 This directory contains data needed by Bison.
 
-# Directory content
+# Directory Content
 ## Skeletons
 Bison skeletons: the general shapes of the different parser kinds, that are
 specialized for specific grammars by the bison program.
@@ -48,7 +48,7 @@ various formats.
 - xml2xhtml.xsl
   Conversion into XHTML.
 
-# Implementation note about the skeletons
+# Implementation Notes About the Skeletons
 
 "Skeleton" in Bison parlance means "backend": a skeleton is fed by the bison
 executable with LR tables, facts about the symbols, etc. and they generate
@@ -83,26 +83,39 @@ field), where field can `has_id`, `id`, etc.: see
 
 The macro `b4_symbol(NUM, FIELD)` gives access to the following FIELDS:
 
-- `has_id`: 0 or 1.
+- `has_id`: 0 or 1
+  Whether the symbol has an `id`.
 
-  Whether the symbol has an id.
+- `id`: string (e.g., `exp`, `NUM`, or `TOK_NUM` with api.token.prefix)
+  If `has_id`, the name of the token kind (prefixed by api.token.prefix if
+  defined), otherwise empty.  Guaranteed to be usable as a C identifier.
+  This is used to define the token kind (i.e., the enum used by the return
+  value of yylex).  Should be named `token_kind`.
 
-- `id`: string
-  If has_id, the id (prefixed by api.token.prefix if defined), otherwise
-  defined as empty.  Guaranteed to be usable as a C identifier.
+- `tag`: string
+  A human readable representation of the symbol.  Can be `'foo'`,
+  `'foo.id'`, `'"foo"'` etc.
 
-- `tag`: string.
-  A representation of the symbol.  Can be 'foo', 'foo.id', '"foo"' etc.
-
-- `user_number`: integer
+- `code`: integer
+  The token code associated to the token kind `id`.
   The external number as used by yylex.  Can be ASCII code when a character,
-  some number chosen by bison, or some user number in the case of
-  %token FOO <NUM>.  Corresponds to yychar in yacc.c.
+  some number chosen by bison, or some user number in the case of `%token
+  FOO <NUM>`.  Corresponds to `yychar` in `yacc.c`.
 
 - `is_token`: 0 or 1
   Whether this is a terminal symbol.
 
+- `kind_base`: string (e.g., `YYSYMBOL_exp`, `YYSYMBOL_NUM`)
+  The base of the symbol kind, i.e., the enumerator of this symbol (token or
+  nonterminal) which is mapped to its `number`.
+
+- `kind`: string
+  Same as `kind_base`, but possibly with a prefix in some languages.  E.g.,
+  EOF's `kind_base` and `kind` are `YYSYMBOL_YYEOF` in C, but are
+  `S_YYEMPTY` and `symbol_kind::S_YYEMPTY` in C++.
+
 - `number`: integer
+  The code associated to the `kind`.
   The internal number (computed from the external number by yytranslate).
   Corresponds to yytoken in yacc.c.  This is the same number that serves as
   key in b4_symbol(NUM, FIELD).
@@ -166,7 +179,7 @@ The data corresponding to the symbol `#POS`, where the current rule has
 Expansion of `$<TYPE>POS`, where the current rule has `RULE-LENGTH` symbols
 on RHS.
 
------
+<!--
 
 Local Variables:
 mode: markdown
@@ -190,3 +203,5 @@ GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+-->

data/bison-default.css

@@ -29,3 +29,33 @@
 
 /* "Sections" in traces (--trace).  */
 .trace0    { color: green; }
+
+/* Syntax error messages.  */
+.expected   { color: green; }
+.unexpected { color: red; }
+
+
+/* Counterexamples.  */
+
+/* Cex: point in rule.  */
+.cex-dot { color: red; }
+
+/* Cex: coloring various rules.  */
+.cex-0   { color: yellow; }
+.cex-1   { color: green; }
+.cex-2   { color: blue; }
+.cex-3   { color: purple; }
+.cex-4   { color: violet; }
+.cex-5   { color: orange; }
+.cex-6   { color: brown; }
+.cex-7   { color: mauve; }
+.cex-8   { color: #013220; } /* Dark green. */
+.cex-9   { color: #e75480; } /* Dark pink. */
+.cex-10  { color: cyan; }
+.cex-11  { color: orange; }
+
+/* Cex: derivation rewriting steps.  */
+.cex-step { font-style: italic; }
+
+/* Cex: leaves of a derivation.  */
+.cex-leaf { font-weight: 600; }

data/local.mk

@@ -34,6 +34,7 @@ dist_skeletons_DATA =                           \
   data/skeletons/lalr1.java                     \
   data/skeletons/location.cc                    \
   data/skeletons/stack.hh                       \
+  data/skeletons/traceon.m4                     \
   data/skeletons/variant.hh                     \
   data/skeletons/yacc.c
 

data/skeletons/README-D.txt

@@ -27,7 +27,7 @@ public interface Lexer
    * to the next token and prepares to return the semantic value
    * and beginning/ending positions of the token.
    * @return the token identifier corresponding to the next token. */
-  YYTokenType yylex ();
+  TokenKind yylex ();
 
   /**
    * Entry point for error reporting.  Emits an error
@@ -39,7 +39,7 @@ public interface Lexer
    void yyerror (YYLocation loc, string s);
 }
 
-- semantic types are handled by D usions (same as for C/C++ parsers)
+- semantic types are handled by D unions (same as for C/C++ parsers)
 
 - the following (non-standard) %defines are supported:
 

data/skeletons/bison.m4

@@ -19,6 +19,29 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+
+# m4_gsub(STRING, RE1, SUBST1, RE2, SUBST2, ...)
+# ----------------------------------------------
+# m4 equivalent of
+#
+#   $_ = STRING;
+#   s/RE1/SUBST1/g;
+#   s/RE2/SUBST2/g;
+#   ...
+#
+# Really similar to m4_bpatsubsts, but behaves properly with quotes.
+# See m4.at's "Generating Comments".  Super inelegant, but so far, I
+# did not find any better solution.
+m4_define([b4_gsub],
+[m4_bpatsubst(m4_bpatsubst(m4_bpatsubst([[[[$1]]]],
+                                        [$2], [$3]),
+                           [$4], [$5]),
+              [$6], [$7])])
+
+# m4_shift2 and m4_shift3 are provided by m4sugar.
+m4_define([m4_shift4], [m4_shift(m4_shift(m4_shift(m4_shift($@))))])
+
+
 ## ---------------- ##
 ## Identification.  ##
 ## ---------------- ##
@@ -72,8 +95,9 @@ version 2.2 of Bison.])
 # -------------
 # Issue a warning about private implementation details.
 m4_define([b4_disclaimer],
-[b4_comment([Undocumented macros, especially those whose name start with YY_,
-are private implementation details.  Do not rely on them.])
+[b4_comment([DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual,
+especially those whose name start with YY_ or yy_.  They are
+private implementation details that can be changed or removed.])
 ])
 
 
@@ -161,7 +185,7 @@ m4_define([b4_error],
 [b4_cat([[@complain][(]$1[@,]$2[@,]$3[@,]$4[]]dnl
 [m4_if([$#], [4], [],
        [m4_foreach([b4_arg],
-                   m4_dquote(m4_shift(m4_shift(m4_shift(m4_shift($@))))),
+                   m4_dquote(m4_shift4($@)),
                    [[@,]b4_arg])])[@)]])])
 
 # b4_warn(FORMAT, [ARG1], [ARG2], ...)
@@ -369,6 +393,7 @@ m4_define([b4_$3_if],
 # Expand IF-TRUE, if FLAG is true, IF-FALSE otherwise.
 b4_define_flag_if([defines])            # Whether headers are requested.
 b4_define_flag_if([glr])                # Whether a GLR parser is requested.
+b4_define_flag_if([has_translations])   # Whether some tokens are internationalized.
 b4_define_flag_if([nondeterministic])   # Whether conflicts should be handled.
 b4_define_flag_if([token_table])        # Whether yytoken_table is demanded.
 b4_define_flag_if([yacc])               # Whether POSIX Yacc is emulated.
@@ -384,37 +409,72 @@ m4_define([b4_glr_cc_if],
 ## Symbols.  ##
 ## --------- ##
 
-# For a description of the Symbol handling, see README.
+# For a description of the Symbol handling, see README.md.
 #
 # The following macros provide access to symbol related values.
 
 # __b4_symbol(NUM, FIELD)
 # -----------------------
-# Recover a FIELD about symbol #NUM.  Thanks to m4_indir, fails if
-# undefined.
+# Fetch FIELD of symbol #NUM.  Fail if undefined.
 m4_define([__b4_symbol],
 [m4_indir([b4_symbol($1, $2)])])
 
 
 # _b4_symbol(NUM, FIELD)
 # ----------------------
-# Recover a FIELD about symbol #NUM (or "orig NUM").  Fails if
-# undefined.
+# Fetch FIELD of symbol #NUM (or "orig NUM", see README.md).
+# Fail if undefined.
 m4_define([_b4_symbol],
 [m4_ifdef([b4_symbol($1, number)],
           [__b4_symbol(m4_indir([b4_symbol($1, number)]), $2)],
           [__b4_symbol([$1], [$2])])])
 
 
+# b4_symbol_token_kind(NUM)
+# -------------------------
+# The token kind of this symbol.
+m4_define([b4_symbol_token_kind],
+[b4_percent_define_get([api.token.prefix])dnl
+_b4_symbol([$1], [id])])
+
+
+# b4_symbol_kind_base(NUM)
+# ------------------------
+# Build the name of the kind of this symbol.  It must always exist,
+# otherwise some symbols might not be represented in the enum, which
+# might be compiled into too small a type to contain all the symbol
+# numbers.
+m4_define([b4_symbol_prefix], [b4_percent_define_get([api.symbol.prefix])])
+m4_define([b4_symbol_kind_base],
+[b4_percent_define_get([api.symbol.prefix])dnl
+m4_case([$1],
+  [-2],                             [[YYEMPTY]],
+  [0],                              [[YYEOF]],
+  [1],                              [[YYerror]],
+  [2],                              [[YYUNDEF]],
+  [m4_case(b4_symbol([$1], [tag]),
+      [$accept],                    [[YYACCEPT]],
+      [b4_symbol_if([$1], [has_id], _b4_symbol([$1], [id]),
+                                    [m4_bpatsubst([$1-][]_b4_symbol([$1], [tag]), [[^a-zA-Z_0-9]+], [_])])])])])
+
+
+# b4_symbol_kind(NUM)
+# -------------------
+# Same as b4_symbol_kind, but possibly with a prefix in some
+# languages.  E.g., EOF's kind_base and kind are YYSYMBOL_YYEOF in C,
+# but are S_YYEMPTY and symbol_kind::S_YYEMPTY in C++.
+m4_copy([b4_symbol_kind_base], [b4_symbol_kind])
 
 # b4_symbol(NUM, FIELD)
 # ---------------------
-# Recover a FIELD about symbol #NUM (or "orig NUM").  Fails if
-# undefined.  If FIELD = id, prepend the token prefix.
+# Fetch FIELD of symbol #NUM (or "orig NUM").  Fail if undefined.
+#
+# If FIELD = id, prepend the token prefix.
 m4_define([b4_symbol],
 [m4_case([$2],
-         [id],    [m4_do([b4_percent_define_get([api.token.prefix])],
-                         [_b4_symbol([$1], [id])])],
+         [id],        [b4_symbol_token_kind([$1])],
+         [kind_base], [b4_symbol_kind_base([$1])],
+         [kind],      [b4_symbol_kind([$1])],
          [_b4_symbol($@)])])
 
 
@@ -437,9 +497,9 @@ m4_define([b4_symbol_tag_comment],
 ])
 
 
-# b4_symbol_action(SYMBOL-NUM, KIND)
-# ----------------------------------
-# Run the action KIND (destructor or printer) for SYMBOL-NUM.
+# b4_symbol_action(SYMBOL-NUM, ACTION)
+# ------------------------------------
+# Run the action ACTION ("destructor" or "printer") for SYMBOL-NUM.
 m4_define([b4_symbol_action],
 [b4_symbol_if([$1], [has_$2],
 [b4_dollar_pushdef([(*yyvaluep)],
@@ -463,21 +523,21 @@ m4_define([b4_symbol_destructor], [b4_symbol_action([$1], [destructor])])
 m4_define([b4_symbol_printer],    [b4_symbol_action([$1], [printer])])
 
 
-# b4_symbol_actions(KIND, [TYPE = yytype])
-# ----------------------------------------
-# Emit the symbol actions for KIND ("printer" or "destructor").
-# Dispatch on TYPE.
+# b4_symbol_actions(ACTION, [KIND = yykind])
+# ------------------------------------------
+# Emit the symbol actions for ACTION ("destructor" or "printer").
+# Dispatch on KIND.
 m4_define([b4_symbol_actions],
 [m4_pushdef([b4_actions_], m4_expand([b4_symbol_foreach([b4_symbol_$1])]))dnl
 m4_ifval(m4_defn([b4_actions_]),
-[switch (m4_default([$2], [yytype]))
+[switch (m4_default([$2], [yykind]))
     {
 m4_defn([b4_actions_])[]dnl
       default:
         break;
     }dnl
 ],
-[YYUSE (m4_default([$2], [yytype]));])dnl
+[YYUSE (m4_default([$2], [yykind]));])dnl
 m4_popdef([b4_actions_])dnl
 ])
 
@@ -487,7 +547,7 @@ m4_popdef([b4_actions_])dnl
 # easier to use with m4_map, but then, use []dnl to suppress the last
 # one.
 m4_define([_b4_symbol_case],
-[case b4_symbol([$1], [number]): b4_symbol_tag_comment([$1])])
+[case b4_symbol([$1], [kind]): b4_symbol_tag_comment([$1])])
 ])
 
 
@@ -507,13 +567,14 @@ m4_define([b4_symbol_map],
 
 # b4_token_visible_if(NUM, IF-TRUE, IF-FALSE)
 # -------------------------------------------
-# Whether NUM denotes a token that has an exported definition (i.e.,
-# shows in enum yytokentype).
+# Whether NUM denotes a token kind that has an exported definition
+# (i.e., shows in enum yytokentype).
 m4_define([b4_token_visible_if],
 [b4_symbol_if([$1], [is_token],
               [b4_symbol_if([$1], [has_id], [$2], [$3])],
               [$3])])
 
+
 # b4_token_has_definition(NUM)
 # ----------------------------
 # 1 if NUM is visible, nothing otherwise.
@@ -530,13 +591,27 @@ m4_define([b4_any_token_visible_if],
 
 # b4_token_format(FORMAT, NUM)
 # ----------------------------
+# If token NUM has a visible ID, format FORMAT with ID, USER_NUMBER.
 m4_define([b4_token_format],
 [b4_token_visible_if([$2],
 [m4_format([[$1]],
-           m4_quote(b4_symbol([$2], [id])),
-           m4_quote(b4_symbol([$2], b4_api_token_raw_if([[number]], [[user_number]]))))])])
+           b4_symbol([$2], [id]),
+           b4_symbol([$2], b4_api_token_raw_if([[number]], [[code]])))])])
 
 
+# b4_last_enum_token
+# ------------------
+# The code of the last token visible token.
+m4_define([_b4_last_enum_token],
+[b4_token_visible_if([$1],
+   [m4_define([b4_last_enum_token], [$1])])])
+b4_symbol_foreach([_b4_last_enum_token])
+
+# b4_last_symbol
+# --------------
+# The code of the last symbol.
+m4_define([b4_last_symbol], m4_eval(b4_tokens_number + b4_nterms_number - 1))
+
 ## ------- ##
 ## Types.  ##
 ## ------- ##
@@ -603,7 +678,7 @@ m4_define([b4_sync_end],   [ b4_comment([$2:$1])]
 # This generates dependencies on the Bison skeletons hence lots of
 # useless 'git diff'.  This location is useless for the regular
 # user (who does not care about the skeletons) and is actually not
-# useful for Bison developpers too (I, Akim, never used this to locate
+# useful for Bison developers too (I, Akim, never used this to locate
 # the code in skeletons that generated output).  So disable it
 # completely.  If someone thinks this was actually useful, a %define
 # variable should be provided to control the level of verbosity of
@@ -683,7 +758,7 @@ m4_define([b4_check_user_names],
 [m4_pushdef([b4_occurrence], b4_occurrence)dnl
 m4_pushdef([b4_user_name], m4_car(b4_occurrence))dnl
 m4_pushdef([b4_start], m4_car(m4_shift(b4_occurrence)))dnl
-m4_pushdef([b4_end], m4_shift(m4_shift(b4_occurrence)))dnl
+m4_pushdef([b4_end], m4_shift2(b4_occurrence))dnl
 m4_ifndef($3[(]m4_quote(b4_user_name)[)],
           [b4_complain_at([b4_start], [b4_end],
                           [[%s '%s' is not used]],
@@ -1016,21 +1091,23 @@ m4_define([b4_bison_locations_if],
 [b4_locations_if([b4_percent_define_ifdef([[api.location.type]], [], [$1])])])
 
 
-# b4_error_verbose_if([IF-ERRORS-ARE-VERBOSE], [IF-NOT])
+
+# %define parse.error "(custom|detailed|simple|verbose)"
 # ------------------------------------------------------
-# Map %define parse.error "(simple|verbose)" to b4_error_verbose_if and
-# b4_error_verbose_flag.
 b4_percent_define_default([[parse.error]], [[simple]])
 b4_percent_define_check_values([[[[parse.error]],
-                                 [[simple]], [[verbose]]]])
-m4_define([b4_error_verbose_flag],
-          [m4_case(b4_percent_define_get([[parse.error]]),
-                   [simple],  [[0]],
-                   [verbose], [[1]])])
-b4_define_flag_if([error_verbose])
+                                 [[custom]], [[detailed]], [[simple]], [[verbose]]]])
+
+# b4_parse_error_case(CASE1, THEN1, CASE2, THEN2, ..., ELSE)
+# ----------------------------------------------------------
+m4_define([b4_parse_error_case],
+[m4_case(b4_percent_define_get([[parse.error]]), $@)])
+
+# b4_parse_error_bmatch(PATTERN1, THEN1, PATTERN2, THEN2, ..., ELSE)
+# ------------------------------------------------------------------
+m4_define([b4_parse_error_bmatch],
+[m4_bmatch(b4_percent_define_get([[parse.error]]), $@)])
 
-# yytoken_table is needed to support verbose errors.
-b4_error_verbose_if([m4_define([b4_token_table_flag], [1])])
 
 
 # b4_variant_if([IF-VARIANT-ARE-USED], [IF-NOT])
@@ -1116,3 +1193,11 @@ b4_percent_define_ifdef([api.value.type],
 
 # api.value.union.name.
 b4_percent_define_check_kind([api.value.union.name], [keyword])
+
+# parse.error (custom|detailed) >< token-table.
+b4_token_table_if(
+[b4_parse_error_bmatch([custom\|detailed],
+[b4_complain_at(b4_percent_define_get_loc([parse.error]),
+                [['%s' and '%s' cannot be used together]],
+                [%token-table],
+                [%define parse.error (custom|detailed)])])])

data/skeletons/c++.m4

@@ -22,6 +22,8 @@ b4_percent_define_ifdef([[api.value.union.name]],
   [b4_complain_at(b4_percent_define_get_loc([[api.value.union.name]]),
                   [named %union is invalid in C++])])
 
+b4_percent_define_default([[api.symbol.prefix]], [[S_]])
+
 m4_include(b4_skeletonsdir/[c.m4])
 
 b4_percent_define_check_kind([api.namespace], [code], [deprecated])
@@ -103,7 +105,7 @@ b4_percent_define_default([[api.parser.class]], [[parser]])
 #
 # b4_percent_define_default([[api.location.type]], [[location]])
 
-b4_percent_define_default([[filename_type]], [[std::string]])
+b4_percent_define_default([[api.filename.type]], [[const std::string]])
 # Make it a warning for those who used betas of Bison 3.0.
 b4_percent_define_default([[api.namespace]], m4_defn([b4_prefix]))
 
@@ -160,20 +162,42 @@ m4_bpatsubst(m4_dquote(m4_bpatsubst(m4_dquote(b4_namespace_ref[ ]),
              [::\([^][:]\|:[^:]\)*], [} ])[} // ]b4_namespace_ref])])
 
 
+## ------------- ##
+## Token kinds.  ##
+## ------------- ##
+
+
 # b4_token_enums
 # --------------
-# Output the definition of the tokens as enums.
+# Output the definition of the token kinds.
 m4_define([b4_token_enums],
-[[enum yytokentype
+[[enum token_kind_type
       {
-        ]m4_join([,
-        ],
-                 b4_symbol_map([b4_token_enum]))[
-      };]dnl
+        ]b4_symbol([-2], [id])[ = -2,
+]b4_symbol_foreach([b4_token_enum])dnl
+[      };]dnl
 ])
 
 
 
+## -------------- ##
+## Symbol kinds.  ##
+## -------------- ##
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+# Defining YYEMPTY here is important: it forces the compiler
+# to use a signed type, which matters for yytoken.
+m4_define([b4_declare_symbol_enum],
+[[enum symbol_kind_type
+      {
+        YYNTOKENS = ]b4_tokens_number[, ///< Number of tokens.
+        ]b4_symbol(-2, kind_base)[ = -2,
+]b4_symbol_foreach([      b4_symbol_enum])dnl
+[      };]])
+
+
 
 ## ----------------- ##
 ## Semantic Values.  ##
@@ -231,23 +255,31 @@ m4_define([b4_public_types_declare],
       location_type location;])[
     };
 
-    /// Tokens.
+    /// Token kinds.
     struct token
     {
       ]b4_token_enums[
+      /// Backward compatibility alias (Bison 3.6).
+      typedef token_kind_type yytokentype;
     };
 
-    /// (External) token type, as returned by yylex.
-    typedef token::yytokentype token_type;
+    /// Token kind, as returned by yylex.
+    typedef token::yytokentype token_kind_type;
 
-    /// Symbol type: an internal symbol number.
-    typedef int symbol_number_type;
+    /// Backward compatibility alias (Bison 3.6).
+    typedef token_kind_type token_type;
 
-    /// The symbol type number to denote an empty symbol.
-    enum { empty_symbol = -2 };
+    /// Symbol kinds.
+    struct symbol_kind
+    {
+      ]b4_declare_symbol_enum[
+    };
 
-    /// Internal symbol number for tokens (subsumed by symbol_number_type).
-    typedef ]b4_int_type_for([b4_translate])[ token_number_type;
+    /// (Internal) symbol kind.
+    typedef symbol_kind::symbol_kind_type symbol_kind_type;
+
+    /// The number of tokens.
+    static const symbol_kind_type YYNTOKENS = symbol_kind::YYNTOKENS;
 ]])
 
 
@@ -258,8 +290,8 @@ m4_define([b4_public_types_declare],
 m4_define([b4_symbol_type_define],
 [[    /// A complete symbol.
     ///
-    /// Expects its Base type to provide access to the symbol type
-    /// via type_get ().
+    /// Expects its Base type to provide access to the symbol kind
+    /// via kind ().
     ///
     /// Provide access to semantic value]b4_locations_if([ and location])[.
     template <typename Base>
@@ -276,7 +308,14 @@ m4_define([b4_symbol_type_define],
 
 #if 201103L <= YY_CPLUSPLUS
       /// Move constructor.
-      basic_symbol (basic_symbol&& that);
+      basic_symbol (basic_symbol&& that)
+        : Base (std::move (that))
+        , value (]b4_variant_if([], [std::move (that.value)]))b4_locations_if([
+        , location (std::move (that.location))])[
+      {]b4_variant_if([
+        b4_symbol_variant([this->kind ()], [value], [move],
+                          [std::move (that.value)])
+      ])[}
 #endif
 
       /// Copy constructor.
@@ -303,21 +342,47 @@ m4_define([b4_symbol_type_define],
       void clear ()
       {]b4_variant_if([[
         // User destructor.
-        symbol_number_type yytype = this->type_get ();
+        symbol_kind_type yykind = this->kind ();
         basic_symbol<Base>& yysym = *this;
         (void) yysym;
-        switch (yytype)
+        switch (yykind)
         {
 ]b4_symbol_foreach([b4_symbol_destructor])dnl
 [       default:
           break;
         }
 
-        // Type destructor.
-]b4_symbol_variant([[yytype]], [[value]], [[template destroy]])])[
+        // Value type destructor.
+]b4_symbol_variant([[yykind]], [[value]], [[template destroy]])])[
         Base::clear ();
       }
 
+]b4_parse_error_bmatch(
+[custom\|detailed],
+[[      /// The user-facing name of this symbol.
+      const char *name () const YY_NOEXCEPT
+      {
+        return ]b4_parser_class[::symbol_name (this->kind ());
+      }]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+      /// The user-facing name of this symbol.
+      const char *name () const YY_NOEXCEPT
+      {
+        return ]b4_parser_class[::symbol_name (this->kind ());
+      }
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[      /// The user-facing name of this symbol.
+      std::string name () const YY_NOEXCEPT
+      {
+        return ]b4_parser_class[::symbol_name (this->kind ());
+      }]])[
+
+      /// Backward compatibility (Bison 3.6).
+      symbol_kind_type type_get () const YY_NOEXCEPT;
+
       /// Whether empty.
       bool empty () const YY_NOEXCEPT;
 
@@ -338,46 +403,51 @@ m4_define([b4_symbol_type_define],
     };
 
     /// Type access provider for token (enum) based symbols.
-    struct by_type
+    struct by_kind
     {
       /// Default constructor.
-      by_type ();
+      by_kind ();
 
 #if 201103L <= YY_CPLUSPLUS
       /// Move constructor.
-      by_type (by_type&& that);
+      by_kind (by_kind&& that);
 #endif
 
       /// Copy constructor.
-      by_type (const by_type& that);
+      by_kind (const by_kind& that);
 
-      /// The symbol type as needed by the constructor.
-      typedef token_type kind_type;
+      /// The symbol kind as needed by the constructor.
+      typedef token_kind_type kind_type;
 
       /// Constructor from (external) token numbers.
-      by_type (kind_type t);
+      by_kind (kind_type t);
 
       /// Record that this symbol is empty.
       void clear ();
 
-      /// Steal the symbol type from \a that.
-      void move (by_type& that);
+      /// Steal the symbol kind from \a that.
+      void move (by_kind& that);
 
       /// The (internal) type number (corresponding to \a type).
       /// \a empty when empty.
-      symbol_number_type type_get () const YY_NOEXCEPT;
+      symbol_kind_type kind () const YY_NOEXCEPT;
 
-      /// The symbol type.
-      /// \a empty_symbol when empty.
-      /// An int, not token_number_type, to be able to store empty_symbol.
-      int type;
+      /// Backward compatibility (Bison 3.6).
+      symbol_kind_type type_get () const YY_NOEXCEPT;
+
+      /// The symbol kind.
+      /// \a ]b4_symbol_prefix[YYEMPTY when empty.
+      symbol_kind_type kind_;
     };
 
+    /// Backward compatibility for a private implementation detail (Bison 3.6).
+    typedef by_kind by_type;
+
     /// "External" symbols: returned by the scanner.
-    struct symbol_type : basic_symbol<by_type>
+    struct symbol_type : basic_symbol<by_kind>
     {]b4_variant_if([[
       /// Superclass.
-      typedef basic_symbol<by_type> super_type;
+      typedef basic_symbol<by_kind> super_type;
 
       /// Empty symbol.
       symbol_type () {}
@@ -393,25 +463,13 @@ m4_define([b4_symbol_type_define],
 # Provide the implementation needed by the public types.
 m4_define([b4_public_types_define],
 [[  // basic_symbol.
-#if 201103L <= YY_CPLUSPLUS
-  template <typename Base>
-  ]b4_parser_class[::basic_symbol<Base>::basic_symbol (basic_symbol&& that)
-    : Base (std::move (that))
-    , value (]b4_variant_if([], [std::move (that.value)]))b4_locations_if([
-    , location (std::move (that.location))])[
-  {]b4_variant_if([
-    b4_symbol_variant([this->type_get ()], [value], [move],
-                      [std::move (that.value)])
-  ])[}
-#endif
-
   template <typename Base>
   ]b4_parser_class[::basic_symbol<Base>::basic_symbol (const basic_symbol& that)
     : Base (that)
     , value (]b4_variant_if([], [that.value]))b4_locations_if([
     , location (that.location)])[
   {]b4_variant_if([
-    b4_symbol_variant([this->type_get ()], [value], [copy],
+    b4_symbol_variant([this->kind ()], [value], [copy],
                       [YY_MOVE (that.value)])
   ])[}
 
@@ -436,13 +494,20 @@ m4_define([b4_public_types_define],
     , location (YY_MOVE (l))])[
   {]b4_variant_if([[
     (void) v;
-    ]b4_symbol_variant([this->type_get ()], [value], [YY_MOVE_OR_COPY], [YY_MOVE (v)])])[}]])[
+    ]b4_symbol_variant([this->kind ()], [value], [YY_MOVE_OR_COPY], [YY_MOVE (v)])])[}]])[
+
+  template <typename Base>
+  ]b4_parser_class[::symbol_kind_type
+  ]b4_parser_class[::basic_symbol<Base>::type_get () const YY_NOEXCEPT
+  {
+    return this->kind ();
+  }
 
   template <typename Base>
   bool
   ]b4_parser_class[::basic_symbol<Base>::empty () const YY_NOEXCEPT
   {
-    return Base::type_get () == empty_symbol;
+    return this->kind () == ]b4_symbol(-2, kind)[;
   }
 
   template <typename Base>
@@ -450,50 +515,56 @@ m4_define([b4_public_types_define],
   ]b4_parser_class[::basic_symbol<Base>::move (basic_symbol& s)
   {
     super_type::move (s);
-    ]b4_variant_if([b4_symbol_variant([this->type_get ()], [value], [move],
+    ]b4_variant_if([b4_symbol_variant([this->kind ()], [value], [move],
                                       [YY_MOVE (s.value)])],
                    [value = YY_MOVE (s.value);])[]b4_locations_if([
     location = YY_MOVE (s.location);])[
   }
 
-  // by_type.
-  ]b4_inline([$1])b4_parser_class[::by_type::by_type ()
-    : type (empty_symbol)
+  // by_kind.
+  ]b4_inline([$1])b4_parser_class[::by_kind::by_kind ()
+    : kind_ (]b4_symbol(-2, kind)[)
   {}
 
 #if 201103L <= YY_CPLUSPLUS
-  ]b4_inline([$1])b4_parser_class[::by_type::by_type (by_type&& that)
-    : type (that.type)
+  ]b4_inline([$1])b4_parser_class[::by_kind::by_kind (by_kind&& that)
+    : kind_ (that.kind_)
   {
     that.clear ();
   }
 #endif
 
-  ]b4_inline([$1])b4_parser_class[::by_type::by_type (const by_type& that)
-    : type (that.type)
+  ]b4_inline([$1])b4_parser_class[::by_kind::by_kind (const by_kind& that)
+    : kind_ (that.kind_)
   {}
 
-  ]b4_inline([$1])b4_parser_class[::by_type::by_type (token_type t)
-    : type (yytranslate_ (t))
+  ]b4_inline([$1])b4_parser_class[::by_kind::by_kind (token_kind_type t)
+    : kind_ (yytranslate_ (t))
   {}
 
   ]b4_inline([$1])[void
-  ]b4_parser_class[::by_type::clear ()
+  ]b4_parser_class[::by_kind::clear ()
   {
-    type = empty_symbol;
+    kind_ = ]b4_symbol(-2, kind)[;
   }
 
   ]b4_inline([$1])[void
-  ]b4_parser_class[::by_type::move (by_type& that)
+  ]b4_parser_class[::by_kind::move (by_kind& that)
   {
-    type = that.type;
+    kind_ = that.kind_;
     that.clear ();
   }
 
-  ]b4_inline([$1])[int
-  ]b4_parser_class[::by_type::type_get () const YY_NOEXCEPT
+  ]b4_inline([$1])[]b4_parser_class[::symbol_kind_type
+  ]b4_parser_class[::by_kind::kind () const YY_NOEXCEPT
   {
-    return type;
+    return kind_;
+  }
+
+  ]b4_inline([$1])[]b4_parser_class[::symbol_kind_type
+  ]b4_parser_class[::by_kind::type_get () const YY_NOEXCEPT
+  {
+    return this->kind ();
   }
 ]])
 
@@ -510,27 +581,28 @@ m4_define([b4_token_constructor_define], [])
 # Define yytranslate_.  Sometimes used in the header file ($1=hh),
 # sometimes in the cc file.
 m4_define([b4_yytranslate_define],
-[  b4_inline([$1])b4_parser_class[::token_number_type
+[  b4_inline([$1])b4_parser_class[::symbol_kind_type
   ]b4_parser_class[::yytranslate_ (int t)
   {
 ]b4_api_token_raw_if(
-[[    return static_cast<token_number_type> (t);]],
+[[    return static_cast<symbol_kind_type> (t);]],
 [[    // YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to
     // TOKEN-NUM as returned by yylex.
     static
-    const token_number_type
+    const ]b4_int_type_for([b4_translate])[
     translate_table[] =
     {
   ]b4_translate[
     };
-    const int user_token_number_max_ = ]b4_user_token_number_max[;
+    // Last valid token kind.
+    const int code_max = ]b4_code_max[;
 
     if (t <= 0)
-      return yyeof_;
-    else if (t <= user_token_number_max_)
-      return translate_table[t];
+      return symbol_kind::]b4_symbol_prefix[YYEOF;
+    else if (t <= code_max)
+      return YY_CAST (symbol_kind_type, translate_table[t]);
     else
-      return yy_undef_token_;]])[
+      return symbol_kind::]b4_symbol_prefix[YYUNDEF;]])[
   }
 ]])
 

data/skeletons/c-like.m4

@@ -17,15 +17,21 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+
 # _b4_comment(TEXT, OPEN, CONTINUE, END)
 # --------------------------------------
 # Put TEXT in comment.  Avoid trailing spaces: don't indent empty lines.
 # Avoid adding indentation to the first line, as the indentation comes
 # from OPEN.  That's why we don't patsubst([$1], [^\(.\)], [   \1]).
+# Turn "*/" in TEXT into "* /" so that we don't unexpectedly close
+# the comments before its end.
 #
 # Prefix all the output lines with PREFIX.
 m4_define([_b4_comment],
-[$2[]m4_bpatsubst(m4_expand([[$1]]), [
+[$2[]b4_gsub(m4_expand([$1]),
+            [[*]/], [*\\/],
+            [/[*]], [/\\*],
+            [
 \(.\)], [
 $3\1])$4])
 

data/skeletons/c.m4

@@ -101,6 +101,7 @@ m4_define_default([b4_prefix], [b4_api_prefix])
 b4_percent_define_default([[api.value.union.name]],
                           [b4_api_PREFIX[][STYPE]])
 
+b4_percent_define_default([[api.symbol.prefix]], [[YYSYMBOL_]])
 
 ## ------------------------ ##
 ## Pure/impure interfaces.  ##
@@ -129,6 +130,13 @@ m4_define([b4_user_args],
 [m4_ifset([b4_parse_param], [, b4_args(b4_parse_param)])])
 
 
+# b4_user_formals
+# ---------------
+# The possible parse-params formal arguments preceded by a comma.
+m4_define([b4_user_formals],
+[m4_ifset([b4_parse_param], [, b4_formals(b4_parse_param)])])
+
+
 # b4_parse_param
 # --------------
 # If defined, b4_parse_param arrives double quoted, but below we prefer
@@ -197,7 +205,11 @@ m4_define([b4_c99_int_type],
 # Define private types suitable for holding small integers in C99 or later.
 m4_define([b4_c99_int_type_define],
 [m4_copy_force([b4_c99_int_type], [b4_int_type])dnl
-[/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure
+[#ifdef short
+# undef short
+#endif
+
+/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure
    <limits.h> and (if available) <stdint.h> are included
    so that the code can choose integer types of a good width.  */
 
@@ -253,6 +265,50 @@ typedef int yytype_uint16;
 #endif]])
 
 
+# b4_sizes_types_define
+# ---------------------
+# Define YYPTRDIFF_T/YYPTRDIFF_MAXIMUM, YYSIZE_T/YYSIZE_MAXIMUM,
+# and YYSIZEOF.
+m4_define([b4_sizes_types_define],
+[[#ifndef YYPTRDIFF_T
+# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__
+#  define YYPTRDIFF_T __PTRDIFF_TYPE__
+#  define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__
+# elif defined PTRDIFF_MAX
+#  ifndef ptrdiff_t
+#   include <stddef.h> /* INFRINGES ON USER NAME SPACE */
+#  endif
+#  define YYPTRDIFF_T ptrdiff_t
+#  define YYPTRDIFF_MAXIMUM PTRDIFF_MAX
+# else
+#  define YYPTRDIFF_T long
+#  define YYPTRDIFF_MAXIMUM LONG_MAX
+# endif
+#endif
+
+#ifndef YYSIZE_T
+# ifdef __SIZE_TYPE__
+#  define YYSIZE_T __SIZE_TYPE__
+# elif defined size_t
+#  define YYSIZE_T size_t
+# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__
+#  include <stddef.h> /* INFRINGES ON USER NAME SPACE */
+#  define YYSIZE_T size_t
+# else
+#  define YYSIZE_T unsigned
+# endif
+#endif
+
+#define YYSIZE_MAXIMUM                                  \
+  YY_CAST (YYPTRDIFF_T,                                 \
+           (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1)  \
+            ? YYPTRDIFF_MAXIMUM                         \
+            : YY_CAST (YYSIZE_T, -1)))
+
+#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X))
+]])
+
+
 # b4_int_type_for(NAME)
 # ---------------------
 # Return a narrow int type able to handle numbers ranging from
@@ -299,15 +355,23 @@ m4_define([b4_attribute_define],
 #endif
 
 ]m4_bmatch([$1], [\bnoreturn\b], [[/* The _Noreturn keyword of C11.  */
-]dnl This is an exact copy of lib/_Noreturn.h.
+]dnl This is close to lib/_Noreturn.h, except that we do enable
+dnl the use of [[noreturn]], because _Noreturn is used in places
+dnl where [[noreturn]] works in C++.  We need this in particular
+dnl because of glr.cc which compiles code from glr.c in C++.
+dnl And the C++ compiler chokes on _Noreturn.  Also, we do not
+dnl use C' _Noreturn in C++, to avoid -Wc11-extensions warnings.
 [#ifndef _Noreturn
 # if (defined __cplusplus \
       && ((201103 <= __cplusplus && !(__GNUC__ == 4 && __GNUC_MINOR__ == 7)) \
           || (defined _MSC_VER && 1900 <= _MSC_VER)))
 #  define _Noreturn [[noreturn]]
-# elif ((!defined __cplusplus || defined __clang__) \
+# elif (!defined __cplusplus                     \
         && (201112 <= (defined __STDC_VERSION__ ? __STDC_VERSION__ : 0)  \
-            || 4 < __GNUC__ + (7 <= __GNUC_MINOR__)))
+            || 4 < __GNUC__ + (7 <= __GNUC_MINOR__) \
+            || (defined __apple_build_version__ \
+                ? 6000000 <= __apple_build_version__ \
+                : 3 < __clang_major__ + (5 <= __clang_minor__))))
    /* _Noreturn works as-is.  */
 # elif 2 < __GNUC__ + (8 <= __GNUC_MINOR__) || 0x5110 <= __SUNPRO_C
 #  define _Noreturn __attribute__ ((__noreturn__))
@@ -418,9 +482,22 @@ static const b4_int_type_for([$2]) yy$1[[]] =
 ])
 
 
-## ------------------------- ##
-## Assigning token numbers.  ##
-## ------------------------- ##
+## ------------- ##
+## Token kinds.  ##
+## ------------- ##
+
+# Because C enums are not scoped, because tokens are exposed in the
+# header, and because these tokens are common to all the parsers, we
+# need to make sure their names don't collide: use the api.prefix.
+# YYEOF is special, since the user may give it a different name.
+m4_define([b4_symbol(-2, id)],  [b4_api_PREFIX[][EMPTY]])
+m4_define([b4_symbol(-2, tag)], [[No symbol.]])
+
+m4_if(b4_symbol(0, id), [YYEOF],
+     [m4_define([b4_symbol(0, id)],  [b4_api_PREFIX[][EOF]])])
+m4_define([b4_symbol(1, id)],  [b4_api_PREFIX[][error]])
+m4_define([b4_symbol(2, id)],  [b4_api_PREFIX[][UNDEF]])
+
 
 # b4_token_define(TOKEN-NUM)
 # --------------------------
@@ -432,7 +509,7 @@ m4_define([b4_token_define],
 # ----------------
 # Output the definition of the tokens.
 m4_define([b4_token_defines],
-[b4_any_token_visible_if([/* Tokens.  */
+[b4_any_token_visible_if([/* Token kinds.  */
 m4_join([
 ], b4_symbol_map([b4_token_define]))
 ])])
@@ -442,34 +519,81 @@ m4_join([
 # ------------------------
 # Output the definition of this token as an enum.
 m4_define([b4_token_enum],
-[b4_token_format([%s = %s], [$1])])
+[b4_token_visible_if([$1],
+    [m4_format([    %-30s %s],
+               m4_format([[%s = %s%s%s]],
+                         b4_symbol([$1], [id]),
+                         b4_symbol([$1], b4_api_token_raw_if([[number]], [[code]])),
+                         m4_if([$1], b4_last_enum_token, [], [[,]])),
+               [b4_symbol_tag_comment([$1])])])])
 
 
 # b4_token_enums
 # --------------
-# Output the definition of the tokens (if there are) as enums.
+# The definition of the token kinds.
 m4_define([b4_token_enums],
-[b4_any_token_visible_if([[/* Token type.  */
+[b4_any_token_visible_if([[/* Token kinds.  */
 #ifndef ]b4_api_PREFIX[TOKENTYPE
 # define ]b4_api_PREFIX[TOKENTYPE
   enum ]b4_api_prefix[tokentype
   {
-    ]m4_join([,
-    ],
-             b4_symbol_map([b4_token_enum]))[
-  };
+    ]b4_symbol([-2], [id])[ = -2,
+]b4_symbol_foreach([b4_token_enum])dnl
+[  };
+  typedef enum ]b4_api_prefix[tokentype ]b4_api_prefix[token_kind_t;
 #endif
 ]])])
 
 
 # b4_token_enums_defines
 # ----------------------
-# Output the definition of the tokens (if there are any) as enums and,
+# The definition of the tokens (if there are any) as enums and,
 # if POSIX Yacc is enabled, as #defines.
 m4_define([b4_token_enums_defines],
 [b4_token_enums[]b4_yacc_if([b4_token_defines])])
 
 
+# b4_symbol_translate(STRING)
+# ---------------------------
+# Used by "bison" in the array of symbol names to mark those that
+# require translation.
+m4_define([b4_symbol_translate],
+[[N_($1)]])
+
+
+
+## -------------- ##
+## Symbol kinds.  ##
+## -------------- ##
+
+# b4_symbol_enum(SYMBOL-NUM)
+# --------------------------
+# Output the definition of this symbol as an enum.
+m4_define([b4_symbol_enum],
+[m4_format([  %-40s %s],
+           m4_format([[%s = %s%s%s]],
+                     b4_symbol([$1], [kind_base]),
+                     [$1],
+                     m4_if([$1], b4_last_symbol, [], [[,]])),
+           [b4_symbol_tag_comment([$1])])])
+
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+# Defining YYEMPTY here is important: it forces the compiler
+# to use a signed type, which matters for yytoken.
+m4_define([b4_declare_symbol_enum],
+[[/* Symbol kind.  */
+enum yysymbol_kind_t
+{
+  ]b4_symbol([-2], kind_base)[ = -2,
+]b4_symbol_foreach([b4_symbol_enum])dnl
+[};
+typedef enum yysymbol_kind_t yysymbol_kind_t;
+]])])
+
+
 ## ----------------- ##
 ## Semantic Values.  ##
 ## ----------------- ##
@@ -493,15 +617,6 @@ m4_define([b4_symbol_value],
 ## ---------------------- ##
 
 
-# b4_function_define(NAME, RETURN-VALUE, [DECL1, NAME1], ...)
-# -----------------------------------------------------------
-# Declare the function NAME in C.
-m4_define([b4_function_define],
-[$2
-$1 (b4_formals(m4_shift2($@)))[]dnl
-])
-
-
 # b4_formals([DECL1, NAME1], ...)
 # -------------------------------
 # The formal arguments of a C function definition.
@@ -515,21 +630,6 @@ m4_define([b4_formal],
 
 
 
-## ----------------------- ##
-## Declaring C functions.  ##
-## ----------------------- ##
-
-
-# b4_function_declare(NAME, RETURN-VALUE, [DECL1, NAME1], ...)
-# ------------------------------------------------------------
-# Declare the function NAME.
-m4_define([b4_function_declare],
-[$2 $1 (b4_formals(m4_shift2($@)));[]dnl
-])
-
-
-
-
 ## --------------------- ##
 ## Calling C functions.  ##
 ## --------------------- ##
@@ -566,10 +666,10 @@ m4_define([b4_sync_start], [[#]line $1 $2])
 ## User actions.  ##
 ## -------------- ##
 
-# b4_case(LABEL, STATEMENTS)
-# --------------------------
+# b4_case(LABEL, STATEMENTS, [COMMENTS])
+# --------------------------------------
 m4_define([b4_case],
-[  case $1:
+[  case $1:m4_ifval([$3], [ b4_comment([$3])])
 $2
 b4_syncline([@oline@], [@ofile@])dnl
     break;])
@@ -593,18 +693,15 @@ m4_define_default([b4_yydestruct_define],
 | Release the memory associated to this symbol.  |
 `-----------------------------------------------*/
 
-]b4_function_define([yydestruct],
-    [static void],
-    [[const char *yymsg],    [yymsg]],
-    [[int yytype],           [yytype]],
-    [[YYSTYPE *yyvaluep],    [yyvaluep]][]dnl
-b4_locations_if(            [, [[YYLTYPE *yylocationp], [yylocationp]]])[]dnl
-m4_ifset([b4_parse_param], [, b4_parse_param]))[
+static void
+yydestruct (const char *yymsg,
+            yysymbol_kind_t yykind, YYSTYPE *yyvaluep]b4_locations_if(dnl
+[[, YYLTYPE *yylocationp]])[]b4_user_formals[)
 {
 ]b4_parse_param_use([yyvaluep], [yylocationp])dnl
 [  if (!yymsg)
     yymsg = "Deleting";
-  YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp);
+  YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp);
 
   YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
   ]b4_symbol_actions([destructor])[
@@ -622,13 +719,10 @@ m4_define_default([b4_yy_symbol_print_define],
 | Print this symbol's value on YYO.  |
 `-----------------------------------*/
 
-]b4_function_define([yy_symbol_value_print],
-    [static void],
-               [[FILE *yyo],                            [yyo]],
-               [[int yytype],                           [yytype]],
-               [[YYSTYPE const * const yyvaluep],       [yyvaluep]][]dnl
-b4_locations_if([, [[YYLTYPE const * const yylocationp], [yylocationp]]])[]dnl
-m4_ifset([b4_parse_param], [, b4_parse_param]))[
+static void
+yy_symbol_value_print (FILE *yyo,
+                       yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep]b4_locations_if(dnl
+[[, YYLTYPE const * const yylocationp]])[]b4_user_formals[)
 {
   FILE *yyoutput = yyo;
 ]b4_parse_param_use([yyoutput], [yylocationp])dnl
@@ -637,8 +731,8 @@ m4_ifset([b4_parse_param], [, b4_parse_param]))[
 dnl glr.c does not feature yytoknum.
 m4_if(b4_skeleton, ["yacc.c"],
 [[# ifdef YYPRINT
-  if (yytype < YYNTOKENS)
-    YYPRINT (yyo, yytoknum[yytype], *yyvaluep);
+  if (yykind < YYNTOKENS)
+    YYPRINT (yyo, yytoknum[yykind], *yyvaluep);
 # endif
 ]])dnl
 b4_percent_code_get([[pre-printer]])dnl
@@ -653,21 +747,18 @@ b4_percent_code_get([[post-printer]])dnl
 | Print this symbol on YYO.  |
 `---------------------------*/
 
-]b4_function_define([yy_symbol_print],
-    [static void],
-               [[FILE *yyo],                            [yyo]],
-               [[int yytype],                           [yytype]],
-               [[YYSTYPE const * const yyvaluep],       [yyvaluep]][]dnl
-b4_locations_if([, [[YYLTYPE const * const yylocationp], [yylocationp]]])[]dnl
-m4_ifset([b4_parse_param], [, b4_parse_param]))[
+static void
+yy_symbol_print (FILE *yyo,
+                 yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep]b4_locations_if(dnl
+[[, YYLTYPE const * const yylocationp]])[]b4_user_formals[)
 {
   YYFPRINTF (yyo, "%s %s (",
-             yytype < YYNTOKENS ? "token" : "nterm", yytname[yytype]);
+             yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind));
 
 ]b4_locations_if([  YY_LOCATION_PRINT (yyo, *yylocationp);
   YYFPRINTF (yyo, ": ");
 ])dnl
-[  yy_symbol_value_print (yyo, yytype, yyvaluep]dnl
+[  yy_symbol_value_print (yyo, yykind, yyvaluep]dnl
 b4_locations_if([, yylocationp])[]b4_user_args[);
   YYFPRINTF (yyo, ")");
 }]dnl
@@ -693,11 +784,11 @@ m4_define([b4_symbol_type_register],
 [m4_define([b4_symbol($1, type_tag)],
            [b4_symbol_if([$1], [has_id],
                          [b4_symbol([$1], [id])],
-                         [yytype_[]b4_symbol([$1], [number])])])dnl
+                         [yykind_[]b4_symbol([$1], [number])])])dnl
 m4_append([b4_union_members],
-m4_expand([
-  b4_symbol_tag_comment([$1])dnl
-  b4_symbol([$1], [type]) b4_symbol([$1], [type_tag]);]))
+m4_expand([m4_format([  %-40s %s],
+                     m4_expand([b4_symbol([$1], [type]) b4_symbol([$1], [type_tag]);]),
+                     [b4_symbol_tag_comment([$1])])]))
 ])
 
 
@@ -930,16 +1021,14 @@ m4_define([b4_yy_location_print_define],
    This macro was not mandated originally: define only if we know
    we won't break user code: when these are the locations we know.  */
 
-#ifndef YY_LOCATION_PRINT
-# if defined ]b4_api_PREFIX[LTYPE_IS_TRIVIAL && ]b4_api_PREFIX[LTYPE_IS_TRIVIAL
+# ifndef YY_LOCATION_PRINT
+#  if defined ]b4_api_PREFIX[LTYPE_IS_TRIVIAL && ]b4_api_PREFIX[LTYPE_IS_TRIVIAL
 
 /* Print *YYLOCP on YYO.  Private, do not rely on its existence. */
 
 YY_ATTRIBUTE_UNUSED
-]b4_function_define([yy_location_print_],
-    [static int],
-               [[FILE *yyo],                    [yyo]],
-               [[YYLTYPE const * const yylocp], [yylocp]])[
+static int
+yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp)
 {
   int res = 0;
   int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0;
@@ -963,17 +1052,17 @@ YY_ATTRIBUTE_UNUSED
   return res;
  }
 
-#  define YY_LOCATION_PRINT(File, Loc)          \
+#   define YY_LOCATION_PRINT(File, Loc)          \
   yy_location_print_ (File, &(Loc))
 
-# else
-#  define YY_LOCATION_PRINT(File, Loc) ((void) 0)
-# endif
-#endif]],
+#  else
+#   define YY_LOCATION_PRINT(File, Loc) ((void) 0)
+#  endif
+# endif /* !defined YY_LOCATION_PRINT */]],
 [[/* This macro is provided for backward compatibility. */
-#ifndef YY_LOCATION_PRINT
-# define YY_LOCATION_PRINT(File, Loc) ((void) 0)
-#endif]])
+# ifndef YY_LOCATION_PRINT
+#  define YY_LOCATION_PRINT(File, Loc) ((void) 0)
+# endif]])
 ])
 
 # b4_yyloc_default

data/skeletons/d.m4

@@ -148,9 +148,11 @@ private static immutable b4_int_type_for([$2])[[]] yy$1_ =
 ])
 
 
-## ------------------------- ##
-## Assigning token numbers.  ##
-## ------------------------- ##
+## ------------- ##
+## Token kinds.  ##
+## ------------- ##
+
+m4_define([b4_symbol(-2, id)],  [[YYEMPTY]])
 
 # b4_token_enum(TOKEN-NAME, TOKEN-NUMBER)
 # ---------------------------------------
@@ -159,22 +161,61 @@ m4_define([b4_token_enum],
 [b4_token_format([  %s = %s,
 ], [$1])])
 
-# b4_token_enums(LIST-OF-PAIRS-TOKEN-NAME-TOKEN-NUMBER)
-# -----------------------------------------------------
+# b4_token_enums
+# --------------
 # Output the definition of the tokens as enums.
 m4_define([b4_token_enums],
-[/* Tokens.  */
-public enum YYTokenType {
-
-  /** Token returned by the scanner to signal the end of its input.  */
-  EOF = 0,
-b4_symbol_foreach([b4_token_enum])
+[/* Token kinds.  */
+public enum TokenKind {
+  ]b4_symbol_kind([-2])[ = -2,
+b4_symbol_foreach([b4_token_enum])dnl
 }
 ])
 
-# b4-case(ID, CODE)
-# -----------------
-m4_define([b4_case], [    case $1:
+
+
+## -------------- ##
+## Symbol kinds.  ##
+## -------------- ##
+
+b4_percent_define_default([[api.symbol.prefix]], [[S_]])
+
+# b4_symbol_kind(NUM)
+# -------------------
+m4_define([b4_symbol_kind],
+[SymbolKind.b4_symbol_kind_base($@)])
+
+
+# b4_symbol_enum(SYMBOL-NUM)
+# --------------------------
+# Output the definition of this symbol as an enum.
+m4_define([b4_symbol_enum],
+[m4_format([    %-30s %s],
+           m4_format([[%s = %s,]],
+                     b4_symbol([$1], [kind_base]),
+                     [$1]),
+           [b4_symbol_tag_comment([$1])])])
+
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+# Defining YYEMPTY here is important: it forces the compiler
+# to use a signed type, which matters for yytoken.
+m4_define([b4_declare_symbol_enum],
+[[  /* Symbol kinds.  */
+  public enum SymbolKind
+  {
+    ]b4_symbol(-2, kind_base)[ = -2,  /* No symbol.  */
+]b4_symbol_foreach([b4_symbol_enum])dnl
+[  };
+]])])
+
+
+
+# b4-case(ID, CODE, [COMMENTS])
+# -----------------------------
+m4_define([b4_case], [    case $1:m4_ifval([$3], [ b4_comment([$3])])
 $2
       break;])
 

data/skeletons/glr.cc

@@ -72,24 +72,17 @@ m4_defn([b4_parse_param]))],
 # Bypass the default implementation to generate the "yy_symbol_print"
 # and "yy_symbol_value_print" functions.
 m4_define([b4_yy_symbol_print_define],
-[[
-/*--------------------.
+[[/*--------------------.
 | Print this symbol.  |
 `--------------------*/
 
-]b4_function_define([yy_symbol_print],
-    [static void],
-    [[FILE *],      []],
-    [[int yytype],  [yytype]],
-    [[const ]b4_namespace_ref::b4_parser_class[::semantic_type *yyvaluep],
-                    [yyvaluep]][]dnl
-b4_locations_if([,
-    [[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp],
-                    [yylocationp]]]),
-    b4_parse_param)[
+static void
+yy_symbol_print (FILE *, ]b4_namespace_ref::b4_parser_class[::symbol_kind_type yytoken,
+                 const ]b4_namespace_ref::b4_parser_class[::semantic_type *yyvaluep]b4_locations_if([[,
+                 const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp]])[]b4_user_formals[)
 {
 ]b4_parse_param_use[]dnl
-[  yyparser.yy_symbol_print_ (yytype, yyvaluep]b4_locations_if([, yylocationp])[);
+[  yyparser.yy_symbol_print_ (yytoken, yyvaluep]b4_locations_if([, yylocationp])[);
 }
 ]])[
 
@@ -101,45 +94,24 @@ m4_defn([b4_initial_action])]))])[
 # Hijack the post prologue to declare yyerror.
 ]m4_append([b4_post_prologue],
 [b4_syncline([@oline@], [@ofile@])dnl
-b4_function_declare([yyerror],
-    [static void],b4_locations_if([
-    [[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp],
-                        [yylocationp]],])
-    b4_parse_param,
-    [[const char* msg], [msg]])])[
+[static void
+yyerror (]b4_locations_if([[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp,
+         ]])[]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param),
+         ])[const char* msg);]])[
 
-
-#undef yynerrs
-#undef yychar
-#undef yylval]b4_locations_if([
-#undef yylloc])
-
-m4_if(b4_prefix, [yy], [],
-[[/* Substitute the variable and function names.  */
-#define yyparse ]b4_prefix[parse
-#define yylex   ]b4_prefix[lex
-#define yyerror ]b4_prefix[error
-#define yydebug ]b4_prefix[debug]]b4_pure_if([], [[
-#define yylval  ]b4_prefix[lval
-#define yychar  ]b4_prefix[char
-#define yynerrs ]b4_prefix[nerrs]b4_locations_if([[
-#define yylloc  ]b4_prefix[lloc]])]))
-
-# Hijack the epilogue to define implementations (yyerror, parser member
+# Inserted before the epilogue to define implementations (yyerror, parser member
 # functions etc.).
-m4_append([b4_epilogue],
+]m4_define([b4_glr_cc_pre_epilogue],
 [b4_syncline([@oline@], [@ofile@])dnl
 [
 /*------------------.
 | Report an error.  |
 `------------------*/
 
-]b4_function_define([yyerror],
-    [static void],b4_locations_if([
-    [[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp],
-                        [yylocationp]],])
-    b4_parse_param,
-    [[const char* msg], [msg]])[
+static void
+yyerror (]b4_locations_if([[const ]b4_namespace_ref::b4_parser_class[::location_type *yylocationp,
+         ]])[]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param),
+         ])[const char* msg)
 {
 ]b4_parse_param_use[]dnl
 [  yyparser.error (]b4_locations_if([[*yylocationp, ]])[msg);
@@ -181,9 +153,9 @@ m4_pushdef([b4_parse_param], m4_defn([b4_parse_param_orig]))dnl
   `--------------------*/
 
   void
-  ]b4_parser_class[::yy_symbol_value_print_ (int yytype,
+  ]b4_parser_class[::yy_symbol_value_print_ (symbol_kind_type yykind,
                            const semantic_type* yyvaluep]b4_locations_if([[,
-                           const location_type* yylocationp]])[)
+                           const location_type* yylocationp]])[) const
   {]b4_locations_if([[
     YYUSE (yylocationp);]])[
     YYUSE (yyvaluep);
@@ -195,14 +167,14 @@ m4_pushdef([b4_parse_param], m4_defn([b4_parse_param_orig]))dnl
 
 
   void
-  ]b4_parser_class[::yy_symbol_print_ (int yytype,
+  ]b4_parser_class[::yy_symbol_print_ (symbol_kind_type yykind,
                            const semantic_type* yyvaluep]b4_locations_if([[,
-                           const location_type* yylocationp]])[)
+                           const location_type* yylocationp]])[) const
   {
-    *yycdebug_ << (yytype < YYNTOKENS ? "token" : "nterm")
-               << ' ' << yytname[yytype] << " ("]b4_locations_if([[
+    *yycdebug_ << (yykind < YYNTOKENS ? "token" : "nterm")
+               << ' ' << yytname[yykind] << " ("]b4_locations_if([[
                << *yylocationp << ": "]])[;
-    yy_symbol_value_print_ (yytype, yyvaluep]b4_locations_if([[, yylocationp]])[);
+    yy_symbol_value_print_ (yykind, yyvaluep]b4_locations_if([[, yylocationp]])[);
     *yycdebug_ << ')';
   }
 
@@ -238,6 +210,57 @@ b4_namespace_close[]dnl
 ])
 
 
+m4_define([b4_define_symbol_kind],
+[m4_format([#define %-15s %s],
+           b4_symbol($][1, kind_base),
+           b4_namespace_ref[::]b4_parser_class[::symbol_kind::]b4_symbol($1, kind_base))
+])
+
+# b4_glr_cc_setup
+# ---------------
+# Setup redirections for glr.c: Map the names used in c.m4 to the ones used
+# in c++.m4.
+m4_define([b4_glr_cc_setup],
+[[#undef ]b4_symbol(-2, [id])[
+#define ]b4_symbol(-2, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(-2, [id])[
+#undef ]b4_symbol(0, [id])[
+#define ]b4_symbol(0, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(0, [id])[
+#undef ]b4_symbol(1, [id])[
+#define ]b4_symbol(1, [id])[ ]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(1, [id])[
+
+#ifndef ]b4_api_PREFIX[STYPE
+# define ]b4_api_PREFIX[STYPE ]b4_namespace_ref[::]b4_parser_class[::semantic_type
+#endif
+#ifndef ]b4_api_PREFIX[LTYPE
+# define ]b4_api_PREFIX[LTYPE ]b4_namespace_ref[::]b4_parser_class[::location_type
+#endif
+
+typedef ]b4_namespace_ref[::]b4_parser_class[::symbol_kind_type yysymbol_kind_t;
+
+// Expose C++ symbol kinds to C.
+]b4_define_symbol_kind(-2)dnl
+b4_symbol_foreach([b4_define_symbol_kind])])[
+]])
+
+
+m4_define([b4_undef_symbol_kind],
+[[#undef ]b4_symbol($1, kind_base)[
+]])
+
+
+# b4_glr_cc_cleanup
+# -----------------
+# Remove redirections for glr.c.
+m4_define([b4_glr_cc_cleanup],
+[[#undef ]b4_symbol(-2, [id])[
+#undef ]b4_symbol(0, [id])[
+#undef ]b4_symbol(1, [id])[
+
+]b4_undef_symbol_kind(-2)dnl
+b4_symbol_foreach([b4_undef_symbol_kind])dnl
+])
+
+
 # b4_shared_declarations(hh|cc)
 # -----------------------------
 # Declaration that might either go into the header (if --defines, $1 = hh)
@@ -328,53 +351,43 @@ b4_percent_code_get([[requires]])[
 # if ]b4_api_PREFIX[DEBUG
   public:
     /// \brief Report a symbol value on the debug stream.
-    /// \param yytype       The token type.
+    /// \param yykind       The symbol kind.
     /// \param yyvaluep     Its semantic value.]b4_locations_if([[
     /// \param yylocationp  Its location.]])[
-    virtual void yy_symbol_value_print_ (int yytype,
+    virtual void yy_symbol_value_print_ (symbol_kind_type yykind,
                                          const semantic_type* yyvaluep]b4_locations_if([[,
-                                         const location_type* yylocationp]])[);
+                                         const location_type* yylocationp]])[) const;
     /// \brief Report a symbol on the debug stream.
-    /// \param yytype       The token type.
+    /// \param yykind       The symbol kind.
     /// \param yyvaluep     Its semantic value.]b4_locations_if([[
     /// \param yylocationp  Its location.]])[
-    virtual void yy_symbol_print_ (int yytype,
+    virtual void yy_symbol_print_ (symbol_kind_type yykind,
                                    const semantic_type* yyvaluep]b4_locations_if([[,
-                                   const location_type* yylocationp]])[);
+                                   const location_type* yylocationp]])[) const;
   private:
-    // Debugging.
+    /// Debug stream.
     std::ostream* yycdebug_;
 #endif
 
 ]b4_parse_param_vars[
   };
 
-]dnl Redirections for glr.c.
-b4_percent_define_flag_if([[global_tokens_and_yystype]],
-[b4_token_defines])
-[
-#ifndef ]b4_api_PREFIX[STYPE
-# define ]b4_api_PREFIX[STYPE ]b4_namespace_ref[::]b4_parser_class[::semantic_type
-#endif
-#ifndef ]b4_api_PREFIX[LTYPE
-# define ]b4_api_PREFIX[LTYPE ]b4_namespace_ref[::]b4_parser_class[::location_type
-#endif
-
 ]b4_namespace_close[
+
 ]b4_percent_code_get([[provides]])[
 ]m4_popdef([b4_parse_param])dnl
-])
+])[
 
-b4_defines_if(
+]b4_defines_if(
 [b4_output_begin([b4_spec_header_file])
 b4_copyright([Skeleton interface for Bison GLR parsers in C++],
              [2002-2015, 2018-2020])[
 // C++ GLR parser skeleton written by Akim Demaille.
 
 ]b4_disclaimer[
-]b4_cpp_guard_open([b4_spec_header_file])[
+]b4_cpp_guard_open([b4_spec_mapped_header_file])[
 ]b4_shared_declarations[
-]b4_cpp_guard_close([b4_spec_header_file])[
+]b4_cpp_guard_close([b4_spec_mapped_header_file])[
 ]b4_output_end])
 
 # Let glr.c (and b4_shared_declarations) believe that the user

data/skeletons/java.m4

@@ -106,10 +106,13 @@ m4_define([b4_null], [null])
 
 # b4_typed_parser_table_define(TYPE, NAME, DATA, COMMENT)
 # -------------------------------------------------------
+# We use intermediate functions (e.g., yypact_init) to work around the
+# 64KB limit for JVM methods.  See
+# https://lists.gnu.org/r/help-bison/2008-11/msg00004.html.
 m4_define([b4_typed_parser_table_define],
 [m4_ifval([$4], [b4_comment([$4])
   ])dnl
-[private static final ]$1[ yy$2_[] = yy$2_init();
+[private static final ]$1[[] yy$2_ = yy$2_init();
   private static final ]$1[[] yy$2_init()
   {
     return new ]$1[[]
@@ -125,35 +128,151 @@ m4_define([b4_integral_parser_table_define],
 [b4_typed_parser_table_define([b4_int_type_for([$2])], [$1], [$2], [$3])])
 
 
-## ------------------------- ##
-## Assigning token numbers.  ##
-## ------------------------- ##
+## ------------- ##
+## Token kinds.  ##
+## ------------- ##
+
 
 # b4_token_enum(TOKEN-NUM)
 # ------------------------
 # Output the definition of this token as an enum.
 m4_define([b4_token_enum],
-[b4_token_format([    /** Token number, to be returned by the scanner.  */
-    static final int %s = %s;
-], [$1])])
+[b4_token_visible_if([$1],
+    [m4_format([[    /** Token %s, to be returned by the scanner.  */
+    static final int %s = %s%s;
+]],
+               b4_symbol([$1], [tag]),
+               b4_symbol([$1], [id]),
+               b4_symbol([$1], b4_api_token_raw_if([[number]], [[code]])))])])
+
 
 # b4_token_enums
 # --------------
 # Output the definition of the tokens (if there are) as enums.
 m4_define([b4_token_enums],
-[b4_any_token_visible_if([/* Tokens.  */
+[b4_any_token_visible_if([    /* Token kinds.  */
 b4_symbol_foreach([b4_token_enum])])])
 
-# b4-case(ID, CODE)
-# -----------------
+
+
+## -------------- ##
+## Symbol kinds.  ##
+## -------------- ##
+
+
+# b4_symbol_kind(NUM)
+# -------------------
+m4_define([b4_symbol_kind],
+[SymbolKind.b4_symbol_kind_base($@)])
+
+
+# b4_symbol_enum(SYMBOL-NUM)
+# --------------------------
+# Output the definition of this symbol as an enum.
+m4_define([b4_symbol_enum],
+[m4_format([    %-30s %s],
+           m4_format([[%s(%s)%s]],
+                     b4_symbol([$1], [kind_base]),
+                     [$1],
+                     m4_if([$1], b4_last_symbol, [[;]], [[,]])),
+           [b4_symbol_tag_comment([$1])])])
+
+
+# b4_declare_symbol_enum
+# ----------------------
+# The definition of the symbol internal numbers as an enum.
+m4_define([b4_declare_symbol_enum],
+[[  public enum SymbolKind
+  {
+]b4_symbol_foreach([b4_symbol_enum])[
+
+    private final int yycode_;
+
+    SymbolKind (int n) {
+      this.yycode_ = n;
+    }
+
+    private static final SymbolKind[] values_ = {
+      ]m4_map_args_sep([b4_symbol_kind(], [)], [,
+      ], b4_symbol_numbers)[
+    };
+
+    static final SymbolKind get(int code) {
+      return values_[code];
+    }
+
+    public final int getCode() {
+      return this.yycode_;
+    }
+
+]b4_parse_error_bmatch(
+[simple\|verbose],
+[[    /* Return YYSTR after stripping away unnecessary quotes and
+       backslashes, so that it's suitable for yyerror.  The heuristic is
+       that double-quoting is unnecessary unless the string contains an
+       apostrophe, a comma, or backslash (other than backslash-backslash).
+       YYSTR is taken from yytname.  */
+    private static String yytnamerr_(String yystr)
+    {
+      if (yystr.charAt (0) == '"')
+        {
+          StringBuffer yyr = new StringBuffer();
+          strip_quotes: for (int i = 1; i < yystr.length(); i++)
+            switch (yystr.charAt(i))
+              {
+              case '\'':
+              case ',':
+                break strip_quotes;
+
+              case '\\':
+                if (yystr.charAt(++i) != '\\')
+                  break strip_quotes;
+                /* Fall through.  */
+              default:
+                yyr.append(yystr.charAt(i));
+                break;
+
+              case '"':
+                return yyr.toString();
+              }
+        }
+      return yystr;
+    }
+
+    /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+       First, the terminals, then, starting at \a YYNTOKENS_, nonterminals.  */
+    ]b4_typed_parser_table_define([String], [tname], [b4_tname])[
+
+    /* The user-facing name of this symbol.  */
+    public final String getName() {
+      return yytnamerr_(yytname_[yycode_]);
+    }
+]],
+[custom\|detailed],
+[[    /* YYNAMES_[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+       First, the terminals, then, starting at \a YYNTOKENS_, nonterminals.  */
+    ]b4_typed_parser_table_define([String], [names], [b4_symbol_names])[
+
+    /* The user-facing name of this symbol.  */
+    public final String getName() {
+      return yynames_[yycode_];
+    }]])[
+  };
+]])])
+
+
+
+# b4_case(ID, CODE, [COMMENTS])
+# -----------------------------
 # We need to fool Java's stupid unreachable code detection.
 m4_define([b4_case],
-[  case $1:
+[  case $1:m4_ifval([$3], [ b4_comment([$3])])
   if (yyn == $1)
     $2;
   break;
 ])
 
+
 # b4_predicate_case(LABEL, CONDITIONS)
 # ------------------------------------
 m4_define([b4_predicate_case],
@@ -185,6 +304,7 @@ b4_percent_define_check_kind([[throws]],            [code], [deprecated])
 
 m4_define([b4_yystype], [b4_percent_define_get([[api.value.type]])])
 b4_percent_define_default([[api.value.type]], [[Object]])
+b4_percent_define_default([[api.symbol.prefix]], [[S_]])
 
 # b4_api_prefix, b4_api_PREFIX
 # ----------------------------
@@ -224,6 +344,22 @@ m4_define([b4_position_type], [b4_percent_define_get([[api.position.type]])])
 ## ----------------- ##
 
 
+# b4_symbol_translate(STRING)
+# ---------------------------
+# Used by "bison" in the array of symbol names to mark those that
+# require translation.
+m4_define([b4_symbol_translate],
+[[i18n($1)]])
+
+
+# b4_trans(STRING)
+# ----------------
+# Translate a string if i18n is enabled.  Avoid collision with b4_translate.
+m4_define([b4_trans],
+[b4_has_translations_if([i18n($1)], [$1])])
+
+
+
 # b4_symbol_value(VAL, [SYMBOL-NUM], [TYPE-TAG])
 # ----------------------------------------------
 # See README.

data/skeletons/lalr1.cc

@@ -30,6 +30,15 @@ m4_define([b4_lac_flag],
                  [none], [[0]], [[1]])])
 
 
+# b4_tname_if(TNAME-NEEDED, TNAME-NOT-NEEDED)
+# -------------------------------------------
+m4_define([b4_tname_if],
+[m4_case(b4_percent_define_get([[parse.error]]),
+         [verbose],         [$1],
+         [b4_token_table_if([$1],
+                            [$2])])])
+
+
 # b4_integral_parser_table_declare(TABLE-NAME, CONTENT, COMMENT)
 # --------------------------------------------------------------
 # Declare "parser::yy<TABLE-NAME>_" whose contents is CONTENT.
@@ -50,6 +59,13 @@ m4_define([b4_integral_parser_table_define],
   };dnl
 ])
 
+
+# b4_symbol_kind(NUM)
+# -------------------
+m4_define([b4_symbol_kind],
+[symbol_kind::b4_symbol_kind_base($@)])
+
+
 # b4_symbol_value_template(VAL, SYMBOL-NUM, [TYPE])
 # -------------------------------------------------
 # Same as b4_symbol_value, but used in a template method.  It makes
@@ -195,6 +211,13 @@ m4_define([b4_shared_declarations],
     ]b4_parser_class[ (]b4_parse_param_decl[);
     virtual ~]b4_parser_class[ ();
 
+#if 201103L <= YY_CPLUSPLUS
+    /// Non copyable.
+    ]b4_parser_class[ (const ]b4_parser_class[&) = delete;
+    /// Non copyable.
+    ]b4_parser_class[& operator= (const ]b4_parser_class[&) = delete;
+#endif
+
     /// Parse.  An alias for parse ().
     /// \returns  0 iff parsing succeeded.
     int operator() ();
@@ -225,19 +248,57 @@ m4_define([b4_shared_declarations],
     /// Report a syntax error.
     void error (const syntax_error& err);
 
+]b4_parse_error_bmatch(
+[custom\|detailed],
+[[    /// The user-facing name of the symbol whose (internal) number is
+    /// YYSYMBOL.  No bounds checking.
+    static const char *symbol_name (symbol_kind_type yysymbol);]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+    /// The user-facing name of the symbol whose (internal) number is
+    /// YYSYMBOL.  No bounds checking.
+    static const char *symbol_name (symbol_kind_type yysymbol);
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[    /// The user-facing name of the symbol whose (internal) number is
+    /// YYSYMBOL.  No bounds checking.
+    static std::string symbol_name (symbol_kind_type yysymbol);]])[
+
 ]b4_token_constructor_define[
+]b4_parse_error_bmatch([custom\|detailed\|verbose], [[
+    class context
+    {
+    public:
+      context (const ]b4_parser_class[& yyparser, const symbol_type& yyla);
+      const symbol_type& lookahead () const { return yyla_; }
+      symbol_kind_type token () const { return yyla_.kind (); }]b4_locations_if([[
+      const location_type& location () const { return yyla_.location; }
+]])[
+      /// Put in YYARG at most YYARGN of the expected tokens, and return the
+      /// number of tokens stored in YYARG.  If YYARG is null, return the
+      /// number of expected tokens (guaranteed to be less than YYNTOKENS).
+      int expected_tokens (symbol_kind_type yyarg[], int yyargn) const;
 
+    private:
+      const ]b4_parser_class[& yyparser_;
+      const symbol_type& yyla_;
+    };
+]])[
   private:
-    /// This class is not copyable.
+#if YY_CPLUSPLUS < 201103L
+    /// Non copyable.
     ]b4_parser_class[ (const ]b4_parser_class[&);
-    ]b4_parser_class[& operator= (const ]b4_parser_class[&);]b4_lac_if([[
-
+    /// Non copyable.
+    ]b4_parser_class[& operator= (const ]b4_parser_class[&);
+#endif
+]b4_lac_if([[
     /// Check the lookahead yytoken.
     /// \returns  true iff the token will be eventually shifted.
-    bool yy_lac_check_ (int yytoken) const;
+    bool yy_lac_check_ (symbol_kind_type yytoken) const;
     /// Establish the initial context if no initial context currently exists.
     /// \returns  true iff the token will be eventually shifted.
-    bool yy_lac_establish_ (int yytoken);
+    bool yy_lac_establish_ (symbol_kind_type yytoken);
     /// Discard any previous initial lookahead context because of event.
     /// \param event  the event which caused the lookahead to be discarded.
     ///               Only used for debbuging output.
@@ -245,13 +306,19 @@ m4_define([b4_shared_declarations],
 
     /// Stored state numbers (used for stacks).
     typedef ]b4_int_type(0, m4_eval(b4_states_number - 1))[ state_type;
+]b4_parse_error_bmatch(
+      [custom], [[
+    /// Report a syntax error
+    /// \param yyctx     the context in which the error occurred.
+    void report_syntax_error (const context& yyctx) const;]],
+      [detailed\|verbose], [[
+    /// The arguments of the error message.
+    int yy_syntax_error_arguments_ (const context& yyctx,
+                                    symbol_kind_type yyarg[], int yyargn) const;
 
     /// Generate an error message.
-    /// \param yystate   the state where the error occurred.
-    /// \param yyla      the lookahead token.
-    virtual std::string yysyntax_error_ (state_type yystate,
-                                         const symbol_type& yyla) const;
-
+    /// \param yyctx     the context in which the error occurred.
+    virtual std::string yysyntax_error_ (const context& yyctx) const;]])[
     /// Compute post-reduction state.
     /// \param yystate   the current state
     /// \param yysym     the nonterminal to push on the stack
@@ -268,34 +335,43 @@ m4_define([b4_shared_declarations],
     static const ]b4_int_type(b4_pact_ninf, b4_pact_ninf)[ yypact_ninf_;
     static const ]b4_int_type(b4_table_ninf, b4_table_ninf)[ yytable_ninf_;
 
-    /// Convert a scanner token number \a t to a symbol number.
-    /// In theory \a t should be a token_type, but character literals
+    /// Convert a scanner token kind \a t to a symbol kind.
+    /// In theory \a t should be a token_kind_type, but character literals
     /// are valid, yet not members of the token_type enum.
-    static token_number_type yytranslate_ (int t);
+    static symbol_kind_type yytranslate_ (int t);
 
-    // Tables.
-]b4_parser_tables_declare[]b4_error_verbose_if([
-
-    /// Convert the symbol name \a n to a form suitable for a diagnostic.
-    static std::string yytnamerr_ (const char *n);])[
-
-]b4_token_table_if([], [[#if ]b4_api_PREFIX[DEBUG]])[
+]b4_parse_error_bmatch(
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
     /// For a symbol, its name in clear.
     static const char* const yytname_[];
-]b4_token_table_if([[#if ]b4_api_PREFIX[DEBUG]])[
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[    /// Convert the symbol name \a n to a form suitable for a diagnostic.
+    static std::string yytnamerr_ (const char *yystr);
+
+    /// For a symbol, its name in clear.
+    static const char* const yytname_[];
+]])[
+
+    // Tables.
+]b4_parser_tables_declare[
+
+#if ]b4_api_PREFIX[DEBUG
 ]b4_integral_parser_table_declare([rline], [b4_rline],
      [[YYRLINE[YYN] -- Source line where rule number YYN was defined.]])[
     /// Report on the debug stream that the rule \a r is going to be reduced.
-    virtual void yy_reduce_print_ (int r);
+    virtual void yy_reduce_print_ (int r) const;
     /// Print the state stack on the debug stream.
-    virtual void yystack_print_ ();
+    virtual void yy_stack_print_ () const;
 
     /// Debugging level.
     int yydebug_;
     /// Debug stream.
     std::ostream* yycdebug_;
 
-    /// \brief Display a symbol type, value and location.
+    /// \brief Display a symbol kind, value and location.
     /// \param yyo    The output stream.
     /// \param yysym  The symbol.
     template <typename Base>
@@ -316,7 +392,7 @@ m4_define([b4_shared_declarations],
       /// Default constructor.
       by_state () YY_NOEXCEPT;
 
-      /// The symbol type as needed by the constructor.
+      /// The symbol kind as needed by the constructor.
       typedef state_type kind_type;
 
       /// Constructor.
@@ -328,12 +404,12 @@ m4_define([b4_shared_declarations],
       /// Record that this symbol is empty.
       void clear () YY_NOEXCEPT;
 
-      /// Steal the symbol type from \a that.
+      /// Steal the symbol kind from \a that.
       void move (by_state& that);
 
-      /// The (internal) type number (corresponding to \a state).
-      /// \a empty_symbol when empty.
-      symbol_number_type type_get () const YY_NOEXCEPT;
+      /// The symbol kind (corresponding to \a state).
+      /// \a ]b4_symbol(-2, kind)[ when empty.
+      symbol_kind_type kind () const YY_NOEXCEPT;
 
       /// The state number used to denote an empty symbol.
       /// We use the initial state, as it does not have a value.
@@ -401,35 +477,22 @@ m4_define([b4_shared_declarations],
     /// Pop \a n symbols from the stack.
     void yypop_ (int n = 1);
 
-    /// Some specific tokens.
-    static const token_number_type yy_error_token_ = 1;
-    static const token_number_type yy_undef_token_ = ]b4_undef_token_number[;
-
     /// Constants.
     enum
     {
-      yyeof_ = 0,
       yylast_ = ]b4_last[,     ///< Last index in yytable_.
       yynnts_ = ]b4_nterms_number[,  ///< Number of nonterminal symbols.
-      yyfinal_ = ]b4_final_state_number[, ///< Termination state number.
-      yyntokens_ = ]b4_tokens_number[  ///< Number of tokens.
+      yyfinal_ = ]b4_final_state_number[ ///< Termination state number.
     };
 
 ]b4_parse_param_vars[
+]b4_percent_code_get([[yy_bison_internal_hook]])[
   };
 
 ]b4_token_ctor_if([b4_yytranslate_define([$1])[
 ]b4_public_types_define([$1])])[
 ]b4_namespace_close[
 
-]b4_percent_define_flag_if([[global_tokens_and_yystype]],
-[b4_token_defines
-
-#ifndef ]b4_api_PREFIX[STYPE
- // Redirection for backward compatibility.
-# define ]b4_api_PREFIX[STYPE b4_namespace_ref::b4_parser_class::semantic_type
-#endif
-])[
 ]b4_percent_code_get([[provides]])[
 ]])
 
@@ -442,16 +505,16 @@ b4_defines_if(
 b4_copyright([Skeleton interface for Bison LALR(1) parsers in C++])
 [
 /**
- ** \file ]b4_spec_header_file[
+ ** \file ]b4_spec_mapped_header_file[
  ** Define the ]b4_namespace_ref[::parser class.
  */
 
 // C++ LALR(1) parser skeleton written by Akim Demaille.
 
 ]b4_disclaimer[
-]b4_cpp_guard_open([b4_spec_header_file])[
+]b4_cpp_guard_open([b4_spec_mapped_header_file])[
 ]b4_shared_declarations(hh)[
-]b4_cpp_guard_close([b4_spec_header_file])[
+]b4_cpp_guard_close([b4_spec_mapped_header_file])[
 ]b4_output_end[
 ]])
 
@@ -484,6 +547,11 @@ m4_if(b4_prefix, [yy], [],
 #  define YY_(msgid) msgid
 # endif
 #endif
+]b4_has_translations_if([
+#ifndef N_
+# define N_(Msgid) Msgid
+#endif
+])[
 
 // Whether we are compiled with exception support.
 #ifndef YY_EXCEPTIONS
@@ -523,7 +591,7 @@ m4_if(b4_prefix, [yy], [],
 # define YY_STACK_PRINT()               \
   do {                                  \
     if (yydebug_)                       \
-      yystack_print_ ();                \
+      yy_stack_print_ ();                \
   } while (false)
 
 #else // !]b4_api_PREFIX[DEBUG
@@ -543,49 +611,7 @@ m4_if(b4_prefix, [yy], [],
 #define YYERROR         goto yyerrorlab
 #define YYRECOVERING()  (!!yyerrstatus_)
 
-]b4_namespace_open[]b4_error_verbose_if([[
-
-  /* Return YYSTR after stripping away unnecessary quotes and
-     backslashes, so that it's suitable for yyerror.  The heuristic is
-     that double-quoting is unnecessary unless the string contains an
-     apostrophe, a comma, or backslash (other than backslash-backslash).
-     YYSTR is taken from yytname.  */
-  std::string
-  ]b4_parser_class[::yytnamerr_ (const char *yystr)
-  {
-    if (*yystr == '"')
-      {
-        std::string yyr;
-        char const *yyp = yystr;
-
-        for (;;)
-          switch (*++yyp)
-            {
-            case '\'':
-            case ',':
-              goto do_not_strip_quotes;
-
-            case '\\':
-              if (*++yyp != '\\')
-                goto do_not_strip_quotes;
-              else
-                goto append;
-
-            append:
-            default:
-              yyr += *yyp;
-              break;
-
-            case '"':
-              return yyr;
-            }
-      do_not_strip_quotes: ;
-      }
-
-    return yystr;
-  }
-]])[
-
+]b4_namespace_open[
   /// Build a parser object.
   ]b4_parser_class::b4_parser_class[ (]b4_parse_param_decl[)
 #if ]b4_api_PREFIX[DEBUG
@@ -604,7 +630,7 @@ m4_if(b4_prefix, [yy], [],
   {}
 
   /*---------------.
-  | Symbol types.  |
+  | symbol kinds.  |
   `---------------*/
 
 ]b4_token_ctor_if([], [b4_public_types_define([cc])])[
@@ -635,13 +661,13 @@ m4_if(b4_prefix, [yy], [],
     : state (s)
   {}
 
-  ]b4_parser_class[::symbol_number_type
-  ]b4_parser_class[::by_state::type_get () const YY_NOEXCEPT
+  ]b4_parser_class[::symbol_kind_type
+  ]b4_parser_class[::by_state::kind () const YY_NOEXCEPT
   {
     if (state == empty_state)
-      return empty_symbol;
+      return ]b4_symbol(-2, kind)[;
     else
-      return yystos_[+state];
+      return YY_CAST (symbol_kind_type, yystos_[+state]);
   }
 
   ]b4_parser_class[::stack_symbol_type::stack_symbol_type ()
@@ -650,7 +676,7 @@ m4_if(b4_prefix, [yy], [],
   ]b4_parser_class[::stack_symbol_type::stack_symbol_type (YY_RVREF (stack_symbol_type) that)
     : super_type (YY_MOVE (that.state)]b4_variant_if([], [, YY_MOVE (that.value)])b4_locations_if([, YY_MOVE (that.location)])[)
   {]b4_variant_if([
-    b4_symbol_variant([that.type_get ()],
+    b4_symbol_variant([that.kind ()],
                       [value], [YY_MOVE_OR_COPY], [YY_MOVE (that.value)])])[
 #if 201103L <= YY_CPLUSPLUS
     // that is emptied.
@@ -661,10 +687,10 @@ m4_if(b4_prefix, [yy], [],
   ]b4_parser_class[::stack_symbol_type::stack_symbol_type (state_type s, YY_MOVE_REF (symbol_type) that)
     : super_type (s]b4_variant_if([], [, YY_MOVE (that.value)])[]b4_locations_if([, YY_MOVE (that.location)])[)
   {]b4_variant_if([
-    b4_symbol_variant([that.type_get ()],
+    b4_symbol_variant([that.kind ()],
                       [value], [move], [YY_MOVE (that.value)])])[
     // that is emptied.
-    that.type = empty_symbol;
+    that.kind_ = ]b4_symbol(-2, kind)[;
   }
 
 #if YY_CPLUSPLUS < 201103L
@@ -672,7 +698,7 @@ m4_if(b4_prefix, [yy], [],
   ]b4_parser_class[::stack_symbol_type::operator= (const stack_symbol_type& that)
   {
     state = that.state;
-    ]b4_variant_if([b4_symbol_variant([that.type_get ()],
+    ]b4_variant_if([b4_symbol_variant([that.kind ()],
                                       [value], [copy], [that.value])],
                    [[value = that.value;]])[]b4_locations_if([
     location = that.location;])[
@@ -683,7 +709,7 @@ m4_if(b4_prefix, [yy], [],
   ]b4_parser_class[::stack_symbol_type::operator= (stack_symbol_type& that)
   {
     state = that.state;
-    ]b4_variant_if([b4_symbol_variant([that.type_get ()],
+    ]b4_variant_if([b4_symbol_variant([that.kind ()],
                                       [value], [move], [that.value])],
                    [[value = that.value;]])[]b4_locations_if([
     location = that.location;])[
@@ -701,29 +727,27 @@ m4_if(b4_prefix, [yy], [],
       YY_SYMBOL_PRINT (yymsg, yysym);]b4_variant_if([], [
 
     // User destructor.
-    b4_symbol_actions([destructor], [yysym.type_get ()])])[
+    b4_symbol_actions([destructor], [yysym.kind ()])])[
   }
 
 #if ]b4_api_PREFIX[DEBUG
   template <typename Base>
   void
-  ]b4_parser_class[::yy_print_ (std::ostream& yyo,
-                                     const basic_symbol<Base>& yysym) const
+  ]b4_parser_class[::yy_print_ (std::ostream& yyo, const basic_symbol<Base>& yysym) const
   {
     std::ostream& yyoutput = yyo;
     YYUSE (yyoutput);
-    symbol_number_type yytype = yysym.type_get ();
-#if defined __GNUC__ && ! defined __clang__ && ! defined __ICC && __GNUC__ * 100 + __GNUC_MINOR__ <= 408
-    // Avoid a (spurious) G++ 4.8 warning about "array subscript is
-    // below array bounds".
     if (yysym.empty ())
-      std::abort ();
-#endif
-    yyo << (yytype < yyntokens_ ? "token" : "nterm")
-        << ' ' << yytname_[yytype] << " ("]b4_locations_if([
-        << yysym.location << ": "])[;
-    ]b4_symbol_actions([printer])[
-    yyo << ')';
+      yyo << "empty symbol";
+    else
+      {
+        symbol_kind_type yykind = yysym.kind ();
+        yyo << (yykind < YYNTOKENS ? "token" : "nterm")
+            << ' ' << yysym.name () << " ("]b4_locations_if([
+            << yysym.location << ": "])[;
+        ]b4_symbol_actions([printer])[
+        yyo << ')';
+      }
   }
 #endif
 
@@ -782,11 +806,11 @@ m4_if(b4_prefix, [yy], [],
   ]b4_parser_class[::state_type
   ]b4_parser_class[::yy_lr_goto_state_ (state_type yystate, int yysym)
   {
-    int yyr = yypgoto_[yysym - yyntokens_] + yystate;
+    int yyr = yypgoto_[yysym - YYNTOKENS] + yystate;
     if (0 <= yyr && yyr <= yylast_ && yycheck_[yyr] == yystate)
       return yytable_[yyr];
     else
-      return yydefgoto_[yysym - yyntokens_];
+      return yydefgoto_[yysym - YYNTOKENS];
   }
 
   bool
@@ -854,6 +878,7 @@ b4_dollar_popdef])[]dnl
   `-----------------------------------------------*/
   yynewstate:
     YYCDEBUG << "Entering state " << int (yystack_[0].state) << '\n';
+    YY_STACK_PRINT ();
 
     // Accept?
     if (yystack_[0].state == yyfinal_)
@@ -874,14 +899,14 @@ b4_dollar_popdef])[]dnl
     // Read a lookahead token.
     if (yyla.empty ())
       {
-        YYCDEBUG << "Reading a token: ";
+        YYCDEBUG << "Reading a token\n";
 #if YY_EXCEPTIONS
         try
 #endif // YY_EXCEPTIONS
           {]b4_token_ctor_if([[
             symbol_type yylookahead (]b4_lex[);
             yyla.move (yylookahead);]], [[
-            yyla.type = yytranslate_ (]b4_lex[);]])[
+            yyla.kind_ = yytranslate_ (]b4_lex[);]])[
           }
 #if YY_EXCEPTIONS
         catch (const syntax_error& yyexc)
@@ -894,12 +919,22 @@ b4_dollar_popdef])[]dnl
       }
     YY_SYMBOL_PRINT ("Next token is", yyla);
 
+    if (yyla.kind () == ]b4_symbol(1, kind)[)
+    {
+      // The scanner already issued an error message, process directly
+      // to error recovery.  But do not keep the error token as
+      // lookahead, it is too special and may lead us to an endless
+      // loop in error recovery. */
+      yyla.kind_ = ]b4_symbol(2, kind)[;
+      goto yyerrlab1;
+    }
+
     /* If the proper action on seeing token YYLA.TYPE is to reduce or
        to detect an error, take that action.  */
-    yyn += yyla.type_get ();
-    if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yyla.type_get ())
+    yyn += yyla.kind ();
+    if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yyla.kind ())
       {]b4_lac_if([[
-        if (!yy_lac_establish_ (yyla.type_get ()))
+        if (!yy_lac_establish_ (yyla.kind ()))
            goto yyerrlab;]])[
         goto yydefault;
       }
@@ -910,7 +945,7 @@ b4_dollar_popdef])[]dnl
       {
         if (yy_table_value_is_error_ (yyn))
           goto yyerrlab;]b4_lac_if([[
-        if (!yy_lac_establish_ (yyla.type_get ()))
+        if (!yy_lac_establish_ (yyla.kind ()))
            goto yyerrlab;
 ]])[
         yyn = -yyn;
@@ -992,7 +1027,6 @@ b4_dollar_popdef])[]dnl
       YY_SYMBOL_PRINT ("-> $$ =", yylhs);
       yypop_ (yylen);
       yylen = 0;
-      YY_STACK_PRINT ();
 
       // Shift the result of the reduction.
       yypush_ (YY_NULLPTR, YY_MOVE (yylhs));
@@ -1007,9 +1041,17 @@ b4_dollar_popdef])[]dnl
     // If not already recovering from an error, report this error.
     if (!yyerrstatus_)
       {
-        ++yynerrs_;
-        error (]b4_join(b4_locations_if([yyla.location]),
-                        [[yysyntax_error_ (yystack_[0].state, yyla)]])[);
+        ++yynerrs_;]b4_parse_error_case(
+                  [simple], [[
+        std::string msg = YY_("syntax error");
+        error (]b4_join(b4_locations_if([yyla.location]), [[YY_MOVE (msg)]])[);]],
+                  [custom], [[
+        context yyctx (*this, yyla);
+        report_syntax_error (yyctx);]],
+                  [[
+        context yyctx (*this, yyla);
+        std::string msg = yysyntax_error_ (yyctx);
+        error (]b4_join(b4_locations_if([yyla.location]), [[YY_MOVE (msg)]])[);]])[
       }
 
 ]b4_locations_if([[
@@ -1020,7 +1062,7 @@ b4_dollar_popdef])[]dnl
            error, discard it.  */
 
         // Return failure if at end of input.
-        if (yyla.type_get () == yyeof_)
+        if (yyla.kind () == ]b4_symbol(0, kind)[)
           YYABORT;
         else if (!yyla.empty ())
           {
@@ -1046,6 +1088,7 @@ b4_dollar_popdef])[]dnl
        this YYERROR.  */
     yypop_ (yylen);
     yylen = 0;
+    YY_STACK_PRINT ();
     goto yyerrlab1;
 
 
@@ -1054,31 +1097,33 @@ b4_dollar_popdef])[]dnl
   `-------------------------------------------------------------*/
   yyerrlab1:
     yyerrstatus_ = 3;   // Each real token shifted decrements this.
+    // Pop stack until we find a state that shifts the error token.
+    for (;;)
+      {
+        yyn = yypact_[+yystack_[0].state];
+        if (!yy_pact_value_is_default_ (yyn))
+          {
+            yyn += ]b4_symbol(1, kind)[;
+            if (0 <= yyn && yyn <= yylast_
+                && yycheck_[yyn] == ]b4_symbol(1, kind)[)
+              {
+                yyn = yytable_[yyn];
+                if (0 < yyn)
+                  break;
+              }
+          }
+
+        // Pop the current state because it cannot handle the error token.
+        if (yystack_.size () == 1)
+          YYABORT;
+]b4_locations_if([[
+        yyerror_range[1].location = yystack_[0].location;]])[
+        yy_destroy_ ("Error: popping", yystack_[0]);
+        yypop_ ();
+        YY_STACK_PRINT ();
+      }
     {
       stack_symbol_type error_token;
-      for (;;)
-        {
-          yyn = yypact_[+yystack_[0].state];
-          if (!yy_pact_value_is_default_ (yyn))
-            {
-              yyn += yy_error_token_;
-              if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == yy_error_token_)
-                {
-                  yyn = yytable_[yyn];
-                  if (0 < yyn)
-                    break;
-                }
-            }
-
-          // Pop the current state because it cannot handle the error token.
-          if (yystack_.size () == 1)
-            YYABORT;
-]b4_locations_if([[
-          yyerror_range[1].location = yystack_[0].location;]])[
-          yy_destroy_ ("Error: popping", yystack_[0]);
-          yypop_ ();
-          YY_STACK_PRINT ();
-        }
 ]b4_locations_if([[
       yyerror_range[2].location = yyla.location;
       YYLLOC_DEFAULT (error_token.location, yyerror_range, 2);]])[
@@ -1117,6 +1162,7 @@ b4_dollar_popdef])[]dnl
     /* Do not reclaim the symbols of the rule whose action triggered
        this YYABORT or YYACCEPT.  */
     yypop_ (yylen);
+    YY_STACK_PRINT ();
     while (1 < yystack_.size ())
       {
         yy_destroy_ ("Cleanup: popping", yystack_[0]);
@@ -1149,17 +1195,158 @@ b4_dollar_popdef])[]dnl
   {
     error (]b4_join(b4_locations_if([yyexc.location]),
                     [[yyexc.what ()]])[);
-  }]b4_lac_if([[
+  }
 
+]b4_parse_error_bmatch([custom\|detailed],
+[[  const char *
+  ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+  {
+    static const char *const yy_sname[] =
+    {
+    ]b4_symbol_names[
+    };]b4_has_translations_if([[
+    /* YYTRANSLATABLE[SYMBOL-NUM] -- Whether YY_SNAME[SYMBOL-NUM] is
+       internationalizable.  */
+    static ]b4_int_type_for([b4_translatable])[ yytranslatable[] =
+    {
+    ]b4_translatable[
+    };
+    return (yysymbol < YYNTOKENS && yytranslatable[yysymbol]
+            ? _(yy_sname[yysymbol])
+            : yy_sname[yysymbol]);]], [[
+    return yy_sname[yysymbol];]])[
+  }
+]],
+[simple],
+[[#if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+  const char *
+  ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+  {
+    return yytname_[yysymbol];
+  }
+#endif // #if ]b4_api_PREFIX[DEBUG || ]b4_token_table_flag[
+]],
+[verbose],
+[[  /* Return YYSTR after stripping away unnecessary quotes and
+     backslashes, so that it's suitable for yyerror.  The heuristic is
+     that double-quoting is unnecessary unless the string contains an
+     apostrophe, a comma, or backslash (other than backslash-backslash).
+     YYSTR is taken from yytname.  */
+  std::string
+  ]b4_parser_class[::yytnamerr_ (const char *yystr)
+  {
+    if (*yystr == '"')
+      {
+        std::string yyr;
+        char const *yyp = yystr;
+
+        for (;;)
+          switch (*++yyp)
+            {
+            case '\'':
+            case ',':
+              goto do_not_strip_quotes;
+
+            case '\\':
+              if (*++yyp != '\\')
+                goto do_not_strip_quotes;
+              else
+                goto append;
+
+            append:
+            default:
+              yyr += *yyp;
+              break;
+
+            case '"':
+              return yyr;
+            }
+      do_not_strip_quotes: ;
+      }
+
+    return yystr;
+  }
+
+  std::string
+  ]b4_parser_class[::symbol_name (symbol_kind_type yysymbol)
+  {
+    return yytnamerr_ (yytname_[yysymbol]);
+  }
+]])[
+
+]b4_parse_error_bmatch([custom\|detailed\|verbose], [[
+  // ]b4_parser_class[::context.
+  ]b4_parser_class[::context::context (const ]b4_parser_class[& yyparser, const symbol_type& yyla)
+    : yyparser_ (yyparser)
+    , yyla_ (yyla)
+  {}
+
+  int
+  ]b4_parser_class[::context::expected_tokens (symbol_kind_type yyarg[], int yyargn) const
+  {
+    // Actual number of expected tokens
+    int yycount = 0;
+]b4_lac_if([[
+#if ]b4_api_PREFIX[DEBUG
+    // Execute LAC once. We don't care if it is successful, we
+    // only do it for the sake of debugging output.
+    if (!yyparser_.yy_lac_established_)
+      yyparser_.yy_lac_check_ (yyla_.kind ());
+#endif
+
+    for (int yyx = 0; yyx < YYNTOKENS; ++yyx)
+      {
+        symbol_kind_type yysym = YY_CAST (symbol_kind_type, yyx);
+        if (yysym != ]b4_symbol(1, kind)[
+            && yysym != ]b4_symbol(2, kind)[
+            && yyparser_.yy_lac_check_ (yysym))
+          {
+            if (!yyarg)
+              ++yycount;
+            else if (yycount == yyargn)
+              return 0;
+            else
+              yyarg[yycount++] = yysym;
+          }
+      }]], [[
+    int yyn = yypact_[+yyparser_.yystack_[0].state];
+    if (!yy_pact_value_is_default_ (yyn))
+      {
+        /* Start YYX at -YYN if negative to avoid negative indexes in
+           YYCHECK.  In other words, skip the first -YYN actions for
+           this state because they are default actions.  */
+        int yyxbegin = yyn < 0 ? -yyn : 0;
+        // Stay within bounds of both yycheck and yytname.
+        int yychecklim = yylast_ - yyn + 1;
+        int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
+        for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
+          if (yycheck_[yyx + yyn] == yyx && yyx != ]b4_symbol(1, kind)[
+              && !yy_table_value_is_error_ (yytable_[yyx + yyn]))
+            {
+              if (!yyarg)
+                ++yycount;
+              else if (yycount == yyargn)
+                return 0;
+              else
+                yyarg[yycount++] = YY_CAST (symbol_kind_type, yyx);
+            }
+      }
+]])[
+    if (yyarg && yycount == 0 && 0 < yyargn)
+      yyarg[0] = ]b4_symbol(-2, kind)[;
+    return yycount;
+  }
+
+]])b4_lac_if([[
   bool
-  ]b4_parser_class[::yy_lac_check_ (int yytoken) const
+  ]b4_parser_class[::yy_lac_check_ (symbol_kind_type yytoken) const
   {
     // Logically, the yylac_stack's lifetime is confined to this function.
     // Clear it, to get rid of potential left-overs from previous call.
     yylac_stack_.clear ();
     // Reduce until we encounter a shift and thereby accept the token.
 #if ]b4_api_PREFIX[DEBUG
-    YYCDEBUG << "LAC: checking lookahead " << yytname_[yytoken] << ':';
+    YYCDEBUG << "LAC: checking lookahead " << symbol_name (yytoken) << ':';
 #endif
     std::ptrdiff_t lac_top = 0;
     while (true)
@@ -1223,14 +1410,14 @@ b4_dollar_popdef])[]dnl
                      : yylac_stack_.back ());
         // Push the resulting state of the reduction.
         state_type state = yy_lr_goto_state_ (top_state, yyr1_[yyrule]);
-        YYCDEBUG << " G" << state;
+        YYCDEBUG << " G" << int (state);
         yylac_stack_.push_back (state);
       }
   }
 
   // Establish the initial context if no initial context currently exists.
   bool
-  ]b4_parser_class[::yy_lac_establish_ (int yytoken)
+  ]b4_parser_class[::yy_lac_establish_ (symbol_kind_type yytoken)
   {
     /* Establish the initial context for the current lookahead if no initial
        context is currently established.
@@ -1259,7 +1446,7 @@ b4_dollar_popdef])[]dnl
       {
 #if ]b4_api_PREFIX[DEBUG
         YYCDEBUG << "LAC: initial context established for "
-                 << yytname_[yytoken] << '\n';
+                 << symbol_name (yytoken) << '\n';
 #endif
         yy_lac_established_ = true;
         return yy_lac_check_ (yytoken);
@@ -1288,22 +1475,12 @@ b4_dollar_popdef])[]dnl
                  << evt << '\n';
         yy_lac_established_ = false;
       }
-  }]])[
-
-  // Generate an error message.
-  std::string
-  ]b4_parser_class[::yysyntax_error_ (]dnl
-b4_error_verbose_if([state_type yystate, const symbol_type& yyla],
-                    [state_type, const symbol_type&])[) const
-  {]b4_error_verbose_if([[
-    // Number of reported tokens (one for the "unexpected", one per
-    // "expected").
-    std::ptrdiff_t yycount = 0;
-    // Its maximum.
-    enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 };
-    // Arguments of yyformat.
-    char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
+  }]])b4_parse_error_bmatch([detailed\|verbose], [[
 
+  int
+  ]b4_parser_class[::yy_syntax_error_arguments_ (const context& yyctx,
+                                                 symbol_kind_type yyarg[], int yyargn) const
+  {
     /* There are many possibilities here to consider:
        - If this state is a consistent state with a default action, then
          the only way this function was invoked is if the default action
@@ -1334,46 +1511,26 @@ b4_error_verbose_if([state_type yystate, const symbol_type& yyla],
          one exception: it will still contain any token that will not be
          accepted due to an error action in a later state.]])[
     */
-    if (!yyla.empty ())
+
+    if (!yyctx.lookahead ().empty ())
       {
-        symbol_number_type yytoken = yyla.type_get ();
-        yyarg[yycount++] = yytname_[yytoken];]b4_lac_if([[
-
-#if ]b4_api_PREFIX[DEBUG
-        // Execute LAC once. We don't care if it is succesful, we
-        // only do it for the sake of debugging output.
-        if (!yy_lac_established_)
-          yy_lac_check_ (yytoken);
-#endif]])[
-
-        int yyn = yypact_[+yystate];
-        if (!yy_pact_value_is_default_ (yyn))
-          {]b4_lac_if([[
-            for (int yyx = 0; yyx < yyntokens_; ++yyx)
-              if (yyx != yy_error_token_ && yyx != yy_undef_token_
-                  && yy_lac_check_ (yyx))
-                {]], [[
-            /* Start YYX at -YYN if negative to avoid negative indexes in
-               YYCHECK.  In other words, skip the first -YYN actions for
-               this state because they are default actions.  */
-            int yyxbegin = yyn < 0 ? -yyn : 0;
-            // Stay within bounds of both yycheck and yytname.
-            int yychecklim = yylast_ - yyn + 1;
-            int yyxend = yychecklim < yyntokens_ ? yychecklim : yyntokens_;
-            for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
-              if (yycheck_[yyx + yyn] == yyx && yyx != yy_error_token_
-                  && !yy_table_value_is_error_ (yytable_[yyx + yyn]))
-                {]])[
-                  if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
-                    {
-                      yycount = 1;
-                      break;
-                    }
-                  else
-                    yyarg[yycount++] = yytname_[yyx];
-                }
-          }
+        if (yyarg)
+          yyarg[0] = yyctx.token ();
+        int yyn = yyctx.expected_tokens (yyarg ? yyarg + 1 : yyarg, yyargn - 1);
+        return yyn + 1;
       }
+    return 0;
+  }
+
+  // Generate an error message.
+  std::string
+  ]b4_parser_class[::yysyntax_error_ (const context& yyctx) const
+  {
+    // Its maximum.
+    enum { YYARGS_MAX = 5 };
+    // Arguments of yyformat.
+    symbol_kind_type yyarg[YYARGS_MAX];
+    int yycount = yy_syntax_error_arguments_ (yyctx, yyarg, YYARGS_MAX);
 
     char const* yyformat = YY_NULLPTR;
     switch (yycount)
@@ -1398,14 +1555,13 @@ b4_error_verbose_if([state_type yystate, const symbol_type& yyla],
     for (char const* yyp = yyformat; *yyp; ++yyp)
       if (yyp[0] == '%' && yyp[1] == 's' && yyi < yycount)
         {
-          yyres += yytnamerr_ (yyarg[yyi++]);
+          yyres += symbol_name (yyarg[yyi++]);
           ++yyp;
         }
       else
         yyres += *yyp;
-    return yyres;]], [[
-    return YY_("syntax error");]])[
-  }
+    return yyres;
+  }]])[
 
 
   const ]b4_int_type(b4_pact_ninf, b4_pact_ninf) b4_parser_class::yypact_ninf_ = b4_pact_ninf[;
@@ -1414,21 +1570,23 @@ b4_error_verbose_if([state_type yystate, const symbol_type& yyla],
 
 ]b4_parser_tables_define[
 
-]b4_token_table_if([], [[#if ]b4_api_PREFIX[DEBUG]])[
+]b4_parse_error_bmatch([simple\|verbose],
+[[#if ]b4_api_PREFIX[DEBUG]b4_tname_if([[ || 1]])[
   // YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
-  // First, the terminals, then, starting at \a yyntokens_, nonterminals.
+  // First, the terminals, then, starting at \a YYNTOKENS, nonterminals.
   const char*
   const ]b4_parser_class[::yytname_[] =
   {
   ]b4_tname[
   };
+#endif
+]])[
 
-]b4_token_table_if([[#if ]b4_api_PREFIX[DEBUG]])[
+#if ]b4_api_PREFIX[DEBUG][
 ]b4_integral_parser_table_define([rline], [b4_rline])[
 
-  // Print the state stack on the debug stream.
   void
-  ]b4_parser_class[::yystack_print_ ()
+  ]b4_parser_class[::yy_stack_print_ () const
   {
     *yycdebug_ << "Stack now";
     for (stack_type::const_iterator
@@ -1439,9 +1597,8 @@ b4_error_verbose_if([state_type yystate, const symbol_type& yyla],
     *yycdebug_ << '\n';
   }
 
-  // Report on the debug stream that the rule \a yyrule is going to be reduced.
   void
-  ]b4_parser_class[::yy_reduce_print_ (int yyrule)
+  ]b4_parser_class[::yy_reduce_print_ (int yyrule) const
   {
     int yylno = yyrline_[yyrule];
     int yynrhs = yyr2_[yyrule];

data/skeletons/lalr1.d

@@ -21,7 +21,7 @@ m4_include(b4_skeletonsdir/[d.m4])
 b4_output_begin([b4_parser_file_name])
 b4_copyright([Skeleton implementation for Bison LALR(1) parsers in D],
              [2007-2012, 2019-2020])[
-
+]b4_disclaimer[
 ]b4_percent_define_ifdef([package], [module b4_percent_define_get([package]);
 ])[
 version(D_Version2) {
@@ -182,11 +182,13 @@ b4_user_union_members
 };],
 [m4_if(b4_tag_seen_flag, 0,
 [[private alias int YYSemanticType;]])])[
-]b4_token_enums(b4_tokens)[
+]b4_token_enums[
 ]b4_parser_class_declaration[
 {
   ]b4_identification[
 
+]b4_declare_symbol_enum[
+
 ]b4_locations_if([[
   private final ]b4_location_type[ yylloc_from_stack (ref YYStack rhs, int n)
   {
@@ -261,6 +263,11 @@ b4_user_union_members
   public final void setDebugLevel(int level) { yydebug = level; }
 
   protected final void yycdebug (string s) {
+    if (0 < yydebug)
+      yyDebugStream.write (s);
+  }
+
+  protected final void yycdebugln (string s) {
     if (0 < yydebug)
       yyDebugStream.writeln (s);
   }
@@ -335,7 +342,8 @@ b4_user_union_members
     }
 
 ]b4_parse_trace_if([[
-    yy_symbol_print ("-> $$ =", yyr1_[yyn], yyval]b4_locations_if([, yyloc])[);]])[
+    import std.conv : to;
+    yy_symbol_print ("-> $$ =", to!SymbolKind (yyr1_[yyn]), yyval]b4_locations_if([, yyloc])[);]])[
 
     yystack.pop (yylen);
     yylen = 0;
@@ -393,21 +401,21 @@ b4_user_union_members
   | Print this symbol on YYOUTPUT.  |
   `--------------------------------*/
 
-  private final void yy_symbol_print (string s, int yytype,
+  private final void yy_symbol_print (string s, SymbolKind yykind,
     ref ]b4_yystype[ yyvaluep]dnl
 b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
   {
     if (0 < yydebug)
     {
-      string message = s ~ (yytype < yyntokens_ ? " token " : " nterm ")
-              ~ yytname_[yytype] ~ " ("]b4_locations_if([
+      string message = s ~ (yykind < yyntokens_ ? " token " : " nterm ")
+              ~ yytname_[yykind] ~ " ("]b4_locations_if([
               ~ yylocationp.toString() ~ ": "])[;
       static if (__traits(compiles, message ~= yyvaluep.toString ()))
               message ~= yyvaluep.toString ();
       else
               message ~= format ("%s", &yyvaluep);
       message ~= ")";
-      yycdebug (message);
+      yycdebugln (message);
     }
   }
 ]])[
@@ -420,9 +428,10 @@ b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
    */
   public bool parse ()
   {
-    /// Lookahead and lookahead in internal form.
-    int yychar = yyempty_;
-    int yytoken = 0;
+    // Lookahead token kind.
+    int yychar = TokenKind.YYEMPTY;
+    // Lookahead symbol kind.
+    SymbolKind yytoken = ]b4_symbol(-2, kind)[;
 
     /* State.  */
     int yyn = 0;
@@ -445,9 +454,9 @@ b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
     /// Semantic value of the lookahead.
     ]b4_yystype[ yylval;
 
-    int yyresult;]b4_parse_trace_if([[
+    bool yyresult;]b4_parse_trace_if([[
 
-    yycdebug ("Starting parse\n");]])[
+    yycdebugln ("Starting parse");]])[
     yyerrstatus_ = 0;
 
 ]m4_ifdef([b4_initial_action], [
@@ -468,7 +477,7 @@ m4_popdef([b4_at_dollar])])dnl
         /* New state.  Unlike in the C/C++ skeletons, the state is already
            pushed when we come here.  */
       case YYNEWSTATE:]b4_parse_trace_if([[
-        yycdebug (format("Entering state %d\n", yystate));
+        yycdebugln (format("Entering state %d", yystate));
         if (0 < yydebug)
           yystack.print (yyDebugStream);]])[
 
@@ -485,9 +494,9 @@ m4_popdef([b4_at_dollar])])dnl
         }
 
         /* Read a lookahead token.  */
-        if (yychar == yyempty_)
+        if (yychar == TokenKind.YYEMPTY)
         {]b4_parse_trace_if([[
-          yycdebug ("Reading a token: ");]])[
+          yycdebugln ("Reading a token");]])[
           yychar = yylex ();]b4_locations_if([[
           static if (yy_location_is_class) {
             yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos);
@@ -499,43 +508,55 @@ m4_popdef([b4_at_dollar])])dnl
 
         /* Convert token to internal form.  */
         yytoken = yytranslate_ (yychar);]b4_parse_trace_if([[
-        yy_symbol_print ("Next token is",
-                         yytoken, yylval]b4_locations_if([, yylloc])[);]])[
+        yy_symbol_print ("Next token is", yytoken, yylval]b4_locations_if([, yylloc])[);]])[
 
-        /* If the proper action on seeing token YYTOKEN is to reduce or to
-           detect an error, take that action.  */
-        yyn += yytoken;
-        if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken)
-          label = YYDEFAULT;
-
-        /* <= 0 means reduce or error.  */
-        else if ((yyn = yytable_[yyn]) <= 0)
+        if (yytoken == ]b4_symbol(1, kind)[)
         {
-          if (yy_table_value_is_error_ (yyn))
-            label = YYERRLAB;
-          else
-          {
-            yyn = -yyn;
-            label = YYREDUCE;
-          }
+          // The scanner already issued an error message, process directly
+          // to error recovery.  But do not keep the error token as
+          // lookahead, it is too special and may lead us to an endless
+          // loop in error recovery. */
+          yychar = TokenKind.]b4_symbol(2, id)[;
+          yytoken = ]b4_symbol(2, kind)[;]b4_locations_if([[
+          yyerrloc = yylloc;]])[
+          label = YYERRLAB1;
         }
         else
         {
-          /* Shift the lookahead token.  */]b4_parse_trace_if([[
-          yy_symbol_print ("Shifting", yytoken,
-                            yylval]b4_locations_if([, yylloc])[);]])[
+          /* If the proper action on seeing token YYTOKEN is to reduce or to
+             detect an error, take that action.  */
+          yyn += yytoken;
+          if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken)
+            label = YYDEFAULT;
 
-          /* Discard the token being shifted.  */
-          yychar = yyempty_;
+          /* <= 0 means reduce or error.  */
+          else if ((yyn = yytable_[yyn]) <= 0)
+          {
+            if (yy_table_value_is_error_ (yyn))
+              label = YYERRLAB;
+            else
+            {
+              yyn = -yyn;
+              label = YYREDUCE;
+            }
+          }
+          else
+          {
+            /* Shift the lookahead token.  */]b4_parse_trace_if([[
+            yy_symbol_print ("Shifting", yytoken, yylval]b4_locations_if([, yylloc])[);]])[
 
-          /* Count tokens shifted since error; after three, turn off error
-           * status.  */
-          if (yyerrstatus_ > 0)
-            --yyerrstatus_;
+            /* Discard the token being shifted.  */
+            yychar = TokenKind.YYEMPTY;
 
-          yystate = yyn;
-          yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
-          label = YYNEWSTATE;
+            /* Count tokens shifted since error; after three, turn off error
+             * status.  */
+            if (yyerrstatus_ > 0)
+              --yyerrstatus_;
+
+            yystate = yyn;
+            yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
+            label = YYNEWSTATE;
+          }
         }
         break;
 
@@ -559,33 +580,33 @@ m4_popdef([b4_at_dollar])])dnl
         yystate = yystack.stateAt (0);
         break;
 
-      /*------------------------------------.
-      | yyerrlab -- here on detecting error |
-      `------------------------------------*/
+      /*--------------------------------------.
+      | yyerrlab -- here on detecting error.  |
+      `--------------------------------------*/
       case YYERRLAB:
         /* If not already recovering from an error, report this error.  */
         if (yyerrstatus_ == 0)
         {
           ++yynerrs_;
-          if (yychar == yyempty_)
-            yytoken = yyempty_;
+          if (yychar == TokenKind.]b4_symbol(-2, id)[)
+            yytoken = ]b4_symbol(-2, kind)[;
           yyerror (]b4_locations_if([yylloc, ])[yysyntax_error (yystate, yytoken));
         }
-
-]b4_locations_if([        yyerrloc = yylloc;])[
+]b4_locations_if([
+        yyerrloc = yylloc;])[
         if (yyerrstatus_ == 3)
         {
           /* If just tried and failed to reuse lookahead token after an
            * error, discard it.  */
 
-          if (yychar <= YYTokenType.EOF)
+          if (yychar <= TokenKind.]b4_symbol(0, [id])[)
           {
             /* Return failure if at end of input.  */
-            if (yychar == YYTokenType.EOF)
+            if (yychar == TokenKind.]b4_symbol(0, [id])[)
              return false;
           }
           else
-            yychar = yyempty_;
+            yychar = TokenKind.YYEMPTY;
         }
 
         /* Else will try to reuse lookahead token after shifting the error
@@ -612,13 +633,14 @@ m4_popdef([b4_at_dollar])])dnl
       case YYERRLAB1:
         yyerrstatus_ = 3;       /* Each real token shifted decrements this.  */
 
+        // Pop stack until we find a state that shifts the error token.
         for (;;)
         {
           yyn = yypact_[yystate];
           if (!yy_pact_value_is_default_ (yyn))
           {
-            yyn += yy_error_token_;
-            if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == yy_error_token_)
+            yyn += ]b4_symbol(1, kind)[;
+            if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == ]b4_symbol(1, kind)[)
             {
               yyn = yytable_[yyn];
               if (0 < yyn)
@@ -645,9 +667,8 @@ m4_popdef([b4_at_dollar])])dnl
         yystack.pop (2);])[
 
         /* Shift the error token.  */]b4_parse_trace_if([[
-        yy_symbol_print ("Shifting", yystos_[yyn],
-        yylval]b4_locations_if([, yyloc])[);]])[
-
+        import std.conv : to;
+        yy_symbol_print ("Shifting", to!SymbolKind (yystos_[yyn]), yylval]b4_locations_if([, yyloc])[);]])[
         yystate = yyn;
         yystack.push (yyn, yylval]b4_locations_if([, yyloc])[);
         label = YYNEWSTATE;
@@ -655,17 +676,26 @@ m4_popdef([b4_at_dollar])])dnl
 
       /* Accept.  */
       case YYACCEPT:
-        return true;
+        yyresult = true;
+        label = YYRETURN;
+        break;
 
       /* Abort.  */
       case YYABORT:
-        return false;
+        yyresult = false;
+        label = YYRETURN;
+        break;
+
+      case YYRETURN:]b4_parse_trace_if([[
+        if (0 < yydebug)
+          yystack.print (yyDebugStream);]])[
+        return yyresult;
     }
   }
 
   // Generate an error message.
-  private final string yysyntax_error (int yystate, int tok)
-  {]b4_error_verbose_if([[
+  private final string yysyntax_error (int yystate, SymbolKind tok)
+  {]b4_parse_error_case([verbose], [[
     /* There are many possibilities here to consider:
        - Assume YYFAIL is not used.  It's too flawed to consider.
          See
@@ -697,7 +727,7 @@ m4_popdef([b4_at_dollar])])dnl
          will still contain any token that will not be accepted due
          to an error action in a later state.
       */
-    if (tok != yyempty_)
+    if (tok != ]b4_symbol(-2, kind)[)
     {
       // FIXME: This method of building the message is not compatible
       // with internationalization.
@@ -716,14 +746,14 @@ m4_popdef([b4_at_dollar])])dnl
         int yyxend = yychecklim < yyntokens_ ? yychecklim : yyntokens_;
         int count = 0;
         for (int x = yyxbegin; x < yyxend; ++x)
-          if (yycheck_[x + yyn] == x && x != yy_error_token_
+          if (yycheck_[x + yyn] == x && x != ]b4_symbol(1, kind)[
               && !yy_table_value_is_error_ (yytable_[x + yyn]))
              ++count;
           if (count < 5)
           {
              count = 0;
              for (int x = yyxbegin; x < yyxend; ++x)
-               if (yycheck_[x + yyn] == x && x != yy_error_token_
+               if (yycheck_[x + yyn] == x && x != ]b4_symbol(1, kind)[
                    && !yy_table_value_is_error_ (yytable_[x + yyn]))
                {
                   res ~= count++ == 0 ? ", expecting " : " or ";
@@ -788,47 +818,46 @@ m4_popdef([b4_at_dollar])])dnl
     int yylno = yyrline_[yyrule];
     int yynrhs = yyr2_[yyrule];
     /* Print the symbols being reduced, and their result.  */
-    yycdebug (format("Reducing stack by rule %d (line %d), ",
-              yyrule - 1, yylno));
+    yycdebugln (format("Reducing stack by rule %d (line %d):",
+                yyrule - 1, yylno));
 
     /* The symbols being reduced.  */
+    import std.conv : to;
     for (int yyi = 0; yyi < yynrhs; yyi++)
       yy_symbol_print (format("   $%d =", yyi + 1),
-                       yystos_[yystack.stateAt(yynrhs - (yyi + 1))],
+                       to!SymbolKind (yystos_[yystack.stateAt(yynrhs - (yyi + 1))]),
                        ]b4_rhs_value(yynrhs, yyi + 1)b4_locations_if([,
                        b4_rhs_location(yynrhs, yyi + 1)])[);
   }
 ]])[
 
-  private static token_number_type yytranslate_ (int t)
+  private static SymbolKind yytranslate_ (int t)
   {
 ]b4_api_token_raw_if(
 [[    import std.conv : to;
-    return to!byte (t);]],
+    return to!SymbolKind (t);]],
 [[    /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
-    immutable token_number_type[] translate_table =
+    immutable ]b4_int_type_for([b4_translate])[[] translate_table =
     @{
   ]b4_translate[
     @};
 
-    immutable int user_token_number_max_ = ]b4_user_token_number_max[;
-    immutable token_number_type undef_token_ = ]b4_undef_token_number[;
+    // Last valid token kind.
+    immutable int code_max = ]b4_code_max[;
 
     if (t <= 0)
-      return YYTokenType.EOF;
-    else if (t <= user_token_number_max_)
-      return translate_table[t];
+      return ]b4_symbol(0, kind)[;
+    else if (t <= code_max)
+      {
+        import std.conv : to;
+        return to!SymbolKind (translate_table[t]);
+      }
     else
-      return undef_token_;]])[
+      return ]b4_symbol(2, kind)[;]])[
   }
 
-  alias ]b4_int_type_for([b4_translate])[ token_number_type;
-
-  private static immutable token_number_type yy_error_token_ = 1;
-
   private static immutable int yylast_ = ]b4_last[;
   private static immutable int yynnts_ = ]b4_nterms_number[;
-  private static immutable int yyempty_ = -2;
   private static immutable int yyfinal_ = ]b4_final_state_number[;
   private static immutable int yyntokens_ = ]b4_tokens_number[;
 
@@ -887,9 +916,8 @@ m4_popdef([b4_at_dollar])])dnl
       stream.writeln ();
     }]])[
   }
-
-  /* User implementation code.  */
 ]b4_percent_code_get[
 }
-]b4_epilogue[]dnl
+]b4_percent_code_get([[epilogue]])[]dnl
+b4_epilogue[]dnl
 b4_output_end

data/skeletons/lalr1.java

@@ -49,13 +49,14 @@ b4_use_push_for_pull_if([
   b4_push_if([m4_define([b4_use_push_for_pull_flag], [[0]])],
              [m4_define([b4_push_flag], [[1]])])])
 
-# Define a macro to encapsulate the parse state variables.
-# This allows them to be defined either in parse() when doing
-# pull parsing, or as class instance variable when doing push parsing.
+# Define a macro to encapsulate the parse state variables.  This
+# allows them to be defined either in parse() when doing pull parsing,
+# or as class instance variable when doing push parsing.
 m4_define([b4_define_state],[[
-    /* Lookahead and lookahead in internal form.  */
-    int yychar = yyempty_;
-    int yytoken = 0;
+    /* Lookahead token kind.  */
+    int yychar = YYEMPTY_;
+    /* Lookahead symbol kind.  */
+    SymbolKind yytoken = null;
 
     /* State.  */
     int yyn = 0;
@@ -64,8 +65,7 @@ m4_define([b4_define_state],[[
     YYStack yystack = new YYStack ();
     int label = YYNEWSTATE;
 
-    /* Error handling.  */
-    int yynerrs_ = 0;]b4_locations_if([[
+]b4_locations_if([[
     /* The location where the error started.  */
     ]b4_location_type[ yyerrloc = null;
 
@@ -79,10 +79,12 @@ m4_define([b4_define_state],[[
 ]b4_output_begin([b4_parser_file_name])[
 ]b4_copyright([Skeleton implementation for Bison LALR(1) parsers in Java],
               [2007-2015, 2018-2020])[
-]b4_percent_define_ifdef([package], [package b4_percent_define_get([package]);[
+]b4_disclaimer[
+]b4_percent_define_ifdef([api.package], [package b4_percent_define_get([api.package]);[
 ]])[
 ]b4_user_pre_prologue[
 ]b4_user_post_prologue[
+import java.text.MessageFormat;
 ]b4_percent_code_get([[imports]])[
 /**
  * A Bison parser, automatically generated from <tt>]m4_bpatsubst(b4_file_name, [^"\(.*\)"$], [\1])[</tt>.
@@ -91,8 +93,10 @@ m4_define([b4_define_state],[[
  */
 ]b4_parser_class_declaration[
 {
-  ]b4_identification[
-]b4_error_verbose_if([[
+]b4_identification[
+][
+]b4_parse_error_bmatch(
+           [detailed\|verbose], [[
   /**
    * True if verbose error messages are enabled.
    */
@@ -109,16 +113,16 @@ m4_define([b4_define_state],[[
    */
   public final void setErrorVerbose(boolean verbose)
   { yyErrorVerbose = verbose; }
-]])
+]])[
 
-b4_locations_if([[
+]b4_locations_if([[
   /**
    * A class defining a pair of positions.  Positions, defined by the
    * <code>]b4_position_type[</code> class, denote a point in the input.
    * Locations represent a part of the input through the beginning
    * and ending positions.
    */
-  public class ]b4_location_type[ {
+  public static class ]b4_location_type[ {
     /**
      * The first, inclusive, position in the range.
      */
@@ -161,44 +165,42 @@ b4_locations_if([[
     }
   }
 
-]])[
-
-]b4_locations_if([[
-  private ]b4_location_type[ yylloc (YYStack rhs, int n)
+  private ]b4_location_type[ yylloc(YYStack rhs, int n)
   {
     if (0 < n)
-      return new ]b4_location_type[ (rhs.locationAt (n-1).begin, rhs.locationAt (0).end);
+      return new ]b4_location_type[(rhs.locationAt(n-1).begin, rhs.locationAt(0).end);
     else
-      return new ]b4_location_type[ (rhs.locationAt (0).end);
+      return new ]b4_location_type[(rhs.locationAt(0).end);
   }]])[
 
+]b4_declare_symbol_enum[
+
   /**
    * Communication interface between the scanner and the Bison-generated
    * parser <tt>]b4_parser_class[</tt>.
    */
   public interface Lexer {
-    /** Token returned by the scanner to signal the end of its input.  */
-    public static final int EOF = 0;
-
 ]b4_token_enums[
-
-]b4_locations_if([[/**
+    /** Deprecated, use ]b4_symbol(0, id)[ instead.  */
+    public static final int EOF = ]b4_symbol(0, id)[;
+]b4_pull_if([b4_locations_if([[
+    /**
      * Method to retrieve the beginning position of the last scanned token.
      * @@return the position at which the last scanned token starts.
      */
-    ]b4_position_type[ getStartPos ();
+    ]b4_position_type[ getStartPos();
 
     /**
      * Method to retrieve the ending position of the last scanned token.
      * @@return the first position beyond the last scanned token.
      */
-    ]b4_position_type[ getEndPos ();]])[
+    ]b4_position_type[ getEndPos();]])[
 
     /**
      * Method to retrieve the semantic value of the last scanned token.
      * @@return the semantic value of the last scanned token.
      */
-    ]b4_yystype[ getLVal ();
+    ]b4_yystype[ getLVal();
 
     /**
      * Entry point for the scanner.  Returns the token identifier corresponding
@@ -206,17 +208,26 @@ b4_locations_if([[
      * ]b4_locations_if([and beginning/ending positions ])[of the token.
      * @@return the token identifier corresponding to the next token.
      */
-    int yylex ()]b4_maybe_throws([b4_lex_throws])[;
-
+    int yylex()]b4_maybe_throws([b4_lex_throws])[;
+]])[
     /**
-     * Entry point for error reporting.  Emits an error
-     * ]b4_locations_if([referring to the given location ])[in a user-defined way.
+     * Emit an error]b4_locations_if([ referring to the given location])[in a user-defined way.
      *
      *]b4_locations_if([[ @@param loc The location of the element to which the
-     *                error message is related]])[
+     *                error message is related.]])[
      * @@param msg The string for the error message.
      */
-     void yyerror (]b4_locations_if([b4_location_type[ loc, ]])[String msg);
+     void yyerror(]b4_locations_if([b4_location_type[ loc, ]])[String msg);
+
+]b4_parse_error_bmatch(
+           [custom], [[
+    /**
+     * Build and emit a "syntax error" message in a user-defined way.
+     *
+     * @@param ctx  The context of the error.
+     */
+     void reportSyntaxError (][Context ctx);
+]])[
   }
 
 ]b4_lexer_if([[
@@ -239,7 +250,7 @@ b4_locations_if([[
   public ]b4_parser_class (b4_parse_param_decl([b4_lex_param_decl])[)]b4_maybe_throws([b4_init_throws])[
   {
 ]b4_percent_code_get([[init]])[
-    this.yylexer = new YYLexer(]b4_lex_param_call[);
+    this.yylexer = new YYLexer (]b4_lex_param_call[);
 ]b4_parse_param_cons[
   }
 ]])[
@@ -267,7 +278,7 @@ b4_locations_if([[
    * Set the <tt>PrintStream</tt> on which the debug output is printed.
    * @@param s The stream that is used for debugging output.
    */
-  public final void setDebugStream(java.io.PrintStream s) { yyDebugStream = s; }
+  public final void setDebugStream (java.io.PrintStream s) { yyDebugStream = s; }
 
   private int yydebug = 0;
 
@@ -275,24 +286,30 @@ b4_locations_if([[
    * Answer the verbosity of the debugging output; 0 means that all kinds of
    * output from the parser are suppressed.
    */
-  public final int getDebugLevel() { return yydebug; }
+  public final int getDebugLevel () { return yydebug; }
 
   /**
    * Set the verbosity of the debugging output; 0 means that all kinds of
    * output from the parser are suppressed.
    * @@param level The verbosity level for debugging output.
    */
-  public final void setDebugLevel(int level) { yydebug = level; }
+  public final void setDebugLevel (int level) { yydebug = level; }
 ]])[
 
+  private int yynerrs = 0;
+
+  /**
+   * The number of syntax errors so far.
+   */
+  public final int getNumberOfErrors () { return yynerrs; }
+
   /**
    * Print an error message via the lexer.
    *]b4_locations_if([[ Use a <code>null</code> location.]])[
    * @@param msg The error message.
    */
-  public final void yyerror (String msg)
-  {
-    yylexer.yyerror (]b4_locations_if([[(]b4_location_type[)null, ]])[msg);
+  public final void yyerror(String msg) {
+      yylexer.yyerror(]b4_locations_if([[(]b4_location_type[)null, ]])[msg);
   }
 ]b4_locations_if([[
   /**
@@ -300,9 +317,8 @@ b4_locations_if([[
    * @@param loc The location associated with the message.
    * @@param msg The error message.
    */
-  public final void yyerror (]b4_location_type[ loc, String msg)
-  {
-    yylexer.yyerror (loc, msg);
+  public final void yyerror(]b4_location_type[ loc, String msg) {
+      yylexer.yyerror(loc, msg);
   }
 
   /**
@@ -310,9 +326,8 @@ b4_locations_if([[
    * @@param pos The position associated with the message.
    * @@param msg The error message.
    */
-  public final void yyerror (]b4_position_type[ pos, String msg)
-  {
-    yylexer.yyerror (new ]b4_location_type[ (pos), msg);
+  public final void yyerror(]b4_position_type[ pos, String msg) {
+      yylexer.yyerror(new ]b4_location_type[ (pos), msg);
   }]])[
 ]b4_parse_trace_if([[
   protected final void yycdebug (String s) {
@@ -328,8 +343,7 @@ b4_locations_if([[
     public int size = 16;
     public int height = -1;
 
-    public final void push (int state, ]b4_yystype[ value]dnl
-                            b4_locations_if([, ]b4_location_type[ loc])[) {
+    public final void push (int state, ]b4_yystype[ value]b4_locations_if([, ]b4_location_type[ loc])[) {
       height++;
       if (size == height)
         {
@@ -373,8 +387,8 @@ b4_locations_if([[
     public final ]b4_location_type[ locationAt (int i) {
       return locStack[height - i];
     }
-
-    ]])[public final ]b4_yystype[ valueAt (int i) {
+]])[
+    public final ]b4_yystype[ valueAt (int i) {
       return valueStack[height - i];
     }
 
@@ -446,14 +460,14 @@ b4_locations_if([[
    */
   private int yyLRGotoState (int yystate, int yysym)
   {
-    int yyr = yypgoto_[yysym - yyntokens_] + yystate;
-    if (0 <= yyr && yyr <= yylast_ && yycheck_[yyr] == yystate)
+    int yyr = yypgoto_[yysym - YYNTOKENS_] + yystate;
+    if (0 <= yyr && yyr <= YYLAST_ && yycheck_[yyr] == yystate)
       return yytable_[yyr];
     else
-      return yydefgoto_[yysym - yyntokens_];
+      return yydefgoto_[yysym - YYNTOKENS_];
   }
 
-  private int yyaction (int yyn, YYStack yystack, int yylen)]b4_maybe_throws([b4_throws])[
+  private int yyaction(int yyn, YYStack yystack, int yylen)]b4_maybe_throws([b4_throws])[
   {
     /* If YYLEN is nonzero, implement the default value of the action:
        '$$ = $1'.  Otherwise, use the top of the stack.
@@ -461,10 +475,10 @@ b4_locations_if([[
        Otherwise, the following line sets YYVAL to garbage.
        This behavior is undocumented and Bison
        users should not rely upon it.  */
-    ]b4_yystype[ yyval = (0 < yylen) ? yystack.valueAt (yylen - 1) : yystack.valueAt (0);]b4_locations_if([[
-    ]b4_location_type[ yyloc = yylloc (yystack, yylen);]])[]b4_parse_trace_if([[
+    ]b4_yystype[ yyval = (0 < yylen) ? yystack.valueAt(yylen - 1) : yystack.valueAt(0);]b4_locations_if([[
+    ]b4_location_type[ yyloc = yylloc(yystack, yylen);]])[]b4_parse_trace_if([[
 
-    yyReducePrint (yyn, yystack);]])[
+    yyReducePrint(yyn, yystack);]])[
 
     switch (yyn)
       {
@@ -472,66 +486,30 @@ b4_locations_if([[
         default: break;
       }]b4_parse_trace_if([[
 
-    yySymbolPrint ("-> $$ =", yyr1_[yyn], yyval]b4_locations_if([, yyloc])[);]])[
+    yySymbolPrint("-> $$ =", SymbolKind.get(yyr1_[yyn]), yyval]b4_locations_if([, yyloc])[);]])[
 
-    yystack.pop (yylen);
+    yystack.pop(yylen);
     yylen = 0;
-
     /* Shift the result of the reduction.  */
-    int yystate = yyLRGotoState (yystack.stateAt (0), yyr1_[yyn]);
-    yystack.push (yystate, yyval]b4_locations_if([, yyloc])[);
+    int yystate = yyLRGotoState(yystack.stateAt(0), yyr1_[yyn]);
+    yystack.push(yystate, yyval]b4_locations_if([, yyloc])[);
     return YYNEWSTATE;
   }
 
-]b4_error_verbose_if([[
-  /* Return YYSTR after stripping away unnecessary quotes and
-     backslashes, so that it's suitable for yyerror.  The heuristic is
-     that double-quoting is unnecessary unless the string contains an
-     apostrophe, a comma, or backslash (other than backslash-backslash).
-     YYSTR is taken from yytname.  */
-  private final String yytnamerr_ (String yystr)
-  {
-    if (yystr.charAt (0) == '"')
-      {
-        StringBuffer yyr = new StringBuffer ();
-        strip_quotes: for (int i = 1; i < yystr.length (); i++)
-          switch (yystr.charAt (i))
-            {
-            case '\'':
-            case ',':
-              break strip_quotes;
-
-            case '\\':
-              if (yystr.charAt(++i) != '\\')
-                break strip_quotes;
-              /* Fall through.  */
-            default:
-              yyr.append (yystr.charAt (i));
-              break;
-
-            case '"':
-              return yyr.toString ();
-            }
-      }
-    else if (yystr.equals ("$end"))
-      return "end of input";
-
-    return yystr;
-  }
-]])[
 ]b4_parse_trace_if([[
   /*--------------------------------.
   | Print this symbol on YYOUTPUT.  |
   `--------------------------------*/
 
-  private void yySymbolPrint (String s, int yytype,
-                             ]b4_yystype[ yyvaluep]dnl
-                              b4_locations_if([, Object yylocationp])[)
-  {
-    yycdebug (s + (yytype < yyntokens_ ? " token " : " nterm ")
-              + yytname_[yytype] + " ("]b4_locations_if([
-              + yylocationp + ": "])[
-              + (yyvaluep == null ? "(null)" : yyvaluep.toString ()) + ")");
+  private void yySymbolPrint(String s, SymbolKind yykind,
+                             ]b4_yystype[ yyvalue]b4_locations_if([, ]b4_location_type[ yylocation])[) {
+      if (0 < yydebug) {
+          yycdebug(s
+                   + (yykind.getCode() < YYNTOKENS_ ? " token " : " nterm ")
+                   + yykind.getName() + " ("]b4_locations_if([
+                   + yylocation + ": "])[
+                   + (yyvalue == null ? "(null)" : yyvalue.toString()) + ")");
+      }
   }]])[
 
 ]b4_push_if([],[[
@@ -542,7 +520,7 @@ b4_locations_if([[
    * @@return <tt>true</tt> if the parsing succeeds.  Note that this does not
    *          imply that there were no syntax errors.
    */
-  public boolean parse ()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[
+  public boolean parse()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[
 ]b4_push_if([
   /**
    * Push Parse input from external lexer
@@ -553,14 +531,15 @@ b4_locations_if([[
    *
    * @@return <tt>YYACCEPT, YYABORT, YYPUSH_MORE</tt>
    */
-  public int push_parse (int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc]))b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[
+  public int push_parse(int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc]))b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[
   {]b4_locations_if([[
     /* @@$.  */
     ]b4_location_type[ yyloc;]])[
 ]b4_push_if([],[[
 ]b4_define_state[]b4_parse_trace_if([[
-    yycdebug ("Starting parse\n");]])[
+    yycdebug ("Starting parse");]])[
     yyerrstatus_ = 0;
+    yynerrs = 0;
 
     /* Initialize the stack.  */
     yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
@@ -579,7 +558,7 @@ b4_dollar_pushdef([yylval], [], [], [yylloc])dnl
     b4_user_initial_action
 b4_dollar_popdef[]dnl
 ])[]b4_parse_trace_if([[
-        yycdebug ("Starting parse\n");]])[
+        yycdebug ("Starting parse");]])[
         yyerrstatus_ = 0;
       } else
         label = YYGETTOKEN;
@@ -592,12 +571,12 @@ b4_dollar_popdef[]dnl
         /* New state.  Unlike in the C/C++ skeletons, the state is already
            pushed when we come here.  */
       case YYNEWSTATE:]b4_parse_trace_if([[
-        yycdebug ("Entering state " + yystate + "\n");
+        yycdebug ("Entering state " + yystate);
         if (0 < yydebug)
           yystack.print (yyDebugStream);]])[
 
         /* Accept?  */
-        if (yystate == yyfinal_)
+        if (yystate == YYFINAL_)
           ]b4_push_if([{label = YYACCEPT; break;}],
                       [return true;])[
 
@@ -612,64 +591,78 @@ b4_dollar_popdef[]dnl
 
       case YYGETTOKEN:])[
         /* Read a lookahead token.  */
-        if (yychar == yyempty_)
+        if (yychar == YYEMPTY_)
           {
 ]b4_push_if([[
             if (!push_token_consumed)
               return YYPUSH_MORE;]b4_parse_trace_if([[
-            yycdebug ("Reading a token: ");]])[
+            yycdebug ("Reading a token");]])[
             yychar = yylextoken;
             yylval = yylexval;]b4_locations_if([
             yylloc = yylexloc;])[
             push_token_consumed = false;]], [b4_parse_trace_if([[
-            yycdebug ("Reading a token: ");]])[
+            yycdebug ("Reading a token");]])[
             yychar = yylexer.yylex ();
-            yylval = yylexer.getLVal ();]b4_locations_if([
-            yylloc = new b4_location_type (yylexer.getStartPos (),
-                            yylexer.getEndPos ());])[
+            yylval = yylexer.getLVal();]b4_locations_if([[
+            yylloc = new ]b4_location_type[(yylexer.getStartPos(),
+                                          yylexer.getEndPos());]])[
 ]])[
           }
 
         /* Convert token to internal form.  */
         yytoken = yytranslate_ (yychar);]b4_parse_trace_if([[
-        yySymbolPrint ("Next token is", yytoken,
-                       yylval]b4_locations_if([, yylloc])[);]])[
+        yySymbolPrint("Next token is", yytoken,
+                      yylval]b4_locations_if([, yylloc])[);]])[
 
-        /* If the proper action on seeing token YYTOKEN is to reduce or to
-           detect an error, take that action.  */
-        yyn += yytoken;
-        if (yyn < 0 || yylast_ < yyn || yycheck_[yyn] != yytoken)
-          label = YYDEFAULT;
-
-        /* <= 0 means reduce or error.  */
-        else if ((yyn = yytable_[yyn]) <= 0)
+        if (yytoken == ]b4_symbol(1, kind)[)
           {
-            if (yyTableValueIsError (yyn))
-              label = YYERRLAB;
-            else
-              {
-                yyn = -yyn;
-                label = YYREDUCE;
-              }
+            // The scanner already issued an error message, process directly
+            // to error recovery.  But do not keep the error token as
+            // lookahead, it is too special and may lead us to an endless
+            // loop in error recovery. */
+            yychar = Lexer.]b4_symbol(2, id)[;
+            yytoken = ]b4_symbol(2, kind)[;]b4_locations_if([[
+            yyerrloc = yylloc;]])[
+            label = YYERRLAB1;
           }
-
         else
           {
-            /* Shift the lookahead token.  */]b4_parse_trace_if([[
-            yySymbolPrint ("Shifting", yytoken,
-                           yylval]b4_locations_if([, yylloc])[);
+            /* If the proper action on seeing token YYTOKEN is to reduce or to
+               detect an error, take that action.  */
+            yyn += yytoken.getCode();
+            if (yyn < 0 || YYLAST_ < yyn || yycheck_[yyn] != yytoken.getCode())
+              label = YYDEFAULT;
+
+            /* <= 0 means reduce or error.  */
+            else if ((yyn = yytable_[yyn]) <= 0)
+              {
+                if (yyTableValueIsError (yyn))
+                  label = YYERRLAB;
+                else
+                  {
+                    yyn = -yyn;
+                    label = YYREDUCE;
+                  }
+              }
+
+            else
+              {
+                /* Shift the lookahead token.  */]b4_parse_trace_if([[
+                yySymbolPrint("Shifting", yytoken,
+                              yylval]b4_locations_if([, yylloc])[);
 ]])[
-            /* Discard the token being shifted.  */
-            yychar = yyempty_;
+                /* Discard the token being shifted.  */
+                yychar = YYEMPTY_;
 
-            /* Count tokens shifted since error; after three, turn off error
-               status.  */
-            if (yyerrstatus_ > 0)
-              --yyerrstatus_;
+                /* Count tokens shifted since error; after three, turn off error
+                   status.  */
+                if (yyerrstatus_ > 0)
+                  --yyerrstatus_;
 
-            yystate = yyn;
-            yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
-            label = YYNEWSTATE;
+                yystate = yyn;
+                yystack.push (yystate, yylval]b4_locations_if([, yylloc])[);
+                label = YYNEWSTATE;
+              }
           }
         break;
 
@@ -689,7 +682,7 @@ b4_dollar_popdef[]dnl
       `-----------------------------*/
       case YYREDUCE:
         yylen = yyr2_[yyn];
-        label = yyaction (yyn, yystack, yylen);
+        label = yyaction(yyn, yystack, yylen);
         yystate = yystack.stateAt (0);
         break;
 
@@ -700,12 +693,11 @@ b4_dollar_popdef[]dnl
         /* If not already recovering from an error, report this error.  */
         if (yyerrstatus_ == 0)
           {
-            ++yynerrs_;
-            if (yychar == yyempty_)
-              yytoken = yyempty_;
-            yyerror (]b4_locations_if([yylloc, ])[yysyntax_error (yystate, yytoken));
+            ++yynerrs;
+            if (yychar == YYEMPTY_)
+              yytoken = null;
+            yyreportSyntaxError (new Context (yystack, yytoken]b4_locations_if([[, yylloc]])[));
           }
-
 ]b4_locations_if([[
         yyerrloc = yylloc;]])[
         if (yyerrstatus_ == 3)
@@ -713,14 +705,14 @@ b4_dollar_popdef[]dnl
             /* If just tried and failed to reuse lookahead token after an
                error, discard it.  */
 
-            if (yychar <= Lexer.EOF)
+            if (yychar <= Lexer.]b4_symbol(0, id)[)
               {
                 /* Return failure if at end of input.  */
-                if (yychar == Lexer.EOF)
+                if (yychar == Lexer.]b4_symbol(0, id)[)
                   ]b4_push_if([{label = YYABORT; break;}], [return false;])[
               }
             else
-              yychar = yyempty_;
+              yychar = YYEMPTY_;
           }
 
         /* Else will try to reuse lookahead token after shifting the error
@@ -747,13 +739,15 @@ b4_dollar_popdef[]dnl
       case YYERRLAB1:
         yyerrstatus_ = 3;       /* Each real token shifted decrements this.  */
 
+        // Pop stack until we find a state that shifts the error token.
         for (;;)
           {
             yyn = yypact_[yystate];
             if (!yyPactValueIsDefault (yyn))
               {
-                yyn += yy_error_token_;
-                if (0 <= yyn && yyn <= yylast_ && yycheck_[yyn] == yy_error_token_)
+                yyn += ]b4_symbol(1, kind)[.getCode();
+                if (0 <= yyn && yyn <= YYLAST_
+                    && yycheck_[yyn] == ]b4_symbol(1, kind)[.getCode())
                   {
                     yyn = yytable_[yyn];
                     if (0 < yyn)
@@ -775,8 +769,8 @@ b4_dollar_popdef[]dnl
           }
 
         if (label == YYABORT)
-            /* Leave the switch.  */
-            break;
+          /* Leave the switch.  */
+          break;
 
 ]b4_locations_if([[
         /* Muck with the stack to setup for yylloc.  */
@@ -786,8 +780,8 @@ b4_dollar_popdef[]dnl
         yystack.pop (2);]])[
 
         /* Shift the error token.  */]b4_parse_trace_if([[
-        yySymbolPrint ("Shifting", yystos_[yyn],
-                       yylval]b4_locations_if([, yyloc])[);]])[
+        yySymbolPrint("Shifting", SymbolKind.get(yystos_[yyn]),
+                      yylval]b4_locations_if([, yyloc])[);]])[
 
         yystate = yyn;
         yystack.push (yyn, yylval]b4_locations_if([, yyloc])[);
@@ -811,11 +805,11 @@ b4_dollar_popdef[]dnl
     /**
      * (Re-)Initialize the state of the push parser.
      */
-  public void push_parse_initialize()
+  public void push_parse_initialize ()
   {
     /* Lookahead and lookahead in internal form.  */
-    this.yychar = yyempty_;
-    this.yytoken = 0;
+    this.yychar = YYEMPTY_;
+    this.yytoken = null;
 
     /* State.  */
     this.yyn = 0;
@@ -825,7 +819,7 @@ b4_dollar_popdef[]dnl
     this.label = YYNEWSTATE;
 
     /* Error handling.  */
-    this.yynerrs_ = 0;]b4_locations_if([[
+    this.yynerrs = 0;]b4_locations_if([[
     /* The location where the error started.  */
     this.yyerrloc = null;
     this.yylloc = new ]b4_location_type[ (null, null);]])[
@@ -838,7 +832,7 @@ b4_dollar_popdef[]dnl
     this.push_parse_initialized = true;
 
   }
-]b4_locations_if([
+]b4_locations_if([[
   /**
    * Push parse given input from an external lexer.
    *
@@ -848,110 +842,190 @@ b4_dollar_popdef[]dnl
    *
    * @@return <tt>YYACCEPT, YYABORT, YYPUSH_MORE</tt>
    */
-  public int push_parse (int yylextoken, b4_yystype yylexval, b4_position_type yylexpos)b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])
-  {
-    return push_parse (yylextoken, yylexval, new b4_location_type (yylexpos));
+  public int push_parse(int yylextoken, ]b4_yystype[ yylexval, ]b4_position_type[ yylexpos)]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ {
+      return push_parse(yylextoken, yylexval, new ]b4_location_type[(yylexpos));
   }
-])[]])[
+]])])[
 
 ]b4_both_if([[
   /**
    * Parse input from the scanner that was specified at object construction
    * time.  Return whether the end of the input was reached successfully.
-   * This version of parse () is defined only when api.push-push=both.
+   * This version of parse() is defined only when api.push-push=both.
    *
    * @@return <tt>true</tt> if the parsing succeeds.  Note that this does not
    *          imply that there were no syntax errors.
    */
-  public boolean parse ()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[
-  {
-    if (yylexer == null)
-      throw new NullPointerException("Null Lexer");
-    int status;
-    do {
-      int token = yylexer.yylex();
-      ]b4_yystype[ lval = yylexer.getLVal();
-]b4_locations_if([dnl
-      b4_location_type yyloc = new b4_location_type (yylexer.getStartPos (),
-                                            yylexer.getEndPos ());])[]b4_locations_if([[
-      status = push_parse(token,lval,yyloc);]], [[
-      status = push_parse(token,lval);]])[
-    } while (status == YYPUSH_MORE);
-    return (status == YYACCEPT);
+  public boolean parse()]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ {
+      if (yylexer == null)
+          throw new NullPointerException("Null Lexer");
+      int status;
+      do {
+          int token = yylexer.yylex();
+          ]b4_yystype[ lval = yylexer.getLVal();]b4_locations_if([[
+          ]b4_location_type[ yyloc = new ]b4_location_type[(yylexer.getStartPos(), yylexer.getEndPos());
+          status = push_parse(token, lval, yyloc);]], [[
+          status = push_parse(token, lval);]])[
+      } while (status == YYPUSH_MORE);
+      return status == YYACCEPT;
   }
 ]])[
 
-  // Generate an error message.
-  private String yysyntax_error (int yystate, int tok)
-  {]b4_error_verbose_if([[
-    if (yyErrorVerbose)
-      {
-        /* There are many possibilities here to consider:
-           - If this state is a consistent state with a default action,
-             then the only way this function was invoked is if the
-             default action is an error action.  In that case, don't
-             check for expected tokens because there are none.
-           - The only way there can be no lookahead present (in tok) is
-             if this state is a consistent state with a default action.
-             Thus, detecting the absence of a lookahead is sufficient to
-             determine that there is no unexpected or expected token to
-             report.  In that case, just report a simple "syntax error".
-           - Don't assume there isn't a lookahead just because this
-             state is a consistent state with a default action.  There
-             might have been a previous inconsistent state, consistent
-             state with a non-default action, or user semantic action
-             that manipulated yychar.  (However, yychar is currently out
-             of scope during semantic actions.)
-           - Of course, the expected token list depends on states to
-             have correct lookahead information, and it depends on the
-             parser not to perform extra reductions after fetching a
-             lookahead from the scanner and before detecting a syntax
-             error.  Thus, state merging (from LALR or IELR) and default
-             reductions corrupt the expected token list.  However, the
-             list is correct for canonical LR with one exception: it
-             will still contain any token that will not be accepted due
-             to an error action in a later state.
-        */
-        if (tok != yyempty_)
-          {
-            /* FIXME: This method of building the message is not compatible
-               with internationalization.  */
-            StringBuffer res =
-              new StringBuffer ("syntax error, unexpected ");
-            res.append (yytnamerr_ (yytname_[tok]));
-            int yyn = yypact_[yystate];
-            if (!yyPactValueIsDefault (yyn))
+  /**
+   * Information needed to get the list of expected tokens and to forge
+   * a syntax error diagnostic.
+   */
+  public static final class Context
+  {
+    Context (YYStack stack, SymbolKind token]b4_locations_if([[, ]b4_location_type[ loc]])[)
+    {
+      yystack = stack;
+      yytoken = token;]b4_locations_if([[
+      yylocation = loc;]])[
+    }
+
+    private YYStack yystack;
+
+
+    /**
+     * The symbol kind of the lookahead token.
+     */
+    public final SymbolKind getToken ()
+    {
+      return yytoken;
+    }
+
+    private SymbolKind yytoken;]b4_locations_if([[
+
+    /**
+     * The location of the lookahead.
+     */
+    public final ]b4_location_type[ getLocation ()
+    {
+      return yylocation;
+    }
+
+    private ]b4_location_type[ yylocation;]])[
+    static final int NTOKENS = ]b4_parser_class[.YYNTOKENS_;
+
+    /**
+     * Put in YYARG at most YYARGN of the expected tokens given the
+     * current YYCTX, and return the number of tokens stored in YYARG.  If
+     * YYARG is null, return the number of expected tokens (guaranteed to
+     * be less than YYNTOKENS).
+     */
+    int getExpectedTokens (SymbolKind yyarg[], int yyargn)
+    {
+      return getExpectedTokens (yyarg, 0, yyargn);
+    }
+
+    int getExpectedTokens (SymbolKind yyarg[], int yyoffset, int yyargn)
+    {
+      int yycount = yyoffset;
+      int yyn = yypact_[this.yystack.stateAt (0)];
+      if (!yyPactValueIsDefault (yyn))
+        {
+          /* Start YYX at -YYN if negative to avoid negative
+             indexes in YYCHECK.  In other words, skip the first
+             -YYN actions for this state because they are default
+             actions.  */
+          int yyxbegin = yyn < 0 ? -yyn : 0;
+          /* Stay within bounds of both yycheck and yytname.  */
+          int yychecklim = YYLAST_ - yyn + 1;
+          int yyxend = yychecklim < NTOKENS ? yychecklim : NTOKENS;
+          for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
+            if (yycheck_[yyx + yyn] == yyx && yyx != ]b4_symbol(1, kind)[.getCode()
+                && !yyTableValueIsError(yytable_[yyx + yyn]))
               {
-                /* Start YYX at -YYN if negative to avoid negative
-                   indexes in YYCHECK.  In other words, skip the first
-                   -YYN actions for this state because they are default
-                   actions.  */
-                int yyxbegin = yyn < 0 ? -yyn : 0;
-                /* Stay within bounds of both yycheck and yytname.  */
-                int yychecklim = yylast_ - yyn + 1;
-                int yyxend = yychecklim < yyntokens_ ? yychecklim : yyntokens_;
-                int count = 0;
-                for (int x = yyxbegin; x < yyxend; ++x)
-                  if (yycheck_[x + yyn] == x && x != yy_error_token_
-                      && !yyTableValueIsError (yytable_[x + yyn]))
-                    ++count;
-                if (count < 5)
-                  {
-                    count = 0;
-                    for (int x = yyxbegin; x < yyxend; ++x)
-                      if (yycheck_[x + yyn] == x && x != yy_error_token_
-                          && !yyTableValueIsError (yytable_[x + yyn]))
-                        {
-                          res.append (count++ == 0 ? ", expecting " : " or ");
-                          res.append (yytnamerr_ (yytname_[x]));
-                        }
-                  }
+                if (yyarg == null)
+                  yycount += 1;
+                else if (yycount == yyargn)
+                  return 0; // FIXME: this is incorrect.
+                else
+                  yyarg[yycount++] = SymbolKind.get(yyx);
               }
-            return res.toString ();
-          }
+        }
+      if (yyarg != null && yycount == yyoffset && yyoffset < yyargn)
+        yyarg[yycount] = null;
+      return yycount - yyoffset;
+    }
+  }
+
+]b4_parse_error_bmatch(
+[detailed\|verbose], [[
+  private int yysyntaxErrorArguments (Context yyctx, SymbolKind[] yyarg, int yyargn)
+  {
+    /* There are many possibilities here to consider:
+       - If this state is a consistent state with a default action,
+         then the only way this function was invoked is if the
+         default action is an error action.  In that case, don't
+         check for expected tokens because there are none.
+       - The only way there can be no lookahead present (in tok) is
+         if this state is a consistent state with a default action.
+         Thus, detecting the absence of a lookahead is sufficient to
+         determine that there is no unexpected or expected token to
+         report.  In that case, just report a simple "syntax error".
+       - Don't assume there isn't a lookahead just because this
+         state is a consistent state with a default action.  There
+         might have been a previous inconsistent state, consistent
+         state with a non-default action, or user semantic action
+         that manipulated yychar.  (However, yychar is currently out
+         of scope during semantic actions.)
+       - Of course, the expected token list depends on states to
+         have correct lookahead information, and it depends on the
+         parser not to perform extra reductions after fetching a
+         lookahead from the scanner and before detecting a syntax
+         error.  Thus, state merging (from LALR or IELR) and default
+         reductions corrupt the expected token list.  However, the
+         list is correct for canonical LR with one exception: it
+         will still contain any token that will not be accepted due
+         to an error action in a later state.
+    */
+    int yycount = 0;
+    if (yyctx.getToken() != null)
+      {
+        if (yyarg != null)
+          yyarg[yycount] = yyctx.getToken();
+        yycount += 1;
+        yycount += yyctx.getExpectedTokens(yyarg, 1, yyargn);
       }
+    return yycount;
+  }
 ]])[
-    return "syntax error";
+
+  /**
+   * Build and emit a "syntax error" message in a user-defined way.
+   *
+   * @@param ctx  The context of the error.
+   */
+  private void yyreportSyntaxError(Context yyctx) {]b4_parse_error_bmatch(
+[custom], [[
+      yylexer.reportSyntaxError(yyctx);]],
+[detailed\|verbose], [[
+      if (yyErrorVerbose) {
+          final int argmax = 5;
+          SymbolKind[] yyarg = new SymbolKind[argmax];
+          int yycount = yysyntaxErrorArguments(yyctx, yyarg, argmax);
+          String[] yystr = new String[yycount];
+          for (int yyi = 0; yyi < yycount; ++yyi) {
+              yystr[yyi] = yyarg[yyi].getName();
+          }
+          String yyformat;
+          switch (yycount) {
+              default:
+              case 0: yyformat = ]b4_trans(["syntax error"])[; break;
+              case 1: yyformat = ]b4_trans(["syntax error, unexpected {0}"])[; break;
+              case 2: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1}"])[; break;
+              case 3: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1} or {2}"])[; break;
+              case 4: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1} or {2} or {3}"])[; break;
+              case 5: yyformat = ]b4_trans(["syntax error, unexpected {0}, expecting {1} or {2} or {3} or {4}"])[; break;
+          }
+          yyerror(]b4_locations_if([[yyctx.yylocation, ]])[new MessageFormat(yyformat).format(yystr));
+      } else {
+          yyerror(]b4_locations_if([[yyctx.yylocation, ]])[]b4_trans(["syntax error"])[);
+      }]],
+[simple], [[
+      yyerror(]b4_locations_if([[yyctx.yylocation, ]])[]b4_trans(["syntax error"])[);]])[
   }
 
   /**
@@ -976,11 +1050,7 @@ b4_dollar_popdef[]dnl
   private static final ]b4_int_type_for([b4_pact])[ yypact_ninf_ = ]b4_pact_ninf[;
   private static final ]b4_int_type_for([b4_table])[ yytable_ninf_ = ]b4_table_ninf[;
 
-  ]b4_parser_tables_define[
-
-  /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
-     First, the terminals, then, starting at \a yyntokens_, nonterminals.  */
-  ]b4_typed_parser_table_define([String], [tname], [b4_tname])[
+]b4_parser_tables_define[
 
 ]b4_parse_trace_if([[
   ]b4_integral_parser_table_define([rline], [b4_rline],
@@ -997,50 +1067,44 @@ b4_dollar_popdef[]dnl
     int yynrhs = yyr2_[yyrule];
     /* Print the symbols being reduced, and their result.  */
     yycdebug ("Reducing stack by rule " + (yyrule - 1)
-              + " (line " + yylno + "), ");
+              + " (line " + yylno + "):");
 
     /* The symbols being reduced.  */
     for (int yyi = 0; yyi < yynrhs; yyi++)
-      yySymbolPrint ("   $" + (yyi + 1) + " =",
-                     yystos_[yystack.stateAt(yynrhs - (yyi + 1))],
-                     ]b4_rhs_data(yynrhs, yyi + 1)b4_locations_if([,
-                     b4_rhs_location(yynrhs, yyi + 1)])[);
+      yySymbolPrint("   $" + (yyi + 1) + " =",
+                    SymbolKind.get(yystos_[yystack.stateAt (yynrhs - (yyi + 1))]),
+                    ]b4_rhs_data(yynrhs, yyi + 1)b4_locations_if([,
+                    b4_rhs_location(yynrhs, yyi + 1)])[);
   }]])[
 
   /* YYTRANSLATE_(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
      as returned by yylex, with out-of-bounds checking.  */
-  private static final ]b4_int_type_for([b4_translate])[ yytranslate_ (int t)
+  private static final SymbolKind yytranslate_(int t)
 ]b4_api_token_raw_if(dnl
 [[  {
-    return t;
+    return SymbolKind.get(t);
   }
 ]],
 [[  {
-    int user_token_number_max_ = ]b4_user_token_number_max[;
-    ]b4_int_type_for([b4_translate])[ undef_token_ = ]b4_undef_token_number[;
-
+    // Last valid token kind.
+    int code_max = ]b4_code_max[;
     if (t <= 0)
-      return Lexer.EOF;
-    else if (t <= user_token_number_max_)
-      return yytranslate_table_[t];
+      return ]b4_symbol(0, kind)[;
+    else if (t <= code_max)
+      return SymbolKind.get(yytranslate_table_[t]);
     else
-      return undef_token_;
+      return ]b4_symbol(2, kind)[;
   }
   ]b4_integral_parser_table_define([translate_table], [b4_translate])[
 ]])[
 
-  private static final ]b4_int_type_for([b4_translate])[ yy_error_token_ = 1;
-
-  private static final int yylast_ = ]b4_last[;
-  private static final int yynnts_ = ]b4_nterms_number[;
-  private static final int yyempty_ = -2;
-  private static final int yyfinal_ = ]b4_final_state_number[;
-  private static final int yyntokens_ = ]b4_tokens_number[;
-
-/* User implementation code.  */
-]b4_percent_code_get[]dnl
+  private static final int YYLAST_ = ]b4_last[;
+  private static final int YYEMPTY_ = -2;
+  private static final int YYFINAL_ = ]b4_final_state_number[;
+  private static final int YYNTOKENS_ = ]b4_tokens_number[;
 
+]b4_percent_code_get[
 }
-
+]b4_percent_code_get([[epilogue]])[]dnl
 b4_epilogue[]dnl
 b4_output_end

data/skeletons/location.cc

@@ -48,7 +48,7 @@ m4_ifdef([b4_location_file],
                                   ["b4_location_file"])])
  m4_define([b4_location_path],
            b4_percent_define_get([[api.location.include]],
-                                 ["b4_dir_prefix[]b4_location_file"]))
+                                 ["b4_mapped_dir_prefix[]b4_location_file"]))
  m4_define([b4_location_path],
            m4_substr(m4_defn([b4_location_path]), 1, m4_eval(m4_len(m4_defn([b4_location_path])) - 2)))
  ])
@@ -63,11 +63,13 @@ m4_define([b4_location_define],
   class position
   {
   public:
+    /// Type for file name.
+    typedef ]b4_percent_define_get([[api.filename.type]])[ filename_type;
     /// Type for line and column numbers.
     typedef int counter_type;
 ]m4_ifdef([b4_location_constructors], [[
     /// Construct a position.
-    explicit position (]b4_percent_define_get([[filename_type]])[* f = YY_NULLPTR,
+    explicit position (filename_type* f = YY_NULLPTR,
                        counter_type l = ]b4_location_initial_line[,
                        counter_type c = ]b4_location_initial_column[)
       : filename (f)
@@ -77,7 +79,7 @@ m4_define([b4_location_define],
 
 ]])[
     /// Initialization.
-    void initialize (]b4_percent_define_get([[filename_type]])[* fn = YY_NULLPTR,
+    void initialize (filename_type* fn = YY_NULLPTR,
                      counter_type l = ]b4_location_initial_line[,
                      counter_type c = ]b4_location_initial_column[)
     {
@@ -106,7 +108,7 @@ m4_define([b4_location_define],
     /** \} */
 
     /// File name to which this position refers.
-    ]b4_percent_define_get([[filename_type]])[* filename;
+    filename_type* filename;
     /// Current line number.
     counter_type line;
     /// Current column number.
@@ -184,6 +186,8 @@ m4_define([b4_location_define],
   class location
   {
   public:
+    /// Type for file name.
+    typedef position::filename_type filename_type;
     /// Type for line and column numbers.
     typedef position::counter_type counter_type;
 ]m4_ifdef([b4_location_constructors], [
@@ -200,7 +204,7 @@ m4_define([b4_location_define],
     {}
 
     /// Construct a 0-width location in \a f, \a l, \a c.
-    explicit location (]b4_percent_define_get([[filename_type]])[* f,
+    explicit location (filename_type* f,
                        counter_type l = ]b4_location_initial_line[,
                        counter_type c = ]b4_location_initial_column[)
       : begin (f, l, c)
@@ -209,7 +213,7 @@ m4_define([b4_location_define],
 
 ])[
     /// Initialization.
-    void initialize (]b4_percent_define_get([[filename_type]])[* f = YY_NULLPTR,
+    void initialize (filename_type* f = YY_NULLPTR,
                      counter_type l = ]b4_location_initial_line[,
                      counter_type c = ]b4_location_initial_column[)
     {

data/skeletons/stack.hh

@@ -32,8 +32,8 @@ m4_define([b4_stack_define],
     {
     public:
       // Hide our reversed order.
-      typedef typename S::reverse_iterator iterator;
-      typedef typename S::const_reverse_iterator const_iterator;
+      typedef typename S::iterator iterator;
+      typedef typename S::const_iterator const_iterator;
       typedef typename S::size_type size_type;
       typedef typename std::ptrdiff_t index_type;
 
@@ -41,6 +41,13 @@ m4_define([b4_stack_define],
         : seq_ (n)
       {}
 
+#if 201103L <= YY_CPLUSPLUS
+      /// Non copyable.
+      stack (const stack&) = delete;
+      /// Non copyable.
+      stack& operator= (const stack&) = delete;
+#endif
+
       /// Random access.
       ///
       /// Index 0 returns the topmost element.
@@ -91,24 +98,18 @@ m4_define([b4_stack_define],
         return index_type (seq_.size ());
       }
 
-      std::ptrdiff_t
-      ssize () const YY_NOEXCEPT
-      {
-        return std::ptrdiff_t (size ());
-      }
-
       /// Iterator on top of the stack (going downwards).
       const_iterator
       begin () const YY_NOEXCEPT
       {
-        return seq_.rbegin ();
+        return seq_.begin ();
       }
 
       /// Bottom of the stack.
       const_iterator
       end () const YY_NOEXCEPT
       {
-        return seq_.rend ();
+        return seq_.end ();
       }
 
       /// Present a slice of the top of a stack.
@@ -132,8 +133,12 @@ m4_define([b4_stack_define],
       };
 
     private:
+#if YY_CPLUSPLUS < 201103L
+      /// Non copyable.
       stack (const stack&);
+      /// Non copyable.
       stack& operator= (const stack&);
+#endif
       /// The wrapped container.
       S seq_;
     };

data/skeletons/traceon.m4

@@ -0,0 +1,2 @@
+dnl GNU M4 treats -dV in a position-independent manner.
+m4_debugmode(V)m4_traceon()dnl

data/skeletons/variant.hh

@@ -115,6 +115,13 @@ m4_define([b4_value_type_declare],
       new (yyas_<T> ()) T (YY_MOVE (t));
     }
 
+#if 201103L <= YY_CPLUSPLUS
+    /// Non copyable.
+    semantic_type (const self_type&) = delete;
+    /// Non copyable.
+    self_type& operator= (const self_type&) = delete;
+#endif
+
     /// Destruction, allowed only if empty.
     ~semantic_type () YY_NOEXCEPT
     {]b4_parse_assert_if([
@@ -258,9 +265,12 @@ m4_define([b4_value_type_declare],
     }
 
   private:
-    /// Prohibit blind copies.
-    self_type& operator= (const self_type&);
+#if YY_CPLUSPLUS < 201103L
+    /// Non copyable.
     semantic_type (const self_type&);
+    /// Non copyable.
+    self_type& operator= (const self_type&);
+#endif
 
     /// Accessor to raw memory as \a T.
     template <typename T>
@@ -382,7 +392,7 @@ m4_define([_b4_type_clause],
 [b4_symbol_if([$1], [is_token],
               [b4_symbol_if([$1], [has_id],
                             [tok == token::b4_symbol([$1], [id])],
-                            [tok == b4_symbol([$1], [user_number])])])])
+                            [tok == b4_symbol([$1], [code])])])])
 
 
 # _b4_token_constructor_define(SYMBOL-NUM...)

data/xslt/xml2dot.xsl

@@ -159,7 +159,7 @@
   <xsl:param name="prev-rule-number"
     select="preceding-sibling::item[1]/@rule-number"/>
   <xsl:apply-templates select="key('bison:ruleByNumber', @rule-number)">
-    <xsl:with-param name="point" select="@point"/>
+    <xsl:with-param name="dot" select="@dot"/>
     <xsl:with-param name="num" select="@rule-number"/>
     <xsl:with-param name="prev-lhs"
       select="key('bison:ruleByNumber', $prev-rule-number)/lhs[text()]"
@@ -169,7 +169,7 @@
 </xsl:template>
 
 <xsl:template match="rule">
-  <xsl:param name="point"/>
+  <xsl:param name="dot"/>
   <xsl:param name="num"/>
   <xsl:param name="prev-lhs"/>
   <xsl:text>&#10;</xsl:text>
@@ -198,14 +198,14 @@
       <xsl:text>:</xsl:text>
     </xsl:otherwise>
   </xsl:choose>
-  <xsl:if test="$point = 0">
+  <xsl:if test="$dot = 0">
     <xsl:text> .</xsl:text>
   </xsl:if>
 
   <!-- RHS -->
   <xsl:for-each select="rhs/symbol|rhs/empty">
     <xsl:apply-templates select="."/>
-    <xsl:if test="$point = position()">
+    <xsl:if test="$dot = position()">
       <xsl:text> .</xsl:text>
     </xsl:if>
   </xsl:for-each>

data/xslt/xml2text.xsl

@@ -52,7 +52,7 @@
   <xsl:if test="nonterminal[@usefulness='useless-in-grammar']">
     <xsl:text>Nonterminals useless in grammar&#10;&#10;</xsl:text>
     <xsl:for-each select="nonterminal[@usefulness='useless-in-grammar']">
-      <xsl:text>   </xsl:text>
+      <xsl:text>    </xsl:text>
       <xsl:value-of select="@name"/>
       <xsl:text>&#10;</xsl:text>
     </xsl:for-each>
@@ -65,7 +65,7 @@
     <xsl:text>Terminals unused in grammar&#10;&#10;</xsl:text>
     <xsl:for-each select="terminal[@usefulness='unused-in-grammar']">
       <xsl:sort select="@symbol-number" data-type="number"/>
-      <xsl:text>   </xsl:text>
+      <xsl:text>    </xsl:text>
       <xsl:value-of select="@name"/>
       <xsl:text>&#10;</xsl:text>
     </xsl:for-each>
@@ -136,6 +136,7 @@
 </xsl:template>
 
 <xsl:template match="terminal">
+  <xsl:text>    </xsl:text>
   <xsl:value-of select="@name"/>
   <xsl:call-template name="line-wrap">
     <xsl:with-param name="first-line-length">
@@ -148,6 +149,9 @@
     </xsl:with-param>
     <xsl:with-param name="line-length" select="66" />
     <xsl:with-param name="text">
+      <xsl:if test="string-length(@type) != 0">
+        <xsl:value-of select="concat(' &lt;', @type, '&gt;')"/>
+      </xsl:if>
       <xsl:value-of select="concat(' (', @token-number, ')')"/>
       <xsl:for-each select="key('bison:ruleByRhs', @name)">
         <xsl:value-of select="concat(' ', @number)"/>
@@ -157,14 +161,18 @@
 </xsl:template>
 
 <xsl:template match="nonterminal">
+  <xsl:text>    </xsl:text>
   <xsl:value-of select="@name"/>
+  <xsl:if test="string-length(@type) != 0">
+    <xsl:value-of select="concat(' &lt;', @type, '&gt;')"/>
+  </xsl:if>
   <xsl:value-of select="concat(' (', @symbol-number, ')')"/>
   <xsl:text>&#10;</xsl:text>
   <xsl:variable name="output">
     <xsl:call-template name="line-wrap">
       <xsl:with-param name="line-length" select="66" />
       <xsl:with-param name="text">
-        <xsl:text>    </xsl:text>
+        <xsl:text>        </xsl:text>
         <xsl:if test="key('bison:ruleByLhs', @name)">
           <xsl:text>on@left:</xsl:text>
           <xsl:for-each select="key('bison:ruleByLhs', @name)">
@@ -173,7 +181,7 @@
         </xsl:if>
         <xsl:if test="key('bison:ruleByRhs', @name)">
           <xsl:if test="key('bison:ruleByLhs', @name)">
-            <xsl:text>, </xsl:text>
+            <xsl:text>&#10;        </xsl:text>
           </xsl:if>
           <xsl:text>on@right:</xsl:text>
           <xsl:for-each select="key('bison:ruleByRhs', @name)">
@@ -300,7 +308,7 @@
       name="prev-lhs"
       select="key('bison:ruleByNumber', $prev-rule-number)/lhs[text()]"
    />
-    <xsl:with-param name="point" select="@point"/>
+    <xsl:with-param name="dot" select="@dot"/>
     <xsl:with-param name="lookaheads">
       <xsl:apply-templates select="lookaheads"/>
     </xsl:with-param>
@@ -311,7 +319,7 @@
   <xsl:param name="itemset"/>
   <xsl:param name="pad"/>
   <xsl:param name="prev-lhs"/>
-  <xsl:param name="point"/>
+  <xsl:param name="dot"/>
   <xsl:param name="lookaheads"/>
 
   <xsl:if test="$itemset != 'true' and not($prev-lhs = lhs[text()])">
@@ -347,12 +355,12 @@
 
   <!-- RHS -->
   <xsl:for-each select="rhs/*">
-    <xsl:if test="position() = $point + 1">
-      <xsl:text> .</xsl:text>
+    <xsl:if test="position() = $dot + 1">
+      <xsl:text> •</xsl:text>
     </xsl:if>
     <xsl:apply-templates select="."/>
-    <xsl:if test="position() = last() and position() = $point">
-      <xsl:text> .</xsl:text>
+    <xsl:if test="position() = last() and position() = $dot">
+      <xsl:text> •</xsl:text>
     </xsl:if>
   </xsl:for-each>
   <xsl:if test="$lookaheads">

data/xslt/xml2xhtml.xsl

@@ -75,7 +75,7 @@
       ol.lower-alpha {
         list-style-type: lower-alpha;
       }
-      .point {
+      .dot {
         color: #cc0000;
       }
       #footer {
@@ -227,6 +227,7 @@
   <xsl:text>&#10;</xsl:text>
   <p class="pre">
     <xsl:call-template name="style-rule-set">
+      <xsl:with-param name="anchor" select="'true'" />
       <xsl:with-param
         name="rule-set" select="rules/rule[@usefulness!='useless-in-grammar']"
       />
@@ -238,9 +239,11 @@
 </xsl:template>
 
 <xsl:template name="style-rule-set">
+  <xsl:param name="anchor"/>
   <xsl:param name="rule-set"/>
   <xsl:for-each select="$rule-set">
     <xsl:apply-templates select=".">
+      <xsl:with-param name="anchor" select="$anchor"/>
       <xsl:with-param name="pad" select="'3'"/>
       <xsl:with-param name="prev-lhs">
         <xsl:if test="position()>1">
@@ -306,9 +309,10 @@
     <xsl:text> Terminals, with rules where they appear</xsl:text>
   </h3>
   <xsl:text>&#10;&#10;</xsl:text>
-  <p class="pre">
+  <ul>
+    <xsl:text>&#10;</xsl:text>
     <xsl:apply-templates select="terminal"/>
-  </p>
+  </ul>
   <xsl:text>&#10;&#10;</xsl:text>
 </xsl:template>
 
@@ -318,41 +322,64 @@
     <xsl:text> Nonterminals, with rules where they appear</xsl:text>
   </h3>
   <xsl:text>&#10;&#10;</xsl:text>
-  <p class="pre">
+  <ul>
+    <xsl:text>&#10;</xsl:text>
     <xsl:apply-templates
       select="nonterminal[@usefulness!='useless-in-grammar']"
     />
-  </p>
+  </ul>
 </xsl:template>
 
 <xsl:template match="terminal">
-  <b><xsl:value-of select="@name"/></b>
-  <xsl:value-of select="concat(' (', @token-number, ')')"/>
-  <xsl:for-each select="key('bison:ruleByRhs', @name)">
-    <xsl:apply-templates select="." mode="number-link"/>
-  </xsl:for-each>
+  <xsl:text>  </xsl:text>
+  <li>
+    <b><xsl:value-of select="@name"/></b>
+    <xsl:if test="string-length(@type) != 0">
+      <xsl:value-of select="concat(' &lt;', @type, '&gt;')"/>
+    </xsl:if>
+    <xsl:value-of select="concat(' (', @token-number, ')')"/>
+    <xsl:for-each select="key('bison:ruleByRhs', @name)">
+      <xsl:apply-templates select="." mode="number-link"/>
+    </xsl:for-each>
+  </li>
   <xsl:text>&#10;</xsl:text>
 </xsl:template>
 
 <xsl:template match="nonterminal">
-  <b><xsl:value-of select="@name"/></b>
-  <xsl:value-of select="concat(' (', @symbol-number, ')')"/>
-  <xsl:text>&#10;    </xsl:text>
-  <xsl:if test="key('bison:ruleByLhs', @name)">
-    <xsl:text>on left:</xsl:text>
-    <xsl:for-each select="key('bison:ruleByLhs', @name)">
-      <xsl:apply-templates select="." mode="number-link"/>
-    </xsl:for-each>
-  </xsl:if>
-  <xsl:if test="key('bison:ruleByRhs', @name)">
-    <xsl:if test="key('bison:ruleByLhs', @name)">
-      <xsl:text>&#10;    </xsl:text>
+  <xsl:text>  </xsl:text>
+  <li>
+    <b><xsl:value-of select="@name"/></b>
+    <xsl:if test="string-length(@type) != 0">
+      <xsl:value-of select="concat(' &lt;', @type, '&gt;')"/>
     </xsl:if>
-    <xsl:text>on right:</xsl:text>
-    <xsl:for-each select="key('bison:ruleByRhs', @name)">
-      <xsl:apply-templates select="." mode="number-link"/>
-    </xsl:for-each>
-  </xsl:if>
+    <xsl:value-of select="concat(' (', @symbol-number, ')')"/>
+    <xsl:text>&#10;    </xsl:text>
+    <ul>
+      <xsl:text>&#10;</xsl:text>
+      <xsl:if test="key('bison:ruleByLhs', @name)">
+        <xsl:text>      </xsl:text>
+        <li>
+          <xsl:text>on left:</xsl:text>
+          <xsl:for-each select="key('bison:ruleByLhs', @name)">
+            <xsl:apply-templates select="." mode="number-link"/>
+          </xsl:for-each>
+        </li>
+        <xsl:text>&#10;</xsl:text>
+      </xsl:if>
+      <xsl:if test="key('bison:ruleByRhs', @name)">
+        <xsl:text>      </xsl:text>
+        <li>
+          <xsl:text>on right:</xsl:text>
+          <xsl:for-each select="key('bison:ruleByRhs', @name)">
+            <xsl:apply-templates select="." mode="number-link"/>
+          </xsl:for-each>
+        </li>
+        <xsl:text>&#10;</xsl:text>
+      </xsl:if>
+    <xsl:text>    </xsl:text>
+    </ul>
+    <xsl:text>&#10;  </xsl:text>
+  </li>
   <xsl:text>&#10;</xsl:text>
 </xsl:template>
 
@@ -385,7 +412,7 @@
         <xsl:value-of select="concat('state_', @number)"/>
       </xsl:attribute>
     </a>
-    <xsl:text>state </xsl:text>
+    <xsl:text>State </xsl:text>
     <xsl:value-of select="@number"/>
   </h3>
   <xsl:text>&#10;&#10;</xsl:text>
@@ -457,35 +484,44 @@
     <xsl:with-param name="prev-lhs"
       select="key('bison:ruleByNumber', $prev-rule-number)/lhs[text()]"
    />
-    <xsl:with-param name="point" select="@point"/>
+    <xsl:with-param name="dot" select="@dot"/>
     <xsl:with-param name="lookaheads">
       <xsl:apply-templates select="lookaheads"/>
     </xsl:with-param>
   </xsl:apply-templates>
 </xsl:template>
 
+<!--
+anchor = 'true': define as an <a> anchor.
+itemset = 'true': show the items.
+ -->
 <xsl:template match="rule">
+  <xsl:param name="anchor"/>
   <xsl:param name="itemset"/>
   <xsl:param name="pad"/>
   <xsl:param name="prev-lhs"/>
-  <xsl:param name="point"/>
+  <xsl:param name="dot"/>
   <xsl:param name="lookaheads"/>
 
   <xsl:if test="$itemset != 'true' and not($prev-lhs = lhs[text()])">
     <xsl:text>&#10;</xsl:text>
   </xsl:if>
 
-  <xsl:if test="$itemset != 'true'">
-    <a>
-      <xsl:attribute name="name">
-        <xsl:value-of select="concat('rule_', @number)"/>
-      </xsl:attribute>
-    </a>
-  </xsl:if>
   <xsl:text>  </xsl:text>
 
   <xsl:choose>
-    <xsl:when test="$itemset = 'true'">
+    <xsl:when test="$anchor = 'true'">
+      <a>
+        <xsl:attribute name="name">
+          <xsl:value-of select="concat('rule_', @number)"/>
+        </xsl:attribute>
+        <xsl:call-template name="lpad">
+          <xsl:with-param name="str" select="string(@number)"/>
+          <xsl:with-param name="pad" select="number($pad)"/>
+        </xsl:call-template>
+      </a>
+    </xsl:when>
+    <xsl:otherwise>
       <a>
         <xsl:attribute name="href">
           <xsl:value-of select="concat('#rule_', @number)"/>
@@ -495,25 +531,13 @@
           <xsl:with-param name="pad" select="number($pad)"/>
         </xsl:call-template>
       </a>
-    </xsl:when>
-    <xsl:otherwise>
-      <xsl:call-template name="lpad">
-        <xsl:with-param name="str" select="string(@number)"/>
-        <xsl:with-param name="pad" select="number($pad)"/>
-      </xsl:call-template>
     </xsl:otherwise>
   </xsl:choose>
   <xsl:text> </xsl:text>
 
   <!-- LHS -->
   <xsl:choose>
-    <xsl:when test="$itemset != 'true' and $prev-lhs = lhs[text()]">
-      <xsl:call-template name="lpad">
-        <xsl:with-param name="str" select="'|'"/>
-        <xsl:with-param name="pad" select="number(string-length(lhs[text()])) + 2"/>
-      </xsl:call-template>
-    </xsl:when>
-    <xsl:when test="$itemset = 'true' and $prev-lhs = lhs[text()]">
+    <xsl:when test="$prev-lhs = lhs[text()]">
       <xsl:call-template name="lpad">
         <xsl:with-param name="str" select="'|'"/>
         <xsl:with-param name="pad" select="number(string-length(lhs[text()])) + 2"/>
@@ -529,14 +553,14 @@
 
   <!-- RHS -->
   <xsl:for-each select="rhs/*">
-    <xsl:if test="position() = $point + 1">
+    <xsl:if test="position() = $dot + 1">
       <xsl:text> </xsl:text>
-      <span class="point">.</span>
+      <span class="dot">&#x2022;</span>
     </xsl:if>
     <xsl:apply-templates select="."/>
-    <xsl:if test="position() = last() and position() = $point">
+    <xsl:if test="position() = last() and position() = $dot">
       <xsl:text> </xsl:text>
-      <span class="point">.</span>
+      <span class="dot">&#x2022;</span>
     </xsl:if>
   </xsl:for-each>
   <xsl:if test="$lookaheads">

doc/Doxyfile.in

@@ -1,4 +1,4 @@
-# Doxyfile 1.8.17
+# Doxyfile 1.8.18
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -253,12 +253,6 @@ TAB_SIZE               = 8
 
 ALIASES                =
 
-# This tag can be used to specify a number of word-keyword mappings (TCL only).
-# A mapping has the form "name=value". For example adding "class=itcl::class"
-# will allow you to use the command class in the itcl::class meaning.
-
-TCL_SUBST              =
-
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all
@@ -300,13 +294,13 @@ OPTIMIZE_OUTPUT_SLICE  = NO
 # extension. Doxygen has a built-in mapping, but you can override or extend it
 # using this tag. The format is ext=language, where ext is a file extension, and
 # language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
-# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL,
 # Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
 # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
 # tries to guess whether the code is fixed or free formatted code, this is the
-# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat
-# .inc files as Fortran files (default is PHP), and .f files as C (default is
-# Fortran), use: inc=Fortran f=C.
+# default for Fortran type files). For instance to make doxygen treat .inc files
+# as Fortran files (default is PHP), and .f files as C (default is Fortran),
+# use: inc=Fortran f=C.
 #
 # Note: For files without extension you can use no_extension as a placeholder.
 #
@@ -844,7 +838,7 @@ INPUT_ENCODING         = UTF-8
 # *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
 # *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment),
 # *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen
-# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f, *.for, *.tcl, *.vhd,
+# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd,
 # *.vhdl, *.ucf, *.qsf and *.ice.
 
 FILE_PATTERNS          = *.c \
@@ -1490,6 +1484,17 @@ TREEVIEW_WIDTH         = 250
 
 EXT_LINKS_IN_WINDOW    = NO
 
+# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg
+# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see
+# https://inkscape.org) to generate formulas as SVG images instead of PNGs for
+# the HTML output. These images will generally look nicer at scaled resolutions.
+# Possible values are: png The default and svg Looks nicer but requires the
+# pdf2svg tool.
+# The default value is: png.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FORMULA_FORMAT    = png
+
 # Use this tag to change the font size of LaTeX formulas included as images in
 # the HTML documentation. When you change the font size after a successful
 # doxygen run you need to manually remove any form_*.png images from the HTML
@@ -1545,7 +1550,7 @@ MATHJAX_FORMAT         = HTML-CSS
 # Content Delivery Network so you can quickly see the result without installing
 # MathJax. However, it is strongly recommended to install a local copy of
 # MathJax from https://www.mathjax.org before deployment.
-# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/.
+# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
 MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
@@ -1715,7 +1720,7 @@ COMPACT_LATEX          = NO
 # The default value is: a4.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
-PAPER_TYPE             = a4wide
+PAPER_TYPE             = a4
 
 # The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
 # that should be included in the LaTeX output. The package can be specified just

doc/calc.y

@@ -0,0 +1,23 @@
+%union
+{
+  int ival;
+  const char *sval;
+}
+
+%token <ival> NUM
+%nterm <ival> exp
+%token <sval> STR
+%nterm <sval> useless
+
+%left '+' '-'
+%left '*'
+
+%%
+exp:
+  exp '+' exp
+| exp '-' exp
+| exp '*' exp
+| exp '/' exp
+| NUM
+;
+useless: STR;

doc/figs/example-reduce.gv

@@ -3,7 +3,7 @@ digraph "reduce.y"
   node [fontname=courier shape=box]
   edge [fontname=courier]
 
-  1 [label="State 1\n  3 a: \"0\" .  [\".\"]\l  4 b: \"0\" .  [\";\"]\l"]
+  1 [label="State 1\n  3 a: \"0\" •  [\".\"]\l  4 b: \"0\" •  [\";\"]\l"]
   1 -> "1R3" [label="" style=solid]
  "1R3" [style=filled shape=diamond fillcolor=yellowgreen label="R3"]
   1 -> "1R4" [label="[\";\"]" style=solid]

doc/figs/example-reduce.txt

@@ -1,7 +1,7 @@
        .------------------.
        |     State 1      |
-       | 3 a: "0" . [";"] |
-       | 4 b: "0" . ["."] |
+       | 3 a: "0" • [";"] |
+       | 4 b: "0" • ["."] |
        `------------------'
              /     \
             /       \ ["."]

doc/figs/example-shift.gv

@@ -3,7 +3,7 @@ digraph "shift.y"
   node [fontname=courier shape=box]
   edge [fontname=courier]
 
-  3 [label="State 3\n  1 exp: a . \".\"\l"]
+  3 [label="State 3\n  1 exp: a • \".\"\l"]
   3 -> 6 [style=solid label="\".\""]
-  6 [label="State 6\n  1 exp: a \".\" .\l"]
+  6 [label="State 6\n  1 exp: a \".\" •\l"]
 }

doc/figs/example-shift.txt

@@ -1,6 +1,6 @@
 .----------------.
 |    State 3     |
-| 1 exp: a . ";" |
+| 1 exp: a • ";" |
 `----------------'
         |
         | ";"
@@ -8,5 +8,5 @@
         v
 .----------------.
 |    State 6     |
-| 1 exp: a ";" . |
+| 1 exp: a ";" • |
 `----------------'

doc/figs/example.gv

@@ -2,34 +2,34 @@
 // Report bugs to <[email protected]>.
 // Home page: <http://www.gnu.org/software/bison/>.
 
-digraph "/tmp/fo.y"
+digraph "example.y"
 {
   node [fontname = courier, shape = box, colorscheme = paired6]
   edge [fontname = courier]
 
-  0 [label="State 0\n\l  0 $accept: . exp $end\l  1 exp: . a \";\"\l  2    | . b \".\"\l  3 a: . \"0\"\l  4 b: . \"0\"\l"]
+  0 [label="State 0\n\l  0 $accept: • exp $end\l  1 exp: • a \";\"\l  2    | • b \".\"\l  3 a: • \"0\"\l  4 b: • \"0\"\l"]
   0 -> 1 [style=solid label="\"0\""]
   0 -> 2 [style=dashed label="exp"]
   0 -> 3 [style=dashed label="a"]
   0 -> 4 [style=dashed label="b"]
-  1 [label="State 1\n\l  3 a: \"0\" .  [\";\"]\l  4 b: \"0\" .  [\".\"]\l"]
+  1 [label="State 1\n\l  3 a: \"0\" •  [\";\"]\l  4 b: \"0\" •  [\".\"]\l"]
   1 -> "1R3" [style=solid]
  "1R3" [label="R3", fillcolor=3, shape=diamond, style=filled]
   1 -> "1R4" [label="[\".\"]", style=solid]
  "1R4" [label="R4", fillcolor=3, shape=diamond, style=filled]
-  2 [label="State 2\n\l  0 $accept: exp . $end\l"]
+  2 [label="State 2\n\l  0 $accept: exp • $end\l"]
   2 -> 5 [style=solid label="$end"]
-  3 [label="State 3\n\l  1 exp: a . \";\"\l"]
+  3 [label="State 3\n\l  1 exp: a • \";\"\l"]
   3 -> 6 [style=solid label="\";\""]
-  4 [label="State 4\n\l  2 exp: b . \".\"\l"]
+  4 [label="State 4\n\l  2 exp: b • \".\"\l"]
   4 -> 7 [style=solid label="\".\""]
-  5 [label="State 5\n\l  0 $accept: exp $end .\l"]
+  5 [label="State 5\n\l  0 $accept: exp $end •\l"]
   5 -> "5R0" [style=solid]
  "5R0" [label="Acc", fillcolor=1, shape=diamond, style=filled]
-  6 [label="State 6\n\l  1 exp: a \";\" .\l"]
+  6 [label="State 6\n\l  1 exp: a \";\" •\l"]
   6 -> "6R1" [style=solid]
  "6R1" [label="R1", fillcolor=3, shape=diamond, style=filled]
-  7 [label="State 7\n\l  2 exp: b \".\" .\l"]
+  7 [label="State 7\n\l  2 exp: b \".\" •\l"]
   7 -> "7R2" [style=solid]
  "7R2" [label="R2", fillcolor=3, shape=diamond, style=filled]
 }

doc/ids.y

@@ -0,0 +1,5 @@
+%token ID
+%%
+s: a ID
+a: expr
+expr: %empty | expr ID ','

doc/if-then-else.y

@@ -0,0 +1,13 @@
+%%
+stmt:
+  expr
+| if_stmt
+;
+
+if_stmt:
+  "if" expr "then" stmt
+| "if" expr "then" stmt "else" stmt
+;
+
+expr:
+  "identifier"

doc/local.mk

@@ -20,36 +20,36 @@ AM_MAKEINFOFLAGS =                                              \
   --set-customization-variable=AVOID_MENU_REDUNDANCY=true       \
   --set-customization-variable=ICONS=true
 
-info_TEXINFOS = doc/bison.texi
-doc_bison_TEXINFOS =                            \
+info_TEXINFOS = %D%/bison.texi
+%C%_bison_TEXINFOS =                            \
   $(CROSS_OPTIONS_TEXI)                         \
-  doc/fdl.texi                                  \
-  doc/gpl-3.0.texi                              \
-  doc/relocatable.texi
+  %D%/fdl.texi                                  \
+  %D%/gpl-3.0.texi                              \
+  %D%/relocatable.texi
 
 # Cannot express dependencies directly on file names because of Automake.
 # Obfuscate with a variable.
-doc_bison = doc/bison
-$(doc_bison).dvi: $(FIGS_GV:.gv=.eps)
-$(doc_bison).info: $(FIGS_GV:.gv=.txt)
-$(doc_bison).pdf: $(FIGS_GV:.gv=.pdf)
-$(doc_bison).html: $(FIGS_GV:.gv=.svg)
+%C%_bison = %D%/bison
+$(%C%_bison).dvi: $(FIGS_GV:.gv=.eps)
+$(%C%_bison).info: $(FIGS_GV:.gv=.txt)
+$(%C%_bison).pdf: $(FIGS_GV:.gv=.pdf)
+$(%C%_bison).html: $(FIGS_GV:.gv=.svg)
 
-TEXI2DVI = texi2dvi --build-dir=doc/bison.t2d -I doc
-CLEANDIRS += doc/bison.t2d
+TEXI2DVI = texi2dvi --build-dir=%D%/bison.t2d -I %D%
+CLEANDIRS += %D%/bison.t2d
 
-MOSTLYCLEANFILES += $(top_srcdir)/doc/*.tmp
+MOSTLYCLEANFILES += $(top_srcdir)/%D%/*.tmp
 
 CROSS_OPTIONS_PL = $(top_srcdir)/build-aux/cross-options.pl
-CROSS_OPTIONS_TEXI = $(top_srcdir)/doc/cross-options.texi
-$(CROSS_OPTIONS_TEXI): doc/bison.help $(CROSS_OPTIONS_PL)
+CROSS_OPTIONS_TEXI = $(top_srcdir)/%D%/cross-options.texi
+$(CROSS_OPTIONS_TEXI): %D%/bison.help $(CROSS_OPTIONS_PL)
 # Create $@~ which is the previous contents.  Don't use 'mv' here so
 # that even if we are interrupted, the file is still available for
 # diff in the next run.  Note that $@ might not exist yet.
 	$(AM_V_GEN){ test ! -f $@ || cat $@; } >$@~
 	$(AM_V_at)test ! -f $@.tmp || rm -f $@.tmp
 	$(AM_V_at)$(PERL) $(CROSS_OPTIONS_PL) $(top_srcdir)/src/scan-gram.l \
-	  <$(top_srcdir)/doc/bison.help >$@.tmp
+	  <$(top_srcdir)/%D%/bison.help >$@.tmp
 	$(AM_V_at)diff -u $@~ $@.tmp || true
 	$(AM_V_at)mv $@.tmp $@
 MAINTAINERCLEANFILES = $(CROSS_OPTIONS_TEXI)
@@ -57,34 +57,34 @@ MAINTAINERCLEANFILES = $(CROSS_OPTIONS_TEXI)
 
 # Fix Info's @code in @deftype
 # https://lists.gnu.org/archive/html/help-texinfo/2019-11/msg00004.html
-all: $(srcdir)/$(doc_bison).info.bak
-$(srcdir)/$(doc_bison).info.bak: $(srcdir)/$(doc_bison).info
+all-local: $(srcdir)/$(%C%_bison).info.bak
+$(srcdir)/$(%C%_bison).info.bak: $(srcdir)/$(%C%_bison).info
 	$(AM_V_GEN) $(PERL) -pi.bak -0777	\
 	  -e 's{(^ --.*\n(?: {10}.*\n)*)}'	\
 	  -e '{'				\
 	  -e '  $$def = $$1;'			\
 	  -e '  $$def =~ s/‘|’//g;'		\
 	  -e '  $$def;'				\
-	  -e '}gem;' $(srcdir)/$(doc_bison).info
+	  -e '}gem;' $(srcdir)/$(%C%_bison).info
 	@ touch $@
-EXTRA_DIST += $(srcdir)/$(doc_bison).info.bak
-MAINTAINERCLEANFILES += $(srcdir)/$(doc_bison).info.bak
+EXTRA_DIST += $(srcdir)/$(%C%_bison).info.bak
+MAINTAINERCLEANFILES += $(srcdir)/$(%C%_bison).info.bak
 
 
 ## ---------- ##
 ## Ref card.  ##
 ## ---------- ##
 
-EXTRA_DIST += doc/refcard.tex
-CLEANFILES += doc/refcard.pdf
+EXTRA_DIST += %D%/refcard.tex
+CLEANFILES += %D%/refcard.pdf
 
-doc/refcard.pdf: doc/refcard.tex
-	$(AM_V_GEN) cd doc && pdftex $(abs_top_srcdir)/doc/refcard.tex
+%D%/refcard.pdf: %D%/refcard.tex
+	$(AM_V_GEN) cd %D% && pdftex $(abs_top_srcdir)/%D%/refcard.tex
 
 
 
 ## ---------------- ##
-## doc/bison.help.  ##
+## %D%/bison.help.  ##
 ## ---------------- ##
 
 # Some of our targets (cross-options.texi, bison.1) use "bison --help".
@@ -106,18 +106,19 @@ doc/refcard.pdf: doc/refcard.tex
 # compilations of cross-options.texi and bison.1.  At the cost of
 # repeated builds of bison.help.
 
-EXTRA_DIST += $(top_srcdir)/doc/bison.help
+EXTRA_DIST += $(top_srcdir)/%D%/bison.help
 if ! CROSS_COMPILING
-MAINTAINERCLEANFILES += $(top_srcdir)/doc/bison.help
-$(top_srcdir)/doc/bison.help: src/bison$(EXEEXT)
-	$(AM_V_GEN)LC_ALL=C tests/bison --version >doc/bison.help.tmp
+MAINTAINERCLEANFILES += $(top_srcdir)/%D%/bison.help
+$(top_srcdir)/%D%/bison.help: src/bison$(EXEEXT)
+	$(AM_V_GEN)$(MKDIR_P) %D%
+	$(AM_V_at) LC_ALL=C tests/bison --version >%D%/bison.help.tmp
 	$(AM_V_at) LC_ALL=C tests/bison --help | \
 ## Avoid depending on the path to Bison.
 	  sed -e 's,^Usage: .*/bison \[OPTION\],Usage: bison [OPTION],g' \
 ## Avoid variations in the output depending on whether we are
 ## on a glibc system.
-	      -e '/translation bugs/d'  >>doc/bison.help.tmp
-	$(AM_V_at)$(top_srcdir)/build-aux/move-if-change doc/bison.help.tmp $@
+	      -e '/translation bugs/d'  >>%D%/bison.help.tmp
+	$(AM_V_at)$(top_srcdir)/build-aux/move-if-change %D%/bison.help.tmp $@
 endif ! CROSS_COMPILING
 
 
@@ -125,34 +126,48 @@ endif ! CROSS_COMPILING
 ## Man Pages.  ##
 ## ----------- ##
 
-dist_man_MANS = $(top_srcdir)/doc/bison.1
+dist_man_MANS = $(top_srcdir)/%D%/bison.1
 
 EXTRA_DIST += $(dist_man_MANS:.1=.x)
 MAINTAINERCLEANFILES += $(dist_man_MANS)
 
-# Differences to ignore when comparing the man page (the date).
+# Differences to ignore when comparing man pages (the date).
 remove_time_stamp = \
   sed 's/^\(\.TH[^"]*"[^"]*"[^"]*\)"[^"]*"/\1/'
 
 # Depend on configure to get version number changes.
 if ! CROSS_COMPILING
-MAN_DEPS = doc/bison.help doc/bison.x $(top_srcdir)/configure
+MAN_DEPS = %D%/bison.help %D%/bison.x $(top_srcdir)/configure
 endif
 
-$(top_srcdir)/doc/bison.1: $(MAN_DEPS)
-	$(AM_V_GEN)$(HELP2MAN)			\
-	    --include=$(top_srcdir)/doc/bison.x	\
-	    --output=$@.tmp tests/bison
-	$(AM_V_at)if $(remove_time_stamp) $@ >$@a.tmp 2>/dev/null &&		\
-	   $(remove_time_stamp) $@.tmp | cmp $@a.tmp - >/dev/null 2>&1; then	\
-	  touch $@;								\
-	else									\
-	  mv $@.tmp $@;								\
+# If we don't have help2man, just touch the target.  Maintainers must
+# have the real thing, so if there's a .git directory, fail hard.
+#
+# We shouldn't need this, but on some OS the timestamps in the tarball
+# leave us no choice.  See
+# https://lists.gnu.org/r/bug-bison/2020-05/msg00055.html.
+$(top_srcdir)/%D%/bison.1: $(MAN_DEPS)
+	$(AM_V_GEN)if $(HELP2MAN) --version >/dev/null 2>&1; then	\
+	  $(HELP2MAN)							\
+	    --include=$(top_srcdir)/%D%/bison.x				\
+	    --output=$@.tmp tests/bison &&				\
+	  { $(remove_time_stamp) $@     >$@a.tmp || true; } &&		\
+	    $(remove_time_stamp) $@.tmp >$@b.tmp &&			\
+	  if diff $@a.tmp $@b.tmp >/dev/null 2>&1; then			\
+	    touch $@;							\
+	  else								\
+	    mv $@.tmp $@;						\
+	  fi &&								\
+	  rm -f $@*.tmp;						\
+	elif test -d $(srcdir)/.git; then				\
+	  echo >&2 "ERROR: $@: help2man is needed";			\
+	  exit 1;							\
+	else								\
+	  touch $@;							\
 	fi
-	$(AM_V_at)rm -f $@*.tmp
 
 if ENABLE_YACC
-nodist_man_MANS = doc/yacc.1
+nodist_man_MANS = %D%/yacc.1
 endif
 
 ## ----------------------------- ##
@@ -161,8 +176,8 @@ endif
 
 CLEANFILES += $(FIGS_GV:.gv=.eps) $(FIGS_GV:.gv=.pdf) $(FIGS_GV:.gv=.svg)
 FIGS_GV =                                               \
-  doc/figs/example.gv                                   \
-  doc/figs/example-reduce.gv doc/figs/example-shift.gv
+  %D%/figs/example.gv                                   \
+  %D%/figs/example-reduce.gv %D%/figs/example-shift.gv
 EXTRA_DIST +=                                                   \
   $(FIGS_GV) $(FIGS_GV:.gv=.txt)                                \
   $(FIGS_GV:.gv=.eps) $(FIGS_GV:.gv=.pdf) $(FIGS_GV:.gv=.svg)
@@ -193,8 +208,8 @@ DOXYGEN = doxygen
 
 doc: html
 
-html-local: doc/Doxyfile
-	$(AM_V_GEN) $(DOXYGEN) doc/Doxyfile
+html-local: %D%/Doxyfile
+	$(AM_V_GEN) $(DOXYGEN) %D%/Doxyfile
 
 edit = sed -e 's,@PACKAGE_NAME\@,$(PACKAGE_NAME),g' \
 	   -e 's,@PACKAGE_VERSION\@,$(PACKAGE_VERSION),g' \
@@ -202,11 +217,11 @@ edit = sed -e 's,@PACKAGE_NAME\@,$(PACKAGE_NAME),g' \
 	   -e 's,@top_builddir\@,$(top_builddir),g' \
 	   -e 's,@top_srcdir\@,$(top_srcdir),g'
 
-EXTRA_DIST += doc/Doxyfile.in
-CLEANFILES += doc/Doxyfile
+EXTRA_DIST += %D%/Doxyfile.in
+CLEANFILES += %D%/Doxyfile
 # Sed is used to generate Doxyfile from Doxyfile.in instead of
 # configure, because the former is way faster than the latter.
-doc/Doxyfile: $(top_srcdir)/doc/Doxyfile.in
-	$(AM_V_GEN) $(edit) $(top_srcdir)/doc/Doxyfile.in >doc/Doxyfile
+%D%/Doxyfile: $(top_srcdir)/%D%/Doxyfile.in
+	$(AM_V_GEN) $(edit) $(top_srcdir)/%D%/Doxyfile.in >%D%/Doxyfile
 
-CLEANDIRS += doc/html
+CLEANDIRS += %D%/html

doc/sequence.y

@@ -0,0 +1,10 @@
+%%
+sequence:
+  %empty
+| maybeword
+| sequence "word"
+;
+maybeword:
+  %empty
+| "word"
+;

etc/README

@@ -1,38 +0,0 @@
-* bench.pl
-A primitive Perl script to run benches.  Currently its only bench is
-the usual calculator: it has a lightweight processing part (but not
-null), so it is really emphasizing the cost of the whole parsing
-(including scanning and I/O).
-
-Set the envvars BISON and CC to the programs and flags you want to
-use.  They default to bison and gcc.  To run a non-installed bison
-from a tarball built in _build, I suggest the following.
-
-     BISON=_build/tests/bison CC='gcc -O2' ./bench.pl
-
-(The _build/tests/bison wrapper makes sure that _build/src/bison will
-use the tarballs' skeletons, not those already installed as a
-straightforward use of _build/src/bison would.)
-
---
-
-Copyright (C) 2006, 2009-2015, 2018-2020 Free Software Foundation, Inc.
-
-This file is part of Bison, the GNU Compiler Compiler.
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-Local Variables:
-mode: outline
-End:

etc/README.md

@@ -0,0 +1,36 @@
+# bench.pl
+A primitive Perl script to run benches.  Currently its only bench is the
+usual calculator: it has a lightweight processing part (but not null), so it
+is really emphasizing the cost of the whole parsing (including scanning and
+I/O).
+
+Set the envvars `BISON` and `CC` to the programs and flags you want to use.
+They default to `bison` and `gcc`.  To run a non-installed bison from a
+tarball built in _build, I suggest the following.
+
+    BISON=_build/tests/bison CC='gcc -O2' ./bench.pl
+
+(The `_build/tests/bison` wrapper makes sure that `_build/src/bison` will
+use the tarballs' skeletons, not those already installed as a
+straightforward use of `_build/src/bison` would.)
+
+<!--
+
+Local Variables:
+mode: markdown
+fill-column: 76
+ispell-dictionary: "american"
+End:
+
+Copyright (C) 2006, 2009-2015, 2018-2020 Free Software Foundation, Inc.
+
+This file is part of Bison, the GNU Compiler Compiler.
+
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
+
+-->

etc/local.mk

@@ -14,4 +14,5 @@
 ## You should have received a copy of the GNU General Public License
 ## along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-nodist_noinst_SCRIPTS = etc/bench.pl
+dist_noinst_DATA = %D%/README.md
+nodist_noinst_SCRIPTS = %D%/bench.pl

examples/c++/README.md

@@ -8,13 +8,13 @@ afterwards.
 ## simple.yy - Simple example in C++14
 A very simple example in C++, based on variants and symbol constructors.
 Variants allow to use any C++ type as semantic value type, and symbol
-constructors ensure consistency between declared token type and effective
+constructors ensure consistency between declared token kind and effective
 semantic value.
 
 Run as `./simple`.
 
-Extracted from the documentation: "A Simple C++ Example".
-https://www.gnu.org/software/bison/manual/html_node/A-Simple-C_002b_002b-Example.html
+Extracted from the documentation: [A Simple C++
+Example](https://www.gnu.org/software/bison/manual/html_node/A-Simple-C_002b_002b-Example.html).
 
 ## variant.yy - Self-contained example in C++98
 A variation of simple.yy, in C++98.
@@ -36,8 +36,8 @@ starting point for a clean parser in C++.  The previous examples are better
 introductory examples, and the C examples are also useful introductory
 examples.
 
-Extracted from the documentation: "A Complete C++ Example".
-https://www.gnu.org/software/bison/manual/html_node/A-Complete-C_002b_002b-Example.html
+Extracted from the documentation: [A Complete C++
+Example](https://www.gnu.org/software/bison/manual/html_node/A-Complete-C_002b_002b-Example.html).
 
 <!---
 
@@ -55,5 +55,5 @@ Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
 Texts.  A copy of the license is included in the "GNU Free
 Documentation License" file as part of this distribution.
 
-# LocalWords:  mfcalc calc parsers yy ispell american
+LocalWords:  mfcalc calc parsers yy ispell american
 --->

examples/c++/calc++/README.md

@@ -5,9 +5,9 @@ saw the traditional implementation in C, please first read
 examples/c/lexcalc, which can be seen as a C precursor of this example.
 
 Read the corresponding chapter in the documentation: "A Complete C++
-Example".  It is also available on line (maybe with a different version of
-Bison):
-https://www.gnu.org/software/bison/manual/html_node/A-Complete-C_002b_002b-Example.html
+Example".  It is also available [on
+line](https://www.gnu.org/software/bison/manual/html_node/A-Complete-C_002b_002b-Example.html)
+(maybe with a different version of Bison).
 
 To use it, copy this directory into some work directory, and run `make` to
 compile the executable, and try it.  It is a simple calculator which accepts
@@ -38,20 +38,12 @@ End:
 
 Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
-This file is part of Bison, the GNU Compiler Compiler.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# LocalWords:  calc parsers yy MERCHANTABILITY Ctrl ispell american
+LocalWords:  calc parsers yy MERCHANTABILITY Ctrl ispell american
 --->

examples/c++/calc++/calc++.test

@@ -44,11 +44,18 @@ EOF
 run 0 9
 
 
+cat >input <<EOF
+1 +
+EOF
+run 1 'err: -:2.1: syntax error, unexpected end of file, expecting ( or identifier or number'
+
+
+# LAC finds many more tokens.
 cat >input <<EOF
 a := 1
 d := a + b * c
 EOF
-run 1 'err: -:3.1: syntax error, unexpected end of file, expecting ( or identifier or number'
+run 1 'err: -:3.1: syntax error, unexpected end of file'
 
 
 cat >input <<EOF

examples/c++/variant-11.yy

@@ -139,7 +139,7 @@ namespace yy
 int
 main (int argc, const char *argv[])
 {
-  if (2 <= argc && isdigit (*argv[1]))
+  if (2 <= argc && isdigit (static_cast<unsigned char> (*argv[1])))
     {
       auto maxl = strtol (argv[1], nullptr, 10);
       max = INT_MIN <= maxl && maxl <= INT_MAX ? int(maxl) : 4;

examples/c/README.md

@@ -10,8 +10,8 @@ The first example is that of a simple double-precision Reverse Polish
 Notation calculator (a calculator using postfix operators). This example
 provides a good starting point, since operator precedence is not an issue.
 
-Extracted from the documentation: "Reverse Polish Notation Calculator"
-https://www.gnu.org/software/bison/manual/html_node/RPN-Calc.html
+Extracted from the documentation: [Reverse Polish Notation
+Calculator](https://www.gnu.org/software/bison/manual/html_node/RPN-Calc.html).
 
 ## calc - Simple Calculator
 This example is slightly more complex than rpcalc: it features infix
@@ -24,19 +24,55 @@ A more complete C example: a multi-function calculator.  More complex than
 the previous example.  Using precedence directives to support infix
 operators.
 
-Extracted from the documentation: "Multi-Function Calculator: mfcalc".
-https://www.gnu.org/software/bison/manual/html_node/Multi_002dfunction-Calc.html
+Extracted from the documentation: [Multi-Function Calculator:
+mfcalc](https://www.gnu.org/software/bison/manual/html_node/Multi_002dfunction-Calc.html).
 
 ## lexcalc - calculator with Flex and Bison
-The calculator, redux.  This time using a scanner generated by Flex.
+The calculator with precedence directives and location tracking.  It uses
+Flex to generate the scanner.
 
 ## reccalc - recursive calculator with Flex and Bison
-The example builds on top of the previous one to provide a reentrant parser.
-Such parsers can be called concurrently in different threads, or even
-recursively.  To demonstrate this feature, expressions in parentheses are
-tokenized as strings, and then recursively parsed from the parser.  So
+This example builds on top of the previous one to provide a reentrant
+parser.  Such parsers can be called concurrently in different threads, or
+even recursively.  To demonstrate this feature, expressions in parentheses
+are tokenized as strings, and then recursively parsed from the parser.  So
 `(((1)+(2))*((3)+(4)))` uses eight parsers, with a depth of four.
 
+## pushcalc - calculator implemented with a push parser
+All the previous examples are so called "pull parsers": the user invokes the
+parser once, which repeatedly calls the scanner until the input is drained.
+
+This example demonstrates the "push parsers": the user calls the scanner to
+fetch the next token, passes it to the parser, and repeats the operation
+until the input is drained.
+
+This example is a straightforward conversion of the 'calc' example to the
+push-parser model.
+
+## bistromathic - all the bells and whistles
+This example demonstrates best practices when using Bison.
+- Its hand-written scanner tracks locations.
+- Its interface is pure.
+- It uses %params to pass user information to the parser and scanner.
+- Its scanner uses the `error` token to signal lexical errors and enter
+  error recovery.
+- Its interface is "incremental", well suited for interaction: it uses the
+  push-parser API to feed the parser with the incoming tokens.
+- It features an interactive command line with completion based on the
+  parser state, based on `yyexpected_tokens`.
+- It uses Bison's standard catalogue for internationalization of generated
+  messages.
+- It uses a custom syntax error with location, lookahead correction and
+  token internationalization.
+- Error messages quote the source with squiggles that underline the error:
+```
+> 123 456
+1.5-7: syntax error: expected end of file or + or - or * or / or ^ before number
+    1 | 123 456
+      |     ^~~
+```
+- It supports debug traces with semantic values.
+- It uses named references instead of the traditional $1, $2, etc.
 
 <!---
 
@@ -47,6 +83,8 @@ End:
 
 Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
+This file is part of GNU bison, the GNU Compiler Compiler.
+
 Permission is granted to copy, distribute and/or modify this document
 under the terms of the GNU Free Documentation License, Version 1.3 or
 any later version published by the Free Software Foundation; with no
@@ -54,6 +92,8 @@ Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
 Texts.  A copy of the license is included in the "GNU Free
 Documentation License" file as part of this distribution.
 
-# LocalWords:  mfcalc calc parsers yy rpcalc lexcalc redux reccalc ispell
-# LocalWords:  reentrant tokenized american postfix
---->
+LocalWords:  mfcalc calc parsers yy rpcalc lexcalc redux reccalc ispell
+LocalWords:  reentrant tokenized american postfix pushcalc bistromathic
+LocalWords:  lookahead
+
+-->

examples/c/bistromathic/Makefile

@@ -0,0 +1,40 @@
+# This Makefile is designed to be simple and readable.  It does not
+# aim at portability.  It requires GNU Make.
+
+BASE = bistromathic
+BISON = bison
+XSLTPROC = xsltproc
+
+# We need to find the headers and libs for readline (and possibly intl).
+# You probably need to customize this for your own environment.
+CPPFLAGS = -I/opt/local/include
+LDFLAGS = -L/opt/local/lib
+
+# Find the translation catalogue for Bison's generated messagess.
+BISON_LOCALEDIR = $(shell $(BISON) $(BISON_FLAGS) --print-localedir)
+CPPFLAGS += -DENABLE_NLS -DBISON_LOCALEDIR='"$(BISON_LOCALEDIR)"'
+
+LIBS = -lreadline -lm # In some environments, -lintl is needed.
+
+all: $(BASE)
+
+%.c %.h %.xml %.gv: %.y
+	$(BISON) $(BISONFLAGS) --defines --xml --graph=$*.gv -o $*.c $<
+
+$(BASE): parse.o
+	$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS)
+
+run: $(BASE)
+	@echo "Type bistromathic expressions.  Quit with ctrl-d."
+	./$<
+
+html: $(BASE).html
+%.html: %.xml
+	$(XSLTPROC) $(XSLTPROCFLAGS) -o $@ $$($(BISON) --print-datadir)/xslt/xml2xhtml.xsl $<
+
+CLEANFILES =						\
+  $(BASE) *.o						\
+  parse.[ch] parse.output parse.xml parse.html parse.gv
+
+clean:
+	rm -f $(CLEANFILES)

examples/c/bistromathic/README.md

@@ -0,0 +1,49 @@
+# bistromathic - all the bells and whistles
+This example demonstrates best practices when using Bison.
+- Its hand-written scanner tracks locations.
+- Its interface is pure.
+- It uses %params to pass user information to the parser and scanner.
+- Its scanner uses the `error` token to signal lexical errors and enter
+  error recovery.
+- Its interface is "incremental", well suited for interaction: it uses the
+  push-parser API to feed the parser with the incoming tokens.
+- It features an interactive command line with completion based on the
+  parser state, based on `yyexpected_tokens`.
+- It uses Bison's standard catalogue for internationalization of generated
+  messages.
+- It uses a custom syntax error with location, lookahead correction and
+  token internationalization.
+- Error messages quote the source with squiggles that underline the error:
+```
+> 123 456
+1.5-7: syntax error: expected end of file or + or - or * or / or ^ before number
+    1 | 123 456
+      |     ^~~
+```
+- It supports debug traces with semantic values.
+- It uses named references instead of the traditional $1, $2, etc.
+
+To customize the interaction with bistromathic, see the GNU Readline user
+manual (see `info rluserman`).
+
+<!---
+Local Variables:
+fill-column: 76
+ispell-dictionary: "american"
+End:
+
+Copyright (C) 2020 Free Software Foundation, Inc.
+
+This file is part of Bison, the GNU Compiler Compiler.
+
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
+
+LocalWords:  bistromathic yyexpected lookahead ispell american
+LocalWords:  MERCHANTABILITY
+
+--->

examples/c/bistromathic/bistromathic.test

@@ -0,0 +1,361 @@
+#! /bin/sh
+
+# Copyright (C) 2020 Free Software Foundation, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Users may customize the behavior of readline, which might break our
+# expected results.
+INPUTRC=/dev/null
+export INPUTRC
+
+# Beware of portability issues of readline when not feeding it from a
+# terminal.
+#
+# With recent versions of GNU Readline, input "1+2*3\n" gives
+# "> 1+2*3\n7\n> \n"
+#
+# macOS' version does not display the prompt and does not repeat stdin
+# on stdout, so input "1+2*3\n" gives "7\n" as output.  Let's try to
+# cope with this.
+#
+# On OpenBSD 6.5 the prompt is displayed, but the input is not
+# repeated (!).  So input "1+2*3\n" gives "> 7\n> \n" as output.
+#
+# On AIX, you get some escaping sequence before the prompt:
+# "<ESC>[?1034h> 1+2*3".  It appears to pass the terminfo capability
+# "smm", to put the terminal in "meta mode": as if the user had hit
+# META.
+
+echo >perfect '> 0
+0
+> '
+
+echo >ok '0'
+echo '0' | prog >effective
+
+echo "checking for readline output..."
+if diff perfect effective; then
+    # Alles ist gut.
+    strip_prompt=false
+elif diff ok effective; then
+    strip_prompt=true
+else
+    skip "this is not the GNU Readline we expect"
+fi
+
+
+cat >input <<EOF
+1+2*3
+EOF
+run 0 '> 1+2*3
+7
+> '
+
+cat >input <<EOF
+(1+2) * 3
+EOF
+run 0  '> (1+2) * 3
+9
+> '
+run -noerr 0 '> (1+2) * 3
+9
+> ' -p
+
+cat >input <<EOF
+a = 256
+sqrt (a)
+EOF
+run 0 '> a = 256
+256
+> sqrt (a)
+16
+> '
+
+cat >input <<EOF
+a = .16
+b = 10 ^ 2
+sqrt (a * b)
+EOF
+run 0 '> a = .16
+0.16
+> b = 10 ^ 2
+100
+> sqrt (a * b)
+4
+> '
+
+cat >input <<EOF
+*
+EOF
+run 0 '> *
+> ''
+err: 1.1: syntax error: expected end of file or - or ( or exit or number or function etc., before *
+err:     1 | *
+err:       | ^'
+
+# Underline long errors.
+cat >input <<EOF
+123 123456
+EOF
+run 0 '> 123 123456
+> ''
+err: 1.5-10: syntax error: expected end of file or + or - or * or / or ^ before number
+err:     1 | 123 123456
+err:       |     ^~~~~~'
+
+cat >input <<EOF
+1 + 2 * * 3
+EOF
+run 0 '> 1 + 2 * * 3
+> ''
+err: 1.9: syntax error: expected - or ( or number or function or variable before *
+err:     1 | 1 + 2 * * 3
+err:       |         ^'
+
+cat >input <<EOF
+1 / 0
+EOF
+run 0 '> 1 / 0
+> ''
+err: 1.1-5: error: division by zero'
+
+
+## ---------------- ##
+## Error recovery.  ##
+## ---------------- ##
+
+cat >input <<EOF
+((1 ++ 2) ** 3)
+(1 ++ 2) + (3 ** 4)
+EOF
+run 0 '> ((1 ++ 2) ** 3)
+666
+> (1 ++ 2) + (3 ** 4)
+1332
+> ''
+err: 1.6: syntax error: expected - or ( or number or function or variable before +
+err:     1 | ((1 ++ 2) ** 3)
+err:       |      ^
+err: 2.5: syntax error: expected - or ( or number or function or variable before +
+err:     2 | (1 ++ 2) + (3 ** 4)
+err:       |     ^
+err: 2.16: syntax error: expected - or ( or number or function or variable before *
+err:     2 | (1 ++ 2) + (3 ** 4)
+err:       |                ^'
+
+# The rule "( error )" should work even if there are no tokens between "(" and ")".
+cat >input <<EOF
+()
+EOF
+run 0 '> ()
+666
+> ''
+err: 1.2: syntax error: expected - or ( or number or function or variable before )
+err:     1 | ()
+err:       |  ^'
+
+
+cat >input <<EOF
+100% + 10
+EOF
+run 0 '> 100% + 10
+> ''
+err: 1.4: syntax error: invalid character: %'
+
+# Traces.  This allows to check the location of the error.  If we
+# forget to map YYerror to YYUNDEF, error recovery enters an endless
+# loop with this input.
+cat >input <<EOF
+(+_)
+EOF
+run 0 '> (+_)
+666
+> ''
+err: Starting parse
+err: Entering state 0
+err: Stack now 0
+err: Reading a token
+err: Next token is token ( (1.1: )
+err: Shifting token ( (1.1: )
+err: Entering state 2
+err: Stack now 0 2
+err: Return for a new token:
+err: Reading a token
+err: Next token is token + (1.2: )
+err: LAC: initial context established for +
+err: LAC: checking lookahead +: Err
+err: LAC: checking lookahead end of file: Err
+err: LAC: checking lookahead +: Err
+err: LAC: checking lookahead -: S1
+err: LAC: checking lookahead *: Err
+err: LAC: checking lookahead /: Err
+err: LAC: checking lookahead ^: Err
+err: LAC: checking lookahead (: S2
+err: LAC: checking lookahead ): Err
+err: LAC: checking lookahead =: Err
+err: LAC: checking lookahead exit: Err
+err: LAC: checking lookahead number: S4
+err: LAC: checking lookahead function: S5
+err: LAC: checking lookahead variable: S6
+err: LAC: checking lookahead NEG: Err
+err: 1.2: syntax error: expected - or ( or number or function or variable before +
+err:     1 | (+_)
+err:       |  ^
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Next token is token + (1.2: )
+err: LAC: initial context established for +
+err: LAC: checking lookahead +: Err
+err: Error: discarding token + (1.2: )
+err: Error: popping token error (1.2: )
+err: Stack now 0 2
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Return for a new token:
+err: 1.3: syntax error: invalid character: _
+err: Reading a token
+err: Error: popping token error (1.2: )
+err: Stack now 0 2
+err: Shifting token error (1.2-3: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Next token is token invalid token (1.3: )
+err: LAC: initial context established for invalid token
+err: LAC: checking lookahead invalid token: Always Err
+err: Error: discarding token invalid token (1.3: )
+err: Error: popping token error (1.2-3: )
+err: Stack now 0 2
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2-3: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Return for a new token:
+err: Reading a token
+err: Next token is token ) (1.4: )
+err: Shifting token ) (1.4: )
+err: Entering state 20
+err: Stack now 0 2 10 20
+err: Reducing stack by rule XX (line XXX):
+err:    $1 = token ( (1.1: )
+err:    $2 = token error (1.2-3: )
+err:    $3 = token ) (1.4: )
+err: -> $$ = nterm exp (1.1-4: 666)
+err: Entering state 8
+err: Stack now 0 8
+err: Return for a new token:
+err: Reading a token
+err: Now at end of input.
+err: LAC: initial context established for end of file
+err: LAC: checking lookahead end of file: R2 G7 S14
+err: Reducing stack by rule XX (line XXX):
+err:    $1 = nterm exp (1.1-4: 666)
+err: -> $$ = nterm input (1.1-4: )
+err: Entering state 7
+err: Stack now 0 7
+err: Now at end of input.
+err: Shifting token end of file (1.5: )
+err: LAC: initial context discarded due to shift
+err: Entering state 14
+err: Stack now 0 7 14
+err: Stack now 0 7 14
+err: Cleanup: popping token end of file (1.5: )
+err: Cleanup: popping nterm input (1.1-4: )' -p
+
+
+
+## ------------ ##
+## Completion.  ##
+## ------------ ##
+
+# From now on, the differences between versions of GNU Readline are
+# too painful to try to cope with.
+if $strip_prompt; then
+  echo "SKIP: this is not the GNU Readline we expect"
+  exit $status
+fi
+
+# On Windows10/MSYS2 the ^G coming from <tab> completion is not
+# emitted the same way
+# (https://lists.gnu.org/r/bug-bison/2020-05/msg00076.html).
+echo "checking for kernel name... $(uname -s)"
+case $(uname -s) in
+  (MSYS*)
+    echo "SKIP: this is Windows/MSYS"
+    exit $status
+    ;;
+esac
+
+
+# Check completion after an operator.
+sed -e 's/\\t/	/g' >input <<EOF
+(1+\t\t
+EOF
+run 0 '> (1+
+(       -       atan    cos     exp     ln      number  sin     sqrt
+> (1+
+> ''
+err: 1.4: syntax error: expected - or ( or number or function or variable before end of file
+err:     1 | (1+
+err:       |    ^'
+
+# Check the completion of a word.
+sed -e 's/\\t/	/g' >input <<EOF
+(at\t\t
+EOF
+run 0 '> (atan ( ''
+> ''
+err: 1.9: syntax error: expected - or ( or number or function or variable before end of file
+err:     1 | (atan ( ''
+err:       |         ^'
+
+# Check the completion at the very beginning.
+sed -e 's/\\t/	/g' >input <<EOF
+e\t\t
+EOF
+run -n 0 '> e
+end of file  exit         exp          ''
+> e
+0
+> ''
+err: '
+
+# Check that completion prints valid locations even when there is an error.
+sed -e 's/\\t/	/g' >input <<EOF
+1++\t
+EOF
+run -n 0 '> 1++ ''
+> ''
+err: 1.3: syntax error: expected - or ( or number or function or variable before +
+err:     1 | 1++ ''
+err:       |   ^
+'
+
+# And even when the error was recovered from.
+sed -e 's/\\t/	/g' >input <<EOF
+(1++2) + 3 +\t\t
+EOF
+run -n 0 '> (1++2) + 3 +  ''
+> ''
+err: 1.4: syntax error: expected - or ( or number or function or variable before +
+err:     1 | (1++2) + 3 +  ''
+err:       |    ^
+err: 1.15: syntax error: expected - or ( or number or function or variable before end of file
+err:     1 | (1++2) + 3 +  ''
+err:       |               ^
+'

examples/c/bistromathic/local.mk

@@ -0,0 +1,41 @@
+## Copyright (C) 2020 Free Software Foundation, Inc.
+##
+## This program is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+bistromathicdir = $(docdir)/%D%
+
+## --------------- ##
+## Bistromathics.  ##
+## --------------- ##
+
+%D%/parse.c: $(dependencies)
+
+if ENABLE_BISTROMATHIC
+  check_PROGRAMS += %D%/bistromathic
+  TESTS += %D%/bistromathic.test
+  nodist_%C%_bistromathic_SOURCES = %D%/parse.y
+
+  # Don't use gnulib's system headers.
+  %C%_bistromathic_CPPFLAGS =			\
+    -DBISON_LOCALEDIR='"$(localdir)"'		\
+    -DLOCALEDIR='"$(localdir)"'			\
+    -I$(top_srcdir)/%D% -I$(top_builddir)/%D%
+  %C%_bistromathic_CFLAGS = $(TEST_CFLAGS)
+  %C%_bistromathic_LDADD = -lm $(LIBREADLINE) $(LIBINTL)
+endif
+
+EXTRA_DIST += %D%/bistromathic.test
+dist_bistromathic_DATA = %D%/parse.y %D%/Makefile %D%/README.md
+CLEANFILES += %D%/parse.[ch] %D%/parse.output
+CLEANDIRS += %D%/*.dSYM

examples/c/bistromathic/parse.y

@@ -0,0 +1,670 @@
+/* Parser and scanner for bistromathic.   -*- C -*-
+
+   Copyright (C) 2019-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+%require "3.7"
+
+// Emitted on top of the implementation file.
+%code top {
+  #include <ctype.h>  // isdigit
+  #include <locale.h> // LC_ALL
+  #include <math.h>   // cos, sin, etc.
+  #include <stdarg.h> // va_start
+  #include <stdio.h>  // printf
+  #include <stdlib.h> // calloc
+  #include <string.h> // strcmp
+
+  #include <readline/readline.h>
+  #include <readline/history.h>
+
+  #if defined ENABLE_NLS && ENABLE_NLS
+  // Unable the translation of Bison's generated messages.
+  # define YYENABLE_NLS 1
+  # include <libintl.h>
+  // Unless specified otherwise, we expect bistromathic's own
+  // catalogue to be installed in the same tree as Bison's catalogue.
+  # ifndef LOCALEDIR
+  #  define LOCALEDIR BISON_LOCALEDIR
+  # endif
+  #endif
+}
+
+// Emitted in the header file, before the definition of YYSTYPE.
+%code requires {
+  // Function type.
+  typedef double (func_t) (double);
+
+  // Data type for links in the chain of symbols.
+  typedef struct symrec symrec;
+  struct symrec
+  {
+    char *name;  // name of symbol
+    int type;    // type of symbol: either VAR or FUN
+    union
+    {
+      double var;    // value of a VAR
+      func_t *fun;   // value of a FUN
+    } value;
+    symrec *next;  // link field
+  };
+
+  symrec *putsym (char const *name, int sym_type);
+  symrec *getsym (char const *name);
+
+  // Exchanging information with the parser.
+  typedef struct
+  {
+    // Whether to not emit error messages.
+    int silent;
+    // The current input line.
+    const char *line;
+  } user_context;
+}
+
+// Emitted in the header file, after the definition of YYSTYPE.
+%code provides {
+# ifndef __attribute__
+#  ifndef __GNUC__
+#   define __attribute__(Spec) /* empty */
+#  endif
+# endif
+
+  yytoken_kind_t
+  yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc,
+         const user_context *uctx);
+  void yyerror (YYLTYPE *loc, const user_context *uctx,
+                char const *format, ...)
+    __attribute__ ((__format__ (__printf__, 3, 4)));
+}
+
+// Emitted in the implementation file.
+%code {
+  #if defined ENABLE_NLS && ENABLE_NLS
+  # define _(Msgid)  gettext (Msgid)
+  #else
+  # define _(Msgid)  (Msgid)
+  #endif
+
+  // Whether to quit.
+  int done = 0;
+}
+
+// Include the header in the implementation rather than duplicating it.
+%define api.header.include {"parse.h"}
+
+// Don't share global variables between the scanner and the parser.
+%define api.pure full
+
+// Generate a push parser.
+%define api.push-pull push
+
+// To avoid name clashes (e.g., with C's EOF) prefix token definitions
+// with TOK_ (e.g., TOK_EOF).
+%define api.token.prefix {TOK_}
+
+// Customized syntax error messages (see yyreport_syntax_error).
+%define parse.error custom
+
+// with locations.
+%locations
+
+// and accurate list of expected tokens.
+%define parse.lac full
+
+// Generate the parser description file (calc.output).
+%verbose
+
+// User information exchanged with the parser and scanner.
+%param {const user_context *uctx}
+
+// Generate YYSTYPE from the types assigned to symbols.
+%define api.value.type union
+%token
+    PLUS   "+"
+    MINUS  "-"
+    STAR   "*"
+    SLASH  "/"
+    CARET  "^"
+    LPAREN "("
+    RPAREN ")"
+    EQUAL  "="
+    EXIT   "exit"
+  <double>
+    NUM _("number")
+  <symrec*>
+    FUN _("function")
+    VAR _("variable")
+
+%nterm <double>  exp
+
+// Enable run-time traces (yydebug).
+%define parse.trace
+
+// Formatting semantic values in debug traces.
+%printer { fprintf (yyo, "%s", $$->name); } VAR;
+%printer { fprintf (yyo, "%s()", $$->name); } FUN;
+%printer { fprintf (yyo, "%g", $$); } <double>;
+
+
+// Precedence (from lowest to highest) and associativity.
+%precedence "="
+%left "+" "-"
+%left "*" "/"
+%precedence NEG // negation--unary minus
+%right "^"      // exponentiation
+
+%% // The grammar follows.
+input:
+  %empty
+| exp     { printf ("%.10g\n", $exp); }
+| "exit"  { done = 1; }
+;
+
+exp:
+  NUM
+| VAR               { $$ = $VAR->value.var; }
+| VAR "=" exp       { $$ = $3; $VAR->value.var = $3; }
+| FUN "(" exp ")"   { $$ = $FUN->value.fun ($3); }
+| exp[l] "+" exp[r] { $$ = $l + $r; }
+| exp[l] "-" exp[r] { $$ = $l - $r; }
+| exp[l] "*" exp[r] { $$ = $l * $r; }
+| exp[l] "/" exp[r]
+  {
+    if ($r == 0)
+      {
+        yyerror (&@$, uctx, "error: division by zero");
+        YYERROR;
+      }
+    else
+      $$ = $l / $r;
+  }
+| "-" exp  %prec NEG { $$ = -$2; }
+| exp[l] "^" exp[r]  { $$ = pow ($l, $r); }
+| "(" exp ")"        { $$ = $2; }
+| "(" error ")"      { $$ = 666; }
+;
+
+// End of grammar.
+%%
+
+/*------------.
+| Functions.  |
+`------------*/
+
+struct init
+{
+  char const *name;
+  func_t *fun;
+};
+
+static struct init const funs[] =
+{
+  { "atan", atan },
+  { "cos",  cos  },
+  { "exp",  exp  },
+  { "ln",   log  },
+  { "sin",  sin  },
+  { "sqrt", sqrt },
+  { 0, 0 },
+};
+
+// The symbol table: a chain of 'struct symrec'.
+static symrec *sym_table;
+
+// Put functions in table.
+static void
+init_table (void)
+{
+  for (int i = 0; funs[i].name; i++)
+    {
+      symrec *ptr = putsym (funs[i].name, TOK_FUN);
+      ptr->value.fun = funs[i].fun;
+    }
+}
+
+symrec *
+putsym (char const *name, int sym_type)
+{
+  symrec *res = (symrec *) malloc (sizeof (symrec));
+  res->name = strdup (name);
+  res->type = sym_type;
+  res->value.var = 0; // Set value to 0 even if fun.
+  res->next = sym_table;
+  sym_table = res;
+  return res;
+}
+
+symrec *
+getsym (char const *name)
+{
+  for (symrec *p = sym_table; p; p = p->next)
+    if (strcmp (p->name, name) == 0)
+      return p;
+  return NULL;
+}
+
+// How many symbols are registered.
+static int
+symbol_count (void)
+{
+  int res = 0;
+  for (symrec *p = sym_table; p; p = p->next)
+    ++res;
+  return res;
+}
+
+
+/*----------.
+| Scanner.  |
+`----------*/
+
+yytoken_kind_t
+yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc,
+       const user_context *uctx)
+{
+  int c;
+
+  // Ignore white space, get first nonwhite character.
+  do {
+    // Move the first position onto the last.
+    yylloc->first_line = yylloc->last_line;
+    yylloc->first_column = yylloc->last_column;
+
+    yylloc->last_column += 1;
+    c = *((*line)++);
+  } while (c == ' ' || c == '\t');
+
+  switch (c)
+    {
+    case '+': return TOK_PLUS;
+    case '-': return TOK_MINUS;
+    case '*': return TOK_STAR;
+    case '/': return TOK_SLASH;
+    case '^': return TOK_CARET;
+    case '=': return TOK_EQUAL;
+    case '(': return TOK_LPAREN;
+    case ')': return TOK_RPAREN;
+
+    case '!': return TOK_YYUNDEF;
+
+    case '\0': return TOK_YYEOF;
+
+      // Numbers.
+    case '.':
+    case '0': case '1': case '2': case '3': case '4':
+    case '5': case '6': case '7': case '8': case '9':
+      {
+        int nchars = 0;
+        if (sscanf (*line - 1, "%lf%n", &yylval->TOK_NUM, &nchars) != 1)
+          abort ();
+        *line += nchars - 1;
+        yylloc->last_column += nchars - 1;
+        return TOK_NUM;
+      }
+
+      // Identifiers.
+    case 'a': case 'b': case 'c': case 'd': case 'e':
+    case 'f': case 'g': case 'h': case 'i': case 'j':
+    case 'k': case 'l': case 'm': case 'n': case 'o':
+    case 'p': case 'q': case 'r': case 's': case 't':
+    case 'u': case 'v': case 'w': case 'x': case 'y':
+    case 'z':
+      {
+        int nchars = 0;
+        char buf[100];
+        if (sscanf (*line - 1, "%99[a-z]%n", buf, &nchars) != 1)
+          abort ();
+        *line += nchars - 1;
+        yylloc->last_column += nchars - 1;
+        if (strcmp (buf, "exit") == 0)
+          return TOK_EXIT;
+        else
+          {
+            symrec *s = getsym (buf);
+            if (!s)
+              s = putsym (buf, TOK_VAR);
+            yylval->TOK_VAR = s;
+            return s->type;
+          }
+      }
+
+      // Stray characters.
+    default:
+      yyerror (yylloc, uctx, "syntax error: invalid character: %c", c);
+      return TOK_YYerror;
+    }
+}
+
+
+/*---------.
+| Parser.  |
+`---------*/
+
+
+static const char *
+error_format_string (int argc)
+{
+  switch (argc)
+    {
+    default: // Avoid compiler warnings.
+    case 0: return _("%@: syntax error");
+    case 1: return _("%@: syntax error: unexpected %u");
+      // TRANSLATORS: '%@' is a location in a file, '%u' is an
+      // "unexpected token", and '%0e', '%1e'... are expected tokens
+      // at this point.
+      //
+      // For instance on the expression "1 + * 2", you'd get
+      //
+      // 1.5: syntax error: expected - or ( or number or function or variable before *
+    case 2: return _("%@: syntax error: expected %0e before %u");
+    case 3: return _("%@: syntax error: expected %0e or %1e before %u");
+    case 4: return _("%@: syntax error: expected %0e or %1e or %2e before %u");
+    case 5: return _("%@: syntax error: expected %0e or %1e or %2e or %3e before %u");
+    case 6: return _("%@: syntax error: expected %0e or %1e or %2e or %3e or %4e before %u");
+    case 7: return _("%@: syntax error: expected %0e or %1e or %2e or %3e or %4e or %5e before %u");
+    case 8: return _("%@: syntax error: expected %0e or %1e or %2e or %3e or %4e or %5e etc., before %u");
+    }
+}
+
+
+int
+yyreport_syntax_error (const yypcontext_t *ctx, const user_context *uctx)
+{
+  if (uctx->silent)
+    return 0;
+
+  enum { ARGS_MAX = 6 };
+  yysymbol_kind_t arg[ARGS_MAX];
+  int argsize = yypcontext_expected_tokens (ctx, arg, ARGS_MAX);
+  if (argsize < 0)
+    return argsize;
+  const int too_many_expected_tokens = argsize == 0 && arg[0] != YYSYMBOL_YYEMPTY;
+  if (too_many_expected_tokens)
+    argsize = ARGS_MAX;
+  const char *format = error_format_string (1 + argsize + too_many_expected_tokens);
+
+  const YYLTYPE *loc = yypcontext_location (ctx);
+  while (*format)
+    // %@: location.
+    if (format[0] == '%' && format[1] == '@')
+      {
+        YY_LOCATION_PRINT (stderr, *loc);
+        format += 2;
+      }
+    // %u: unexpected token.
+    else if (format[0] == '%' && format[1] == 'u')
+      {
+        fputs (yysymbol_name (yypcontext_token (ctx)), stderr);
+        format += 2;
+      }
+    // %0e, %1e...: expected token.
+    else if (format[0] == '%'
+             && isdigit ((unsigned char) format[1])
+             && format[2] == 'e'
+             && (format[1] - '0') < argsize)
+      {
+        int i = format[1] - '0';
+        fputs (yysymbol_name (arg[i]), stderr);
+        format += 3;
+      }
+    else
+      {
+        fputc (*format, stderr);
+        ++format;
+      }
+  fputc ('\n', stderr);
+
+  // Quote the source line.
+  {
+    fprintf (stderr, "%5d | %s\n", loc->first_line, uctx->line);
+    fprintf (stderr, "%5s | %*s", "", loc->first_column, "^");
+    for (int i = loc->last_column - loc->first_column - 1; 0 < i; --i)
+      putc ('~', stderr);
+    putc ('\n', stderr);
+  }
+  return 0;
+}
+
+
+// Called by yyparse on error.
+void yyerror (YYLTYPE *loc, const user_context *uctx, char const *format, ...)
+{
+  if (uctx->silent)
+    return;
+
+  YY_LOCATION_PRINT (stderr, *loc);
+  fputs (": ", stderr);
+  va_list args;
+  va_start (args, format);
+  vfprintf (stderr, format, args);
+  va_end (args);
+  putc ('\n', stderr);
+}
+
+
+// Return a newly allocated copy of at most N bytes of STRING.  In
+// other words, return a copy of the initial segment of length N of
+// STRING.
+static char *
+xstrndup (const char *string, size_t n)
+{
+  // len = strnlen (string, n), portably.
+  const char *end = memchr (string, '\0', n);
+  size_t len = end ? (size_t) (end - string) : n;
+  char *new = malloc (len + 1);
+  if (!new)
+    abort ();
+  new[len] = '\0';
+  return memcpy (new, string, len);
+}
+
+
+/*-----------.
+| Readline.  |
+`-----------*/
+
+// Parse (and execute) this line.
+static int
+process_line (YYLTYPE *lloc, const char *line)
+{
+  user_context uctx = {0, line};
+  yypstate *ps = yypstate_new ();
+  int status = 0;
+  do {
+    YYSTYPE lval;
+    yytoken_kind_t token = yylex (&line, &lval, lloc, &uctx);
+    status = yypush_parse (ps, token, &lval, lloc, &uctx);
+  } while (status == YYPUSH_MORE);
+  yypstate_delete (ps);
+  lloc->last_line++;
+  lloc->last_column = 1;
+  return status;
+}
+
+// Get the list of possible tokens after INPUT was read.
+// Returns a nonnegative.
+static int
+expected_tokens (const char *input,
+                 int *tokens, int ntokens)
+{
+  YYDPRINTF ((stderr, "expected_tokens (\"%s\")", input));
+  user_context uctx = {1, input};
+
+  // Parse the current state of the line.
+  yypstate *ps = yypstate_new ();
+  int status = 0;
+  YYLTYPE lloc = { 1, 1, 1, 1 };
+  do {
+    YYSTYPE lval;
+    yytoken_kind_t token = yylex (&input, &lval, &lloc, &uctx);
+    // Don't let the parse know when we reach the end of input.
+    if (token == TOK_YYEOF)
+      break;
+    status = yypush_parse (ps, token, &lval, &lloc, &uctx);
+  } while (status == YYPUSH_MORE);
+
+  int res = 0;
+  // If there were parse errors, don't propose completions.
+  if (!ps->yynerrs)
+    {
+      // Then query for the accepted tokens at this point.
+      res = yypstate_expected_tokens (ps, tokens, ntokens);
+      if (res < 0)
+        abort ();
+    }
+  yypstate_delete (ps);
+  return res;
+}
+
+// Attempt to complete on the contents of TEXT.  START and END bound
+// the region of rl_line_buffer that contains the word to complete.
+// TEXT is the word to complete.  We can use the entire contents of
+// rl_line_buffer in case we want to do some simple parsing.  Return
+// the array of matches, or NULL if there aren't any.
+static char **
+completion (const char *text, int start, int end)
+{
+  YYDPRINTF ((stderr, "completion (\"%.*s[%.*s]%s\")\n",
+              start, rl_line_buffer,
+              end - start, rl_line_buffer + start,
+              rl_line_buffer + end));
+
+  // Get list of token numbers.
+  int tokens[YYNTOKENS];
+  char *line = xstrndup (rl_line_buffer, (size_t) start);
+  int ntokens = expected_tokens (line, tokens, YYNTOKENS);
+  free (line);
+
+  // Build MATCHES, the list of possible completions.
+  const size_t len = strlen (text);
+  // Need initial prefix and final NULL.
+  char **matches
+    = calloc ((size_t) ntokens + (size_t) symbol_count () + 2, sizeof *matches);
+  if (!matches)
+    abort ();
+  int match = 1;
+  for (int i = 0; i < ntokens; ++i)
+    switch (tokens[i])
+      {
+      case YYSYMBOL_FUN:
+        for (symrec *s = sym_table; s; s = s->next)
+          if (s->type == TOK_FUN && strncmp (text, s->name, len) == 0)
+            matches[match++] = strdup (s->name);
+        break;
+      case YYSYMBOL_VAR:
+        for (symrec *s = sym_table; s; s = s->next)
+          if (s->type == TOK_VAR && strncmp (text, s->name, len) == 0)
+            matches[match++] = strdup (s->name);
+        break;
+      default:
+        {
+          const char* token = yysymbol_name (tokens[i]);
+          if (strncmp (text, token, len) == 0)
+            matches[match++] = strdup (token);
+          break;
+        }
+      }
+
+  // Find the longest common prefix, and install it in matches[0], as
+  // required by readline.
+  if (match == 1)
+    matches[0] = strdup (text);
+  else
+    {
+      size_t lcplen = strlen (matches[1]);
+      for (int i = 2; i < match && lcplen; ++i)
+        for (size_t j = 0; j < lcplen; ++j)
+          if (matches[1][j] != matches[i][j])
+            lcplen = j;
+      matches[0] = xstrndup (matches[1], lcplen);
+    }
+
+  if (yydebug)
+    {
+      fprintf (stderr, "completion (\"%.*s[%.*s]%s\") = ",
+               start, rl_line_buffer,
+               end - start, rl_line_buffer + start,
+               rl_line_buffer + end);
+      for (int i = 1; matches[i]; ++i)
+        fprintf (stderr, "%s%s",
+                 i == 1 ? "{" : ", ",
+                 matches[i]);
+      fprintf (stderr, "}\n");
+    }
+
+  // Don't fall back to proposing file names.
+  rl_attempted_completion_over = 1;
+  return matches;
+}
+
+static void
+init_readline (void)
+{
+  // Allow conditional parsing of the ~/.inputrc file.
+  rl_readline_name = "bistromathic";
+
+  // Tell the completer that we want a crack first.
+  rl_attempted_completion_function = completion;
+
+  // The basic list of characters that signal a break between words
+  // for the completer routine.
+  rl_basic_word_break_characters = " \t\n\"\\'`@$><=;|&{(+-*/^)";
+}
+
+
+
+/*-------.
+| Main.  |
+`-------*/
+
+int
+main (int argc, char const* argv[])
+{
+#if defined ENABLE_NLS && ENABLE_NLS
+  // Set up internationalization.
+  setlocale (LC_ALL, "");
+  // Use Bison's standard translation catalogue for error messages
+  // (the generated messages).
+  bindtextdomain ("bison-runtime", BISON_LOCALEDIR);
+  // The translation catalogue of bistromathic is actually included in
+  // Bison's.  In your own project, use the name of your project.
+  bindtextdomain ("bison", LOCALEDIR);
+  textdomain ("bison");
+#endif
+
+  // Enable parse traces on option -p.
+  if (1 < argc && strcmp (argv[1], "-p") == 0)
+    yydebug = 1;
+  init_table ();
+  init_readline ();
+  YYLTYPE lloc = {1, 1, 1, 1};
+  while (!done)
+    {
+      char *line = readline ("> ");
+      if (!line)
+        {
+          // Finish the line started by the prompt.
+          putchar ('\n');
+          break;
+        }
+      if (*line)
+        add_history (line);
+      process_line (&lloc, line);
+      free (line);
+    }
+}

examples/c/calc/README.md

@@ -13,16 +13,11 @@ Copyright (C) 2019-2020 Free Software Foundation, Inc.
 
 This file is part of Bison, the GNU Compiler Compiler.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
 --->

examples/c/calc/calc.test

@@ -34,4 +34,10 @@ cat >input <<EOF
 (1+2) * 3
 EOF
 run 0 9
+# Check the traces.
 run -noerr 0 9 -p
+
+cat >input <<EOF
+1++2
+EOF
+run 0 "err: syntax error, unexpected '+', expecting number or '('"

examples/c/calc/calc.y

@@ -1,6 +1,8 @@
 %code top {
+  #include <assert.h>
   #include <ctype.h>  /* isdigit. */
-  #include <stdio.h>  /* For printf, etc. */
+  #include <stdio.h>  /* printf. */
+  #include <stdlib.h> /* abort. */
   #include <string.h> /* strcmp. */
 
   int yylex (void);
@@ -8,16 +10,22 @@
 }
 
 %define api.header.include {"calc.h"}
-%define api.value.type union /* Generate YYSTYPE from these types:  */
+
+/* Generate YYSTYPE from the types used in %token and %type.  */
+%define api.value.type union
 %token <double> NUM "number"
 %type  <double> expr term fact
 
-/* Generate the parser description file.  */
+/* Generate the parser description file (calc.output).  */
 %verbose
+
+/* Nice error messages with details. */
+%define parse.error detailed
+
 /* Enable run-time traces (yydebug).  */
 %define parse.trace
 
-/* Formatting semantic values.  */
+/* Formatting semantic values in debug traces.  */
 %printer { fprintf (yyo, "%g", $$); } <double>;
 
 %% /* The grammar follows.  */
@@ -67,7 +75,8 @@ yylex (void)
   if (c == '.' || isdigit (c))
     {
       ungetc (c, stdin);
-      scanf ("%lf", &yylval.NUM);
+      if (scanf ("%lf", &yylval.NUM) != 1)
+        abort ();
       return NUM;
     }
 

examples/c/calc/local.mk

@@ -27,6 +27,7 @@ nodist_%C%_calc_SOURCES = %D%/calc.y
 
 # Don't use gnulib's system headers.
 %C%_calc_CPPFLAGS = -I$(top_srcdir)/%D% -I$(top_builddir)/%D%
+%C%_calc_CFLAGS = $(TEST_CFLAGS)
 
 dist_calc_DATA = %D%/calc.y %D%/Makefile %D%/README.md
 CLEANFILES += %D%/calc.[ch] %D%/calc.output %D%/scan.c

examples/c/lexcalc/README.md

@@ -3,6 +3,8 @@
 This directory contains lexcalc, the traditional example of using Flex and
 Bison to build a simple calculator.
 
+It features detailed syntax errors with locations.
+
 <!---
 Local Variables:
 fill-column: 76
@@ -13,16 +15,10 @@ Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
 This file is part of Bison, the GNU Compiler Compiler.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 --->

examples/c/lexcalc/lexcalc.test

@@ -29,4 +29,9 @@ run -noerr 0 9 -p
 cat >input <<EOF
 (1+2) *
 EOF
-run 1 'err: syntax error, unexpected end-of-line, expecting ( or number'
+run 1 'err: 1.8-2.0: syntax error, unexpected end of line, expecting ( or number'
+
+cat >input <<EOF
+1 / (2 - 2)
+EOF
+run 1 'err: 1.1-11: error: division by zero'

examples/c/lexcalc/local.mk

@@ -25,6 +25,8 @@ if FLEX_WORKS
   nodist_%C%_lexcalc_SOURCES = %D%/parse.y %D%/parse.h %D%/scan.l
   # Don't use gnulib's system headers.
   %C%_lexcalc_CPPFLAGS = -I$(top_srcdir)/%D% -I$(top_builddir)/%D%
+  # Fighting warnings triggered by Flex is just too painful.
+  # %C%_lexcalc_CFLAGS = $(TEST_CFLAGS)
 endif FLEX_WORKS
 
 %D%/parse.c: $(dependencies)

examples/c/lexcalc/parse.y

@@ -1,3 +1,22 @@
+/* Parser for lexcalc.   -*- C -*-
+
+   Copyright (C) 2018-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
 // Prologue (directives).
 %expect 0
 
@@ -6,10 +25,10 @@
 {
   // Tell Flex the expected prototype of yylex.
 #define YY_DECL                                 \
-  enum yytokentype yylex (YYSTYPE* yylval, int *nerrs)
+  yytoken_kind_t yylex (YYSTYPE* yylval, YYLTYPE *yylloc, int *nerrs)
   YY_DECL;
 
-  void yyerror (int *nerrs, const char *msg);
+  void yyerror (YYLTYPE *loc, int *nerrs, const char *msg);
 }
 
 // Emitted on top of the implementation file.
@@ -19,12 +38,30 @@
 #include <stdlib.h> // getenv.
 }
 
+// Include the header in the implementation rather than duplicating it.
+%define api.header.include {"parse.h"}
+
+// Don't share global variables between the scanner and the parser.
 %define api.pure full
+
+// To avoid name clashes (e.g., with C's EOF) prefix token definitions
+// with TOK_ (e.g., TOK_EOF).
 %define api.token.prefix {TOK_}
+
+// %token and %type use genuine types (e.g., "%token <int>").  Let
+// %bison define YYSTYPE as a union of all these types.
 %define api.value.type union
-%define parse.error verbose
+
+// Generate detailed error messages.
+%define parse.error detailed
+
+// with locations.
+%locations
+
+// Enable debug traces (see yydebug in main).
 %define parse.trace
- // Error count, exchanged between main, yyparse and yylex.
+
+// Error count, exchanged between main, yyparse and yylex.
 %param {int *nerrs}
 
 %token
@@ -34,8 +71,7 @@
   SLASH  "/"
   LPAREN "("
   RPAREN ")"
-  EOL    "end-of-line"
-  EOF 0  "end-of-file"
+  EOL    "end of line"
 ;
 
 %token <int> NUM "number"
@@ -66,7 +102,7 @@ exp:
   {
     if ($3 == 0)
       {
-        yyerror (nerrs, "invalid division by zero");
+        yyerror (&@$, nerrs, "error: division by zero");
         YYERROR;
       }
     else
@@ -77,9 +113,11 @@ exp:
 ;
 %%
 // Epilogue (C code).
-void yyerror (int *nerrs, const char *msg)
+
+void yyerror (YYLTYPE *loc, int *nerrs, const char *msg)
 {
-  fprintf (stderr, "%s\n", msg);
+  YY_LOCATION_PRINT (stderr, *loc);
+  fprintf (stderr, ": %s\n", msg);
   ++*nerrs;
 }
 

examples/c/lexcalc/scan.l

@@ -1,17 +1,51 @@
-/* Prologue (directives).   -*- C++ -*- */
+/* Scanner for lexcalc.   -*- C -*-
+
+   Copyright (C) 2018-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* Prologue (directives). */
 
 /* Disable Flex features we don't need, to avoid warnings. */
 %option nodefault noinput nounput noyywrap
 
 %{
-#include <errno.h> /* errno, ERANGE */
+#include <errno.h>  /* errno, ERANGE */
 #include <limits.h> /* INT_MIN */
 #include <stdlib.h> /* strtol */
 
 #include "parse.h"
+
+  // Each time a rule is matched, advance the end cursor/position.
+#define YY_USER_ACTION                          \
+  yylloc->last_column += (int) yyleng;
+
+  // Move the first position onto the last.
+#define LOCATION_STEP()                         \
+  do {                                          \
+    yylloc->first_line = yylloc->last_line;     \
+    yylloc->first_column = yylloc->last_column; \
+  } while (0)
 %}
 
 %%
+%{
+  // Each time yylex is called, move the head position to the end one.
+  LOCATION_STEP ();
+%}
  /* Rules.  */
 
 "+"      return TOK_PLUS;
@@ -27,18 +61,18 @@
   errno = 0;
   long n = strtol (yytext, NULL, 10);
   if (! (INT_MIN <= n && n <= INT_MAX && errno != ERANGE))
-    yyerror (nerrs, "integer is out of range");
+    yyerror (yylloc, nerrs, "integer is out of range");
   yylval->TOK_NUM = (int) n;
   return TOK_NUM;
 }
 
+"\n"     yylloc->last_line++; yylloc->last_column = 1; return TOK_EOL;
+
  /* Ignore white spaces. */
-[ \t]+   continue;
+[ \t]+   LOCATION_STEP (); continue;
 
-"\n"     return TOK_EOL;
+.        yyerror (yylloc, nerrs, "syntax error, invalid character"); continue;
 
-.        yyerror (nerrs, "syntax error, invalid character");
-
-<<EOF>>  return TOK_EOF;
+<<EOF>>  return TOK_YYEOF;
 %%
 /* Epilogue (C code). */

examples/c/local.mk

@@ -16,8 +16,10 @@
 cdir = $(docdir)/%D%
 dist_c_DATA = %D%/README.md
 
+include %D%/bistromathic/local.mk
 include %D%/calc/local.mk
 include %D%/lexcalc/local.mk
 include %D%/mfcalc/local.mk
+include %D%/pushcalc/local.mk
 include %D%/reccalc/local.mk
 include %D%/rpcalc/local.mk

examples/c/mfcalc/local.mk

@@ -31,6 +31,7 @@ nodist_%C%_mfcalc_SOURCES = $(mfcalc_sources)
 %D%/mfcalc.c: $(dependencies)
 # Don't use gnulib's system headers.
 %C%_mfcalc_CPPFLAGS = -I$(top_srcdir)/%D% -I$(top_builddir)/%D%
+%C%_mfcalc_CFLAGS = $(TEST_CFLAGS)
 %C%_mfcalc_LDADD = -lm
 
 dist_TESTS += %D%/mfcalc.test

examples/c/mfcalc/mfcalc.test

@@ -25,3 +25,15 @@ cat >input <<EOF
 EOF
 run 0 9
 run -noerr 0 9 -p
+
+cat >input <<EOF
+a = 256
+sqrt (a)
+EOF
+run 0 '256
+16'
+
+cat >input <<EOF
+16 == 12
+EOF
+run 0 'err: syntax error'

examples/c/pushcalc/Makefile

@@ -0,0 +1,28 @@
+# This Makefile is designed to be simple and readable.  It does not
+# aim at portability.  It requires GNU Make.
+
+BASE = calc
+BISON = bison
+XSLTPROC = xsltproc
+
+all: $(BASE)
+
+%.c %.h %.xml %.gv: %.y
+	$(BISON) $(BISONFLAGS) --defines --xml --graph=$*.gv -o $*.c $<
+
+$(BASE): $(BASE).o
+	$(CC) $(CFLAGS) -o $@ $^
+
+run: $(BASE)
+	@echo "Type arithmetic expressions.  Quit with ctrl-d."
+	./$<
+
+html: $(BASE).html
+%.html: %.xml
+	$(XSLTPROC) $(XSLTPROCFLAGS) -o $@ $$($(BISON) --print-datadir)/xslt/xml2xhtml.xsl $<
+
+CLEANFILES =									\
+  $(BASE) *.o $(BASE).[ch] $(BASE).output $(BASE).xml $(BASE).html $(BASE).gv
+
+clean:
+	rm -f $(CLEANFILES)

examples/c/pushcalc/README.md

@@ -0,0 +1,32 @@
+# pushcalc - push parser with Bison
+
+This directory contains pushcalc, the traditional calculator, implemented as
+a push parser.
+
+Traditionally Bison is used to create so called "pull parsers": the user
+invokes the parser once, which repeatedly calls (pulls) the scanner until
+the input is drained.
+
+This example demonstrates the "push parsers": the user calls scanner to
+fetch the next token, passes (pushes) it to the parser, and repeats the
+operation until the input is drained.
+
+This example is a straightforward conversion of the 'calc' example to the
+push-parser model.
+
+<!---
+Local Variables:
+fill-column: 76
+ispell-dictionary: "american"
+End:
+
+Copyright (C) 2020 Free Software Foundation, Inc.
+
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
+
+--->

examples/c/pushcalc/calc.test

@@ -0,0 +1,42 @@
+#! /bin/sh
+
+# Copyright (C) 2020 Free Software Foundation, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cat >input <<EOF
+1+2*3
+EOF
+run 0 7
+
+cat >input <<EOF
+1 - 2 - 3
+EOF
+run 0 -4
+
+cat >input <<EOF
+8 / 2 / 2
+EOF
+run 0 2
+
+cat >input <<EOF
+(1+2) * 3
+EOF
+run 0 9
+run -noerr 0 9 -p
+
+cat >input <<EOF
+1++2
+EOF
+run 0 "err: syntax error, unexpected '+', expecting number or '('"

examples/c/pushcalc/calc.y

@@ -0,0 +1,134 @@
+/* Parser and scanner for pushcalc.   -*- C -*-
+
+   Copyright (C) 2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+%code top {
+  #include <ctype.h>  /* isdigit. */
+  #include <stdio.h>  /* printf. */
+  #include <stdlib.h> /* abort. */
+  #include <string.h> /* strcmp. */
+}
+
+%code {
+  int yylex (YYSTYPE *yylval);
+  void yyerror (char const *);
+}
+
+%define api.header.include {"calc.h"}
+
+/* Generate YYSTYPE from the types used in %token and %type.  */
+%define api.value.type union
+%token <double> NUM "number"
+%type  <double> expr term fact
+
+/* Don't share global variables between the scanner and the parser.  */
+%define api.pure full
+/* Generate a push parser.  */
+%define api.push-pull push
+
+/* Nice error messages with details. */
+%define parse.error detailed
+
+/* Generate the parser description file (calc.output).  */
+%verbose
+
+ /* Enable run-time traces (yydebug).  */
+%define parse.trace
+
+/* Formatting semantic values in debug traces.  */
+%printer { fprintf (yyo, "%g", $$); } <double>;
+
+%% /* The grammar follows.  */
+input:
+  %empty
+| input line
+;
+
+line:
+  '\n'
+| expr '\n'  { printf ("%.10g\n", $1); }
+| error '\n' { yyerrok; }
+;
+
+expr:
+  expr '+' term { $$ = $1 + $3; }
+| expr '-' term { $$ = $1 - $3; }
+| term
+;
+
+term:
+  term '*' fact { $$ = $1 * $3; }
+| term '/' fact { $$ = $1 / $3; }
+| fact
+;
+
+fact:
+  "number"
+| '(' expr ')' { $$ = $expr; }
+;
+
+%%
+
+int
+yylex (YYSTYPE *yylval)
+{
+  int c;
+
+  /* Ignore white space, get first nonwhite character.  */
+  while ((c = getchar ()) == ' ' || c == '\t')
+    continue;
+
+  if (c == EOF)
+    return 0;
+
+  /* Char starts a number => parse the number.         */
+  if (c == '.' || isdigit (c))
+    {
+      ungetc (c, stdin);
+      if (scanf ("%lf", &yylval->NUM) != 1)
+        abort ();
+      return NUM;
+    }
+
+  /* Any other character is a token by itself.        */
+  return c;
+}
+
+/* Called by yyparse on error.  */
+void
+yyerror (char const *s)
+{
+  fprintf (stderr, "%s\n", s);
+}
+
+int
+main (int argc, char const* argv[])
+{
+  /* Enable parse traces on option -p.  */
+  for (int i = 1; i < argc; ++i)
+    if (!strcmp (argv[i], "-p"))
+      yydebug = 1;
+  int status;
+  yypstate *ps = yypstate_new ();
+  do {
+    YYSTYPE lval;
+    status = yypush_parse (ps, yylex (&lval), &lval);
+  } while (status == YYPUSH_MORE);
+  yypstate_delete (ps);
+  return status;
+}

examples/c/pushcalc/local.mk

@@ -0,0 +1,34 @@
+## Copyright (C) 2020 Free Software Foundation, Inc.
+##
+## This program is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+pushcalcdir = $(docdir)/%D%
+
+## ------ ##
+## Calc.  ##
+## ------ ##
+
+check_PROGRAMS += %D%/calc
+TESTS += %D%/calc.test
+EXTRA_DIST += %D%/calc.test
+nodist_%C%_calc_SOURCES = %D%/calc.y
+%D%/calc.c: $(dependencies)
+
+# Don't use gnulib's system headers.
+%C%_calc_CPPFLAGS = -I$(top_srcdir)/%D% -I$(top_builddir)/%D%
+%C%_calc_CFLAGS = $(TEST_CFLAGS)
+
+dist_pushcalc_DATA = %D%/calc.y %D%/Makefile %D%/README.md
+CLEANFILES += %D%/calc.[ch] %D%/calc.output
+CLEANDIRS += %D%/*.dSYM

examples/c/reccalc/Makefile

@@ -12,7 +12,7 @@ all: $(BASE)
 	$(BISON) $(BISONFLAGS) --defines --xml --graph=$*.gv -o $*.c $<
 
 %.c %.h: %.l
-	$(FLEX) $(FLEXFLAGS) -o$*.c --header-file=$*.h $<
+	$(FLEX) $(FLEXFLAGS) -o$*.c --header=$*.h $<
 
 scan.o: parse.h
 parse.o: scan.h

examples/c/reccalc/README.md

@@ -28,18 +28,11 @@ End:
 
 Copyright (C) 2018-2020 Free Software Foundation, Inc.
 
-This file is part of Bison, the GNU Compiler Compiler.
+Permission is granted to copy, distribute and/or modify this document
+under the terms of the GNU Free Documentation License, Version 1.3 or
+any later version published by the Free Software Foundation; with no
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the "GNU Free
+Documentation License" file as part of this distribution.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
 --->

examples/c/reccalc/local.mk

@@ -26,6 +26,8 @@ if FLEX_WORKS
   BUILT_SOURCES += $(nodist_%C%_reccalc_SOURCES)
   # Don't use gnulib's system headers.
   %C%_reccalc_CPPFLAGS = -I$(top_srcdir)/%D% -I$(top_builddir)/%D%
+  # Fighting warnings triggered by Flex is just too painful.
+  # %C%_reccalc_CFLAGS = $(TEST_CFLAGS)
 endif FLEX_WORKS
 
 %D%/parse.c: $(dependencies)
@@ -69,7 +71,9 @@ DASH = -
 	$(AM_V_LEX)rm -f $@ $@.tmp
 	$(AM_V_at)$(MKDIR_P) %D%
 	$(AM_V_at)touch $@.tmp
-	$(AM_V_at)$(LEX) $(AM_LFLAGS) $(LFLAGS) -o%D%/scan.c --header-file=%D%/scan.h $(srcdir)/%D%/scan.l
+## --header introduced in 2.5.6, renamed as --header-file in 2.6.4.
+## Backward compatibility ensured since --header is an unambiguous prefix.
+	$(AM_V_at)$(LEX) $(AM_LFLAGS) $(LFLAGS) -o%D%/scan.c --header=%D%/scan.h $(srcdir)/%D%/scan.l
 	$(AM_V_at)mv $@.tmp $@
 
 

examples/c/reccalc/parse.y

@@ -1,10 +1,33 @@
+/* Parser for reccalc.   -*- C -*-
+
+   Copyright (C) 2019-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
 // Prologue (directives).
 %expect 0
 
 // Emitted in the header file, before the definition of YYSTYPE.
 %code requires
 {
+  #ifndef YY_TYPEDEF_YY_SCANNER_T
+  # define YY_TYPEDEF_YY_SCANNER_T
   typedef void* yyscan_t;
+  #endif
+
   typedef struct
   {
     // Whether to print the intermediate results.
@@ -22,7 +45,7 @@
   // Tell Flex the expected prototype of yylex.
   // The scanner argument must be named yyscanner.
 #define YY_DECL                                                         \
-  enum yytokentype yylex (YYSTYPE* yylval, yyscan_t yyscanner, result *res)
+  yytoken_kind_t yylex (YYSTYPE* yylval, yyscan_t yyscanner, result *res)
   YY_DECL;
 
   void yyerror (yyscan_t scanner, result *res, const char *msg, ...);
@@ -42,14 +65,29 @@
   result parse (void);
 }
 
+// Include the header in the implementation rather than duplicating it.
+%define api.header.include {"parse.h"}
+
+// Don't share global variables between the scanner and the parser.
 %define api.pure full
+
+// To avoid name clashes (e.g., with C's EOF) prefix token definitions
+// with TOK_ (e.g., TOK_EOF).
 %define api.token.prefix {TOK_}
+
+// Generate YYSTYPE from the types assigned to symbols.
 %define api.value.type union
-%define parse.error verbose
+
+// Error messages with "unexpected XXX, expected XXX...".
+%define parse.error detailed
+
+// Enable run-time traces (yydebug).
 %define parse.trace
+
+// Generate the parser description file (parse.output).
 %verbose
 
- // Scanner and error count are exchanged between main, yyparse and yylex.
+// Scanner and error count are exchanged between main, yyparse and yylex.
 %param {yyscan_t scanner}{result *res}
 
 %token
@@ -132,7 +170,6 @@ exp:
 
 %%
 // Epilogue (C code).
-
 #include "scan.h"
 
 result

examples/c/reccalc/reccalc.test

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-seq 0 >/dev/null || exit 77
+(seq 0) >/dev/null 2>&1 || skip "gimme one seq"
 
 cat >input <<EOF
 1+2*3

examples/c/reccalc/scan.l

@@ -1,4 +1,23 @@
-/* Prologue (directives).   -*- C++ -*- */
+/* Scanner for reccalc.   -*- C -*-
+
+   Copyright (C) 2019-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* Prologue (directives).   -*- C -*- */
 
 /* Disable Flex features we don't need, to avoid warnings. */
 %option nodefault noinput nounput noyywrap
@@ -17,10 +36,14 @@
 
 %%
 %{
+  // Number of opened parentheses.
   int nesting = 0;
+  // A buffer storing the text inside the outer parentheses.
   char *str = NULL;
-  int size = 0;
+  // Its allocated size.
   int capacity = 0;
+  // Its used size.
+  int size = 0;
 #define STR_APPEND()                                    \
   do {                                                  \
     if (capacity < size + yyleng + 1)                   \
@@ -28,9 +51,9 @@
         do                                              \
           capacity = capacity ? 2 * capacity : 128;     \
         while (capacity < size + yyleng + 1);           \
-        str = realloc (str, capacity);                  \
+        str = realloc (str, (size_t) capacity);         \
       }                                                 \
-    memcpy (str + size, yytext, yyleng);                \
+    memcpy (str + size, yytext, (size_t) yyleng);       \
     size += yyleng;                                     \
     assert (size < capacity);                           \
   } while (0)

examples/c/rpcalc/local.mk

@@ -31,6 +31,7 @@ nodist_%C%_rpcalc_SOURCES = $(rpcalc_sources)
 %D%/rpcalc.c: $(dependencies)
 # Don't use gnulib's system headers.
 %C%_rpcalc_CPPFLAGS = -I$(top_builddir)/%D%
+%C%_rpcalc_CFLAGS = $(TEST_CFLAGS)
 %C%_rpcalc_LDADD = -lm
 
 dist_TESTS += %D%/rpcalc.test

examples/d/calc.y

@@ -1,3 +1,22 @@
+/* Parser and scanner for calc in D.   -*- D -*-
+
+   Copyright (C) 2018-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
 %language "D"
 
 %define api.parser.class {Calc}
@@ -99,16 +118,15 @@ class CalcLexer(R) : Lexer
     while (!input.empty && input.front != '\n' && isWhite (input.front))
       input.popFront;
 
-    // Handle EOF.
     if (input.empty)
-      return YYTokenType.EOF;
+      return TokenKind.YYEOF;
 
     // Numbers.
     if (input.front.isNumber)
       {
         import std.conv : parse;
         semanticVal_.ival = input.parse!int;
-        return YYTokenType.NUM;
+        return TokenKind.NUM;
       }
 
     // Individual characters
@@ -116,16 +134,15 @@ class CalcLexer(R) : Lexer
     input.popFront;
     switch (ch)
       {
-      case EOF: return YYTokenType.EOF;
-      case '=': return YYTokenType.EQ;
-      case '+': return YYTokenType.PLUS;
-      case '-': return YYTokenType.MINUS;
-      case '*': return YYTokenType.STAR;
-      case '/': return YYTokenType.SLASH;
-      case '(': return YYTokenType.LPAR;
-      case ')': return YYTokenType.RPAR;
-      case '\n': return YYTokenType.EOL;
-      default:  assert(0);
+      case '=':  return TokenKind.EQ;
+      case '+':  return TokenKind.PLUS;
+      case '-':  return TokenKind.MINUS;
+      case '*':  return TokenKind.STAR;
+      case '/':  return TokenKind.SLASH;
+      case '(':  return TokenKind.LPAR;
+      case ')':  return TokenKind.RPAR;
+      case '\n': return TokenKind.EOL;
+      default: assert(0);
       }
   }
 }

examples/d/local.mk

@@ -27,7 +27,7 @@ EXTRA_DIST += %D%/calc.test
 
 %D%/calc.d: %D%/calc.y $(dependencies)
 	$(AM_V_GEN)$(MKDIR_P) %D%
-	$(AM_V_at)$(BISON) $(srcdir)/%D%/calc.y -o $@
+	$(AM_V_at)$(BISON) -o $@ $(srcdir)/%D%/calc.y
 
 %D%/calc: %D%/calc.d
 	$(AM_V_GEN) $(DC) $(DCFLAGS) -of$@ %D%/calc.d

examples/java/Calc.y

@@ -1,168 +0,0 @@
-%language "Java"
-
-%define api.parser.class {Calc}
-%define api.parser.public
-
-%define parse.error verbose
-%locations
-
-%code imports {
-  import java.io.IOException;
-  import java.io.InputStream;
-  import java.io.InputStreamReader;
-  import java.io.Reader;
-  import java.io.StreamTokenizer;
-}
-
-%code {
-  public static void main (String args[]) throws IOException
-  {
-    CalcLexer l = new CalcLexer (System.in);
-    Calc p = new Calc (l);
-    p.parse ();
-  }
-}
-
-/* Bison Declarations */
-%token <Integer> NUM "number"
-%type  <Integer> exp
-
-%nonassoc '='       /* comparison            */
-%left '-' '+'
-%left '*' '/'
-%precedence NEG     /* negation--unary minus */
-%right '^'          /* exponentiation        */
-
-/* Grammar follows */
-%%
-input:
-  line
-| input line
-;
-
-line:
-  '\n'
-| exp '\n'           { System.out.println ($exp); }
-| error '\n'
-;
-
-exp:
-  NUM                { $$ = $1;                                             }
-| exp '=' exp
-  {
-    if ($1.intValue () != $3.intValue ())
-      yyerror (@$, "calc: error: " + $1 + " != " + $3);
-  }
-| exp '+' exp        { $$ = new Integer ($1.intValue () + $3.intValue ());  }
-| exp '-' exp        { $$ = new Integer ($1.intValue () - $3.intValue ());  }
-| exp '*' exp        { $$ = new Integer ($1.intValue () * $3.intValue ());  }
-| exp '/' exp        { $$ = new Integer ($1.intValue () / $3.intValue ());  }
-| '-' exp  %prec NEG { $$ = new Integer (-$2.intValue ());                  }
-| exp '^' exp        { $$ = new Integer ((int)
-                                         Math.pow ($1.intValue (),
-                                                   $3.intValue ()));        }
-| '(' exp ')'        { $$ = $2;                                             }
-| '(' error ')'      { $$ = new Integer (1111);                             }
-| '!'                { $$ = new Integer (0); return YYERROR;                }
-| '-' error          { $$ = new Integer (0); return YYERROR;                }
-;
-
-
-%%
-class CalcLexer implements Calc.Lexer {
-
-  StreamTokenizer st;
-
-  public CalcLexer (InputStream is)
-  {
-    st = new StreamTokenizer (new InputStreamReader (is));
-    st.resetSyntax ();
-    st.eolIsSignificant (true);
-    st.whitespaceChars ('\t', '\t');
-    st.whitespaceChars (' ', ' ');
-    st.wordChars ('0', '9');
-  }
-
-
-  Position yypos = new Position (1, 0);
-
-  public Position getStartPos () {
-    return yypos;
-  }
-
-  public Position getEndPos () {
-    return yypos;
-  }
-
-  public void yyerror (Calc.Location l, String s)
-  {
-    if (l == null)
-      System.err.println (s);
-    else
-      System.err.println (l + ": " + s);
-  }
-
-
-  Integer yylval;
-
-  public Object getLVal () {
-    return yylval;
-  }
-
-  public int yylex () throws IOException {
-    int ttype = st.nextToken ();
-    yypos = new Position (yypos.lineno (), yypos.token () + 1);
-    if (ttype == st.TT_EOF)
-      return EOF;
-    else if (ttype == st.TT_EOL)
-      {
-        yypos = new Position (yypos.lineno () + 1, 0);
-        return (int) '\n';
-      }
-    else if (ttype == st.TT_WORD)
-      {
-        yylval = new Integer (st.sval);
-        return NUM;
-      }
-    else
-      return st.ttype;
-  }
-}
-
-
-class Position {
-  public int line;
-  public int token;
-
-  public Position ()
-  {
-    line = 0;
-    token = 0;
-  }
-
-  public Position (int l, int t)
-  {
-    line = l;
-    token = t;
-  }
-
-  public boolean equals (Position l)
-  {
-    return l.line == line && l.token == token;
-  }
-
-  public String toString ()
-  {
-    return Integer.toString (line) + "." + Integer.toString(token);
-  }
-
-  public int lineno ()
-  {
-    return line;
-  }
-
-  public int token ()
-  {
-    return token;
-  }
-}

examples/java/README.md

@@ -5,8 +5,11 @@ This directory contains examples of Bison grammar files in Java.
 You can run `make` to compile these examples.  And `make clean` to tidy
 afterwards.
 
-## java/Calc.y
-The usual calculator.
+## simple/Calc.y
+The usual calculator, a very simple version.
+
+## calc/Calc.y
+The calculator, but with location tracking, debug traces, and a push parser.
 
 <!---
 

examples/java/calc/Calc.test

@@ -0,0 +1,38 @@
+#! /bin/sh
+
+# Copyright (C) 2018-2020 Free Software Foundation, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cat >input <<EOF
+1 + 2 * 3
+EOF
+run 0 7
+
+cat >input <<EOF
+1 + 2 * 3 = 7
+(1 + 2) * 3 = 9
+EOF
+run 0 '7
+9'
+
+cat >input <<EOF
+1 + 2 * * 3
+EOF
+run 0 "err: 1.9-1.10: syntax error: expected ! or - or ( or number before *"
+
+cat >input <<EOF
+12   222
+EOF
+run 0 "err: 1.6-1.9: syntax error: expected + or - or * or / or ^ or = or end of line before number"

examples/java/calc/Calc.y

@@ -0,0 +1,311 @@
+/* Parser and scanner for calc in Java.   -*- Java -*-
+
+   Copyright (C) 2018-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+%language "Java"
+
+%define api.parser.class {Calc}
+%define api.parser.public
+%define api.push-pull push
+
+%define parse.error custom
+%define parse.trace
+
+%locations
+
+%code imports {
+  import java.io.BufferedReader;
+  import java.io.IOException;
+  import java.io.InputStream;
+  import java.io.InputStreamReader;
+  import java.io.Reader;
+  import java.io.StreamTokenizer;
+  import java.nio.CharBuffer;
+}
+
+%code {
+  public static void main(String[] args) throws IOException {
+    CalcLexer scanner = new CalcLexer(System.in);
+    Calc parser = new Calc(scanner);
+    for (String arg : args)
+      if (arg.equals("-p"))
+        parser.setDebugLevel(1);
+    int status;
+    do {
+      int token = scanner.getToken();
+      Object lval = scanner.getValue();
+      Calc.Location yyloc = scanner.getLocation();
+      status = parser.push_parse(token, lval, yyloc);
+    } while (status == Calc.YYPUSH_MORE);
+    if (status != Calc.YYACCEPT)
+      System.exit(1);
+  }
+
+  static String i18n(String s) {
+    return s;
+  }
+}
+
+/* Bison Declarations */
+%token
+    BANG   "!"
+    PLUS   "+"
+    MINUS  "-"
+    STAR   "*"
+    SLASH  "/"
+    CARET  "^"
+    LPAREN "("
+    RPAREN ")"
+    EQUAL  "="
+    EOL    _("end of line")
+  <Integer>
+    NUM    _("number")
+%type  <Integer> exp
+
+%nonassoc "="       /* comparison            */
+%left "-" "+"
+%left "*" "/"
+%precedence NEG     /* negation--unary minus */
+%right "^"          /* exponentiation        */
+
+/* Grammar follows */
+%%
+input:
+  line
+| input line
+;
+
+line:
+  EOL
+| exp EOL            { System.out.println($exp); }
+| error EOL
+;
+
+exp:
+  NUM                { $$ = $1; }
+| exp "=" exp
+  {
+    if ($1.intValue() != $3.intValue())
+      yyerror(@$, "calc: error: " + $1 + " != " + $3);
+  }
+| exp "+" exp        { $$ = $1 + $3;  }
+| exp "-" exp        { $$ = $1 - $3;  }
+| exp "*" exp        { $$ = $1 * $3;  }
+| exp "/" exp        { $$ = $1 / $3;  }
+| "-" exp  %prec NEG { $$ = -$2; }
+| exp "^" exp        { $$ = (int) Math.pow($1, $3); }
+| "(" exp ")"        { $$ = $2; }
+| "(" error ")"      { $$ = 1111; }
+| "!"                { $$ = 0; return YYERROR; }
+| "-" error          { $$ = 0; return YYERROR; }
+;
+
+%%
+class CalcLexer implements Calc.Lexer {
+
+  StreamTokenizer st;
+  PositionReader reader;
+
+  public CalcLexer(InputStream is) {
+    reader = new PositionReader(new InputStreamReader(is));
+    st = new StreamTokenizer(reader);
+    st.resetSyntax();
+    st.eolIsSignificant(true);
+    st.wordChars('0', '9');
+  }
+
+  Position start = new Position(1, 0);
+  Position end = new Position(1, 0);
+
+  /**
+   * The location of the last token read.
+   * Implemented with getStartPos and getEndPos in pull parsers.
+   */
+  public Calc.Location getLocation() {
+    return new Calc.Location(new Position(start), new Position(end));
+  }
+
+  /**
+   * Build and emit a syntax error message.
+   */
+  public void reportSyntaxError(Calc.Context ctx) {
+    System.err.print(ctx.getLocation() + ": syntax error");
+    {
+      final int TOKENMAX = 10;
+      Calc.SymbolKind[] arg = new Calc.SymbolKind[TOKENMAX];
+      int n = ctx.getExpectedTokens(arg, TOKENMAX);
+      for (int i = 0; i < n; ++i)
+        System.err.print((i == 0 ? ": expected " : " or ")
+                         + arg[i].getName());
+    }
+    {
+      Calc.SymbolKind lookahead = ctx.getToken();
+      if (lookahead != null)
+        System.err.print(" before " + lookahead.getName());
+    }
+    System.err.println("");
+  }
+
+  /**
+   * Emit an error referring to the given location in a user-defined way.
+   *
+   * @@param loc The location of the element to which the
+   *                error message is related.
+   * @@param msg The string for the error message.
+   */
+  public void yyerror(Calc.Location loc, String msg) {
+    if (loc == null)
+      System.err.println(msg);
+    else
+      System.err.println(loc + ": " + msg);
+  }
+
+  Integer yylval;
+
+  /**
+   * The value of the last token read.  Called getLVal in pull parsers.
+   */
+  public Object getValue() {
+    return yylval;
+  }
+
+  /**
+   * Fetch the next token.  Called yylex in pull parsers.
+   */
+  public int getToken() throws IOException {
+    start.set(reader.getPosition());
+    int ttype = st.nextToken();
+    end.set(reader.getPosition());
+    switch (ttype) {
+    case StreamTokenizer.TT_EOF:
+      return YYEOF;
+    case StreamTokenizer.TT_EOL:
+      end.line += 1;
+      end.column = 0;
+      return EOL;
+    case StreamTokenizer.TT_WORD:
+      yylval = new Integer(st.sval);
+      end.set(reader.getPreviousPosition());
+      return NUM;
+    case ' ': case '\t':
+      return getToken();
+    case '!':
+      return BANG;
+    case '+':
+      return PLUS;
+    case '-':
+      return MINUS;
+    case '*':
+      return STAR;
+    case '/':
+      return SLASH;
+    case '^':
+      return CARET;
+    case '(':
+      return LPAREN;
+    case ')':
+      return RPAREN;
+    case '=':
+      return EQUAL;
+    default:
+      throw new AssertionError("invalid character: " + ttype);
+    }
+  }
+}
+
+/**
+ * A class defining a point in the input.
+ */
+class Position {
+  public int line = 1;
+  public int column = 1;
+
+  public Position() {
+    line = 1;
+    column = 1;
+  }
+
+  public Position(int l, int t) {
+    line = l;
+    column = t;
+  }
+
+  public Position(Position p) {
+    line = p.line;
+    column = p.column;
+  }
+
+  public void set(Position p) {
+    line = p.line;
+    column = p.column;
+  }
+
+  public boolean equals(Position l) {
+    return l.line == line && l.column == column;
+  }
+
+  public String toString() {
+    return Integer.toString(line) + "." + Integer.toString(column);
+  }
+
+  public int line() {
+    return line;
+  }
+
+  public int column() {
+    return column;
+  }
+}
+
+/**
+ * A Stream reader that keeps track of the current Position.
+ */
+class PositionReader extends BufferedReader {
+
+  private Position position = new Position();
+  // Position before the latest call to "read", i.e. position
+  // of the last character of the current token.
+  private Position previousPosition = new Position();
+
+  public PositionReader(Reader reader) {
+    super(reader);
+  }
+
+  public int read() throws IOException {
+    previousPosition.set(position);
+    int res = super.read();
+    if (res > -1) {
+      char c = (char) res;
+      if (c == '\r' || c == '\n') {
+        position.line += 1;
+        position.column = 1;
+      } else {
+        position.column += 1;
+      }
+    }
+    return res;
+  }
+
+  public Position getPosition() {
+    return position;
+  }
+
+  public Position getPreviousPosition() {
+    return previousPosition;
+  }
+}

examples/java/calc/local.mk

@@ -0,0 +1,36 @@
+## Copyright (C) 2018-2020 Free Software Foundation, Inc.
+##
+## This program is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+java_calcdir = $(docdir)/%D%
+
+## ------ ##
+## Calc.  ##
+## ------ ##
+
+if ENABLE_JAVA
+  check_SCRIPTS += %D%/Calc.class
+  TESTS += %D%/Calc.test
+endif
+EXTRA_DIST += %D%/Calc.test
+
+%D%/Calc.java: %D%/Calc.y $(dependencies)
+	$(AM_V_GEN)$(MKDIR_P) %D%
+	$(AM_V_at)$(BISON) -o $@ $(srcdir)/%D%/Calc.y
+
+%D%/Calc.class: %D%/Calc.java
+	$(AM_V_GEN) $(SHELL) $(top_builddir)/javacomp.sh %D%/Calc.java
+
+dist_java_calc_DATA = %D%/Calc.y %D%/Makefile
+CLEANFILES += %D%/*.class %D%/Calc.java

examples/java/local.mk

@@ -1,4 +1,4 @@
-## Copyright (C) 2018-2020 Free Software Foundation, Inc.
+## Copyright (C) 2020 Free Software Foundation, Inc.
 ##
 ## This program is free software: you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -14,23 +14,7 @@
 ## along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 javadir = $(docdir)/%D%
+dist_java_DATA = %D%/README.md
 
-## ------ ##
-## Calc.  ##
-## ------ ##
-
-if ENABLE_JAVA
-  check_SCRIPTS += %D%/Calc.class
-  TESTS += %D%/Calc.test
-endif
-EXTRA_DIST += %D%/Calc.test
-
-%D%/Calc.java: %D%/Calc.y $(dependencies)
-	$(AM_V_GEN)$(MKDIR_P) %D%
-	$(AM_V_at)$(BISON) $(srcdir)/%D%/Calc.y -o $@
-
-%D%/Calc.class: %D%/Calc.java
-	$(AM_V_GEN) $(SHELL) $(top_builddir)/javacomp.sh %D%/Calc.java
-
-dist_java_DATA = %D%/Calc.y %D%/Makefile %D%/README.md
-CLEANFILES += %D%/*.class %D%/Calc.java
+include %D%/calc/local.mk
+include %D%/simple/local.mk

examples/java/simple/Calc.test

@@ -20,7 +20,14 @@ cat >input <<EOF
 EOF
 run 0 7
 
+cat >input <<EOF
+1 + 2 * 3 = 7
+(1 + 2) * 3 = 9
+EOF
+run 0 '7
+9'
+
 cat >input <<EOF
 1 + 2 * * 3
 EOF
-run 0 "err: 1.5: syntax error, unexpected '*', expecting number or '-' or '(' or '!'"
+run 0 "err: syntax error, unexpected '*', expecting number or '-' or '(' or '!'"

examples/java/simple/Calc.y

@@ -0,0 +1,125 @@
+/* Simple parser and scanner in Java.   -*- Java -*-
+
+   Copyright (C) 2018-2020 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+%language "Java"
+
+%define api.parser.class {Calc}
+%define api.parser.public
+
+%define parse.error verbose
+
+%code imports {
+  import java.io.IOException;
+  import java.io.InputStream;
+  import java.io.InputStreamReader;
+  import java.io.Reader;
+  import java.io.StreamTokenizer;
+}
+
+%code {
+  public static void main(String[] args) throws IOException {
+    CalcLexer l = new CalcLexer(System.in);
+    Calc p = new Calc(l);
+    if (!p.parse())
+      System.exit(1);
+  }
+}
+
+/* Bison Declarations */
+%token <Integer> NUM "number"
+%type  <Integer> exp
+
+%nonassoc '='       /* comparison            */
+%left '-' '+'
+%left '*' '/'
+%precedence NEG     /* negation--unary minus */
+%right '^'          /* exponentiation        */
+
+/* Grammar follows */
+%%
+input:
+  line
+| input line
+;
+
+line:
+  '\n'
+| exp '\n'           { System.out.println($exp); }
+| error '\n'
+;
+
+exp:
+  NUM                { $$ = $1; }
+| exp '=' exp
+  {
+    if ($1.intValue() != $3.intValue())
+      yyerror("calc: error: " + $1 + " != " + $3);
+  }
+| exp '+' exp        { $$ = $1 + $3;  }
+| exp '-' exp        { $$ = $1 - $3;  }
+| exp '*' exp        { $$ = $1 * $3;  }
+| exp '/' exp        { $$ = $1 / $3;  }
+| '-' exp  %prec NEG { $$ = -$2; }
+| exp '^' exp        { $$ = (int) Math.pow($1, $3); }
+| '(' exp ')'        { $$ = $2; }
+| '(' error ')'      { $$ = 1111; }
+| '!'                { $$ = 0; return YYERROR; }
+| '-' error          { $$ = 0; return YYERROR; }
+;
+
+
+%%
+class CalcLexer implements Calc.Lexer {
+
+  StreamTokenizer st;
+
+  public CalcLexer(InputStream is) {
+    st = new StreamTokenizer(new InputStreamReader(is));
+    st.resetSyntax();
+    st.eolIsSignificant(true);
+    st.whitespaceChars('\t', '\t');
+    st.whitespaceChars(' ', ' ');
+    st.wordChars('0', '9');
+  }
+
+  public void yyerror(String s) {
+    System.err.println(s);
+  }
+
+  Integer yylval;
+
+  public Object getLVal() {
+    return yylval;
+  }
+
+  public int yylex() throws IOException {
+    int ttype = st.nextToken();
+    switch (ttype) {
+    case StreamTokenizer.TT_EOF:
+      return YYEOF;
+    case StreamTokenizer.TT_EOL:
+      return (int) '\n';
+    case StreamTokenizer.TT_WORD:
+      yylval = new Integer(st.sval);
+      return NUM;
+    default:
+      return ttype;
+    }
+  }
+}

examples/java/simple/Makefile

@@ -0,0 +1,26 @@
+# This Makefile is designed to be simple and readable.  It does not
+# aim at portability.  It requires GNU Make.
+
+BISON = bison
+JAVAC = javac
+JAVA = java
+XSLTPROC = xsltproc
+
+all: Calc.class
+
+%.java %.xml %.gv: %.y
+	$(BISON) $(BISONFLAGS) --xml --graph=$*.gv -o $*.java $<
+
+%.class: %.java
+	$(JAVAC) $(JAVACFLAGS) $<
+
+run: Calc.class
+	@echo "Type arithmetic expressions.  Quit with ctrl-d."
+	$(JAVA) $(JAVAFLAGS) Calc
+
+html: Calc.html
+%.html: %.xml
+	$(XSLTPROC) $(XSLTPROCFLAGS) -o $@ $$($(BISON) --print-datadir)/xslt/xml2xhtml.xsl $<
+
+clean:
+	rm -f *.class Calc.java Calc.html Calc.xml Calc.gv

examples/java/simple/local.mk

@@ -0,0 +1,36 @@
+## Copyright (C) 2018-2020 Free Software Foundation, Inc.
+##
+## This program is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+java_simpledir = $(docdir)/%D%
+
+## ------ ##
+## Calc.  ##
+## ------ ##
+
+if ENABLE_JAVA
+  check_SCRIPTS += %D%/Calc.class
+  TESTS += %D%/Calc.test
+endif
+EXTRA_DIST += %D%/Calc.test
+
+%D%/Calc.java: %D%/Calc.y $(dependencies)
+	$(AM_V_GEN)$(MKDIR_P) %D%
+	$(AM_V_at)$(BISON) -o $@ $(srcdir)/%D%/Calc.y
+
+%D%/Calc.class: %D%/Calc.java
+	$(AM_V_GEN) $(SHELL) $(top_builddir)/javacomp.sh %D%/Calc.java
+
+dist_java_simple_DATA = %D%/Calc.y %D%/Makefile
+CLEANFILES += %D%/*.class %D%/Calc.java