d: change the return value of yylex from TokenKind to YYParser.Symbol

The complete symbol approach was deemed to be the right approach for Dlang.
Now, the user can return from yylex() an instance of YYParser.Symbol structure,
which binds together the TokenKind, the semantic value and the location. Before,
the last two were reported separately to the parser.
Only the user API is changed, Bisons's internal structure is kept the same.

* data/skeletons/d.m4 (struct YYParser.Symbol): New.
* data/skeletons/lalr1.d: Change the return value.
* doc/bison.texi: Document it.
* examples/d/calc/calc.y, examples/d/simple/calc.y: Demonstrate it.
* tests/calc.at, tests/scanner.at: Test it.

data/skeletons/d.m4

@@ -446,3 +446,38 @@ m4_define([b4_var_decls],
 ], [$@])])
 m4_define([b4_var_decl],
           [    protected $1;])
+
+
+# b4_symbol_type_define
+# ---------------------
+# Define symbol_type, the external type for symbols used for symbol
+# constructors.
+m4_define([b4_symbol_type_define],
+[[
+  /**
+    * A complete symbol
+    */
+  struct Symbol
+  {
+    private SymbolKind kind;
+    private ]b4_yystype[ value;]b4_locations_if([[
+    private YYLocation location_;]])[
+    this(TokenKind token]b4_locations_if([[, YYLocation loc]])[)
+    {
+      kind = yytranslate_(token);]b4_locations_if([
+      location_ = loc;])[
+    }
+    static foreach (member; __traits(allMembers, YYSemanticType))
+    {
+      this(TokenKind token, typeof(mixin("YYSemanticType." ~ member)) val]b4_locations_if([[, YYLocation loc]])[)
+      {
+        kind = yytranslate_(token);
+        mixin("value." ~ member ~ " = val;");]b4_locations_if([
+        location_ = loc;])[
+      }
+    }
+    SymbolKind token() { return kind; }
+    ]b4_yystype[ semanticValue() { return value; }]b4_locations_if([[
+    YYLocation location() { return location_; }]])[
+  }
+]])

data/skeletons/lalr1.d

@@ -75,7 +75,7 @@ public interface Lexer
    * to the next token and prepares to return the semantic value
    * ]b4_locations_if([and beginning/ending positions ])[of the token.
    * @@return the token identifier corresponding to the next token. */
-  TokenKind yylex ();
+  ]b4_parser_class[.Symbol yylex ();
 
   /**
    * Entry point for error reporting.  Emits an error
@@ -290,7 +290,7 @@ b4_user_union_members
       yyDebugStream.writeln (s);
   }
 ]])[
-  private final TokenKind yylex () {
+  private final ]b4_parser_class[.Symbol yylex () {
     return yylexer.yylex ();
   }
 
@@ -411,7 +411,9 @@ b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
       yycdebugln (message);
     }
   }
-]])[
+]])
+b4_symbol_type_define
+[
   /**
    * Parse input from the scanner that was specified at object construction
    * time.  Return whether the end of the input was reached successfully.
@@ -493,13 +495,10 @@ m4_popdef([b4_at_dollar])])dnl
         if (yychar == TokenKind.]b4_symbol(empty, id)[)
         {]b4_parse_trace_if([[
           yycdebugln ("Reading a token");]])[
-          yychar = yylex ();]b4_locations_if([[
-          static if (yy_location_is_class) {
-            yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos);
-          } else {
-            yylloc = ]b4_location_type[(yylexer.startPos, yylexer.endPos);
-          }]])
-          yylval = yylexer.semanticVal;[
+          Symbol yysymbol = yylex();
+          yychar = yysymbol.token();
+          yylval = yysymbol.semanticValue();]b4_locations_if([[
+          yylloc = yysymbol.location();]])[
         }
 
         /* Convert token to internal form.  */