Extract the parsing of user actions from the grammar scanner.

As a consequence, the relation between the grammar scanner and
parser is much simpler.  We can also split "composite tokens" back
into simple tokens.
* src/gram.h (ITEM_NUMBER_MAX, RULE_NUMBER_MAX): New.
* src/scan-gram.l (add_column_width, adjust_location): Move to and
rename as...
* src/location.h, src/location.c (add_column_width)
(location_compute): these.
Fix the column count: the initial column is 0.
(location_print): Be robust to ending column being 0.
* src/location.h (boundary_set): New.
* src/main.c: Adjust to scanner_free being renamed as
gram_scanner_free.
* src/output.c: Include scan-code.h.
* src/parse-gram.y: Include scan-gram.h and scan-code.h.
Use boundary_set.
(PERCENT_DESTRUCTOR, PERCENT_PRINTER, PERCENT_INITIAL_ACTION)
(PERCENT_LEX_PARAM, PERCENT_PARSE_PARAM): Remove the {...} part,
which is now, again, a separate token.
Adjust all dependencies.
Whereever actions with $ and @ are used, use translate_code.
(action): Remove this nonterminal which is now useless.
* src/reader.c: Include assert.h, scan-gram.h and scan-code.h.
(grammar_current_rule_action_append): Use translate_code.
(packgram): Bound check ruleno, itemno, and rule_length.
* src/reader.h (gram_in, gram__flex_debug, scanner_cursor)
(last_string, last_braced_code_loc, max_left_semantic_context)
(scanner_initialize, scanner_free, scanner_last_string_free)
(gram_out, gram_lineno, YY_DECL_): Move to...
* src/scan-gram.h: this new file.
(YY_DECL): Rename as...
(GRAM_DECL): this.
* src/scan-code.h, src/scan-code.l, src/scan-code-c.c: New.
* src/scan-gram.l (gram_get_lineno, gram_get_in, gram_get_out):
(gram_get_leng, gram_get_text, gram_set_lineno, gram_set_in):
(gram_set_out, gram_get_debug, gram_set_debug, gram_lex_destroy):
Move these declarations, and...
(obstack_for_string, STRING_GROW, STRING_FINISH, STRING_FREE):
these to...
* src/flex-scanner.h: this new file.
* src/scan-gram.l (rule_length, rule_length_overflow)
(increment_rule_length): Remove.
(last_braced_code_loc): Rename as...
(gram_last_braced_code_loc): this.
Adjust to the changes of the parser.
Move all the handling of $ and @ into...
* src/scan-code.l: here.
* src/scan-gram.l (handle_dollar, handle_at): Remove.
(handle_action_dollar, handle_action_at): Move to...
* src/scan-code.l: here.
* src/Makefile.am (bison_SOURCES): Add flex-scanner.h,
scan-code.h, scan-code-c.c, scan-gram.h.
(EXTRA_bison_SOURCES): Add scan-code.l.
(BUILT_SOURCES): Add scan-code.c.
(yacc): Be robust to white spaces.
* tests/conflicts.at, tests/input.at, tests/reduce.at,
* tests/regression.at: Adjust the column numbers.
* tests/regression.at: Adjust the error message.

src/reader.c

@@ -22,6 +22,7 @@
 
 #include <config.h>
 #include "system.h"
+#include <assert.h>
 
 #include <quotearg.h>
 
@@ -34,6 +35,8 @@
 #include "reader.h"
 #include "symlist.h"
 #include "symtab.h"
+#include "scan-gram.h"
+#include "scan-code.h"
 
 static void check_and_convert_grammar (void);
 
@@ -77,6 +80,8 @@ prologue_augment (const char *prologue, location loc)
     !typed ? &pre_prologue_obstack : &post_prologue_obstack;
 
   obstack_fgrow1 (oout, "]b4_syncline(%d, [[", loc.start.line);
+  /* FIXME: Protection of M4 characters missing here.  See
+     output.c:escaped_output.  */
   MUSCLE_OBSTACK_SGROW (oout,
 			quotearg_style (c_quoting_style, loc.start.file));
   obstack_sgrow (oout, "]])[\n");
@@ -398,9 +403,7 @@ grammar_current_rule_symbol_append (symbol *sym, location loc)
 void
 grammar_current_rule_action_append (const char *action, location loc)
 {
-  /* There's no need to invoke grammar_midrule_action here, since the
-     scanner already did it if necessary.  */
-  current_rule->action = action;
+  current_rule->action = translate_rule_action (current_rule, action, loc);
   current_rule->action_location = loc;
 }
 
@@ -426,6 +429,7 @@ packgram (void)
 
   while (p)
     {
+      int rule_length = 0;
       symbol *ruleprec = p->ruleprec;
       rules[ruleno].user_number = ruleno;
       rules[ruleno].number = ruleno;
@@ -440,18 +444,22 @@ packgram (void)
       rules[ruleno].action = p->action;
       rules[ruleno].action_location = p->action_location;
 
-      p = p->next;
-      while (p && p->sym)
+      for (p = p->next; p && p->sym; p = p->next)
 	{
+	  ++rule_length;
+
+	  /* Don't allow rule_length == INT_MAX, since that might
+	     cause confusion with strtol if INT_MAX == LONG_MAX.  */
+	  if (rule_length == INT_MAX)
+	      fatal_at (rules[ruleno].location, _("rule is too long"));
+
 	  /* item_number = symbol_number.
 	     But the former needs to contain more: negative rule numbers. */
 	  ritem[itemno++] = symbol_number_as_item_number (p->sym->number);
 	  /* A rule gets by default the precedence and associativity
-	     of the last token in it.  */
+	     of its last token.  */
 	  if (p->sym->class == token_sym && default_prec)
 	    rules[ruleno].prec = p->sym;
-	  if (p)
-	    p = p->next;
 	}
 
       /* If this rule has a %prec,
@@ -461,8 +469,11 @@ packgram (void)
 	  rules[ruleno].precsym = ruleprec;
 	  rules[ruleno].prec = ruleprec;
 	}
+      /* An item ends by the rule number (negated).  */
       ritem[itemno++] = rule_number_as_item_number (ruleno);
+      assert (itemno < ITEM_NUMBER_MAX);
       ++ruleno;
+      assert (ruleno < RULE_NUMBER_MAX);
 
       if (p)
 	p = p->next;
@@ -511,7 +522,7 @@ reader (void)
 
   gram__flex_debug = trace_flag & trace_scan;
   gram_debug = trace_flag & trace_parse;
-  scanner_initialize ();
+  gram_scanner_initialize ();
   gram_parse ();
 
   if (! complaint_issued)