Remove double EOF and fix places where token injects text before the next token

diff --git a/bootstrap/markup.py b/bootstrap/markup.py
index 9c5a8fe..543338e 100755 (executable)
--- a/bootstrap/markup.py
+++ b/bootstrap/markup.py
@@ -4,9 +4,7 @@ import lex_yy
 import y_tab
 import sys
 
-lex_yy.piece_append('<root><AST ref=\"0\"><AST_Section1>')
+lex_yy.piece_append('<root>\n  <AST ref=\"0\"><AST_Section1>')
 y_tab.yyparse()
-while lex_yy.flexscan():
-  pass
-lex_yy.piece_append('</AST></root>\n')
+lex_yy.piece_append('</AST>\n</root>\n')
 sys.stdout.write(''.join(lex_yy.piece))

diff --git a/bootstrap/scan.l b/bootstrap/scan.l
index e766394..fbfc040 100644 (file)
--- a/bootstrap/scan.l
+++ b/bootstrap/scan.l
@@ -1446,13 +1446,9 @@ M4QEND      "]""]"
     sectnum = 3
     BEGIN(SECT3_NOESCAPE if state.no_section3_escape else SECT3)
     #outn('/* Begin user sect3 */')
-    piece_pack()
     piece_append('</AST_Section2>')
-    piece_escape(yytext)
-    piece_pack()
+    piece_flush(len(yytext))
     piece_append('<AST_Section3>')
-    # first EOF, we will call scanner until it's returned EOF twice
-    return ~YY_NULL
   }
 
   "["({FIRST_CCL_CHAR}|{CCL_EXPR})({CCL_CHAR}|{CCL_EXPR})* {
@@ -1606,13 +1602,13 @@ M4QEND      "]""]"
   "'"{NAME}"'" |
   "<"{NAME}">" {
     BEGIN(SECT2)
-    piece_flush(1)
     piece_pack()
+    piece_flush(1)
     piece_append('<RegexGroupName_Text>')
     piece_flush(len(yytext) - 1)
     piece_append('</RegexGroupName_Text>')
-    piece_pack()
     piece_flush(1)
+    piece_pack()
     return ~y_tab.NAME
   }
     /* Nick extra rules for action groups */