From: Nick Downing <nick@ndcode.org>
Date: Sat, 19 Jan 2019 12:55:09 +0000 (+1100)
Subject: Remove double EOF and fix places where token injects text before the next token
X-Git-Url: https://git.ndcode.org/public/gitweb.cgi?a=commitdiff_plain;h=0eecfb9de19c3e8d78bf3f05fb874c859128db6d;p=pilex.git

Remove double EOF and fix places where token injects text before the next token
---

diff --git a/bootstrap/markup.py b/bootstrap/markup.py
index 9c5a8fe..543338e 100755
--- a/bootstrap/markup.py
+++ b/bootstrap/markup.py
@@ -4,9 +4,7 @@ import lex_yy
 import y_tab
 import sys
 
-lex_yy.piece_append('<root><AST ref=\"0\"><AST_Section1>')
+lex_yy.piece_append('<root>\n  <AST ref=\"0\"><AST_Section1>')
 y_tab.yyparse()
-while lex_yy.flexscan():
-  pass
-lex_yy.piece_append('</AST></root>\n')
+lex_yy.piece_append('</AST>\n</root>\n')
 sys.stdout.write(''.join(lex_yy.piece))
diff --git a/bootstrap/scan.l b/bootstrap/scan.l
index e766394..fbfc040 100644
--- a/bootstrap/scan.l
+++ b/bootstrap/scan.l
@@ -1446,13 +1446,9 @@ M4QEND      "]""]"
     sectnum = 3
     BEGIN(SECT3_NOESCAPE if state.no_section3_escape else SECT3)
     #outn('/* Begin user sect3 */')
-    piece_pack()
     piece_append('</AST_Section2>')
-    piece_escape(yytext)
-    piece_pack()
+    piece_flush(len(yytext))
     piece_append('<AST_Section3>')
-    # first EOF, we will call scanner until it's returned EOF twice
-    return ~YY_NULL
   }
 
   "["({FIRST_CCL_CHAR}|{CCL_EXPR})({CCL_CHAR}|{CCL_EXPR})* {
@@ -1606,13 +1602,13 @@ M4QEND      "]""]"
   "'"{NAME}"'" |
   "<"{NAME}">" {
     BEGIN(SECT2)
-    piece_flush(1)
     piece_pack()
+    piece_flush(1)
     piece_append('<RegexGroupName_Text>')
     piece_flush(len(yytext) - 1)
     piece_append('</RegexGroupName_Text>')
-    piece_pack()
     piece_flush(1)
+    piece_pack()
     return ~y_tab.NAME
   }
     /* Nick extra rules for action groups */