Fix scanning bug of adding extra text to matched string

diff --git a/ndcode/pilex/cli.py b/ndcode/pilex/cli.py
index e616d6f..46e8ac5 100755 (executable)
--- a/ndcode/pilex/cli.py
+++ b/ndcode/pilex/cli.py
@@ -78,7 +78,7 @@ def main():
       _ast = y_tab.yyparse(t_def.AST)
   #element.serialize(_ast, 'a.xml', 'utf-8')
   #_ast = element.deserialize('a.xml', t_def.factory, 'utf-8')
-  _ast.post_process(python)
+  _ast.post_process()
   #element.serialize(_ast, 'b.xml', 'utf-8')
   #_ast = element.deserialize('b.xml', t_def.factory, 'utf-8')
   (generate_py if python else generate_flex)(

diff --git a/ndcode/pilex/skel/skel_py.py b/ndcode/pilex/skel/skel_py.py
index 9e9dfe8..da776b7 100644 (file)
--- a/ndcode/pilex/skel/skel_py.py
+++ b/ndcode/pilex/skel/skel_py.py
@@ -249,7 +249,8 @@ def yylex():
           # at this point len(yy_state_buf) == number of chars scanned
           i = len(yy_state_buf) - len(match)
           if i:
-            match += block.text[block_pos - i:]
+            match += block.text[block_pos - i:block_pos]
+          assert len(match) == len(yy_state_buf)
 
           # advance past exhausted block, maintaining invariant
           block_prev = block
@@ -283,7 +284,8 @@ def yylex():
     # at this point yy_state_ptr == number of chars scanned
     i = yy_state_ptr - len(match)
     if i:
-      match += block.text[block_pos - i:]
+      match += block.text[block_pos - i:block_pos]
+    assert len(match) = yy_state_ptr
 
     # yy_find_action:
     # the following variables are used to handle trailing context

diff --git a/ndcode/pilex/skel/skel_py_element.py b/ndcode/pilex/skel/skel_py_element.py
index acfde43..4053521 100644 (file)
--- a/ndcode/pilex/skel/skel_py_element.py
+++ b/ndcode/pilex/skel/skel_py_element.py
@@ -284,7 +284,8 @@ def yylex(_class = element.Element, *args, **kwargs):
         else:
           i = match_len - len(match)
           if i:
-            match += block.text[block_pos - i:]
+            match += block.text[block_pos - i:block_pos]
+          assert len(match) == match_len
           block_prev = block
           block = block_prev.next
           if block is not None:
@@ -311,7 +312,8 @@ def yylex(_class = element.Element, *args, **kwargs):
     i = match_len - len(match)
     if i:
       assert block is not None
-      match += block.text[block_pos - i:]
+      match += block.text[block_pos - i:block_pos]
+    assert len(match) == match_len
 
     for i in yy_dfa_states[state][2]:
       yy_group_text = match

diff --git a/ndcode/pilex/skel/skel_py_groups.py b/ndcode/pilex/skel/skel_py_groups.py
index 6b228b6..c104efd 100644 (file)
--- a/ndcode/pilex/skel/skel_py_groups.py
+++ b/ndcode/pilex/skel/skel_py_groups.py
@@ -260,7 +260,8 @@ def yylex():
         else:
           i = match_len - len(match)
           if i:
-            match += block.text[block_pos - i:]
+            match += block.text[block_pos - i:block_pos]
+          assert len(match) == match_len
           block_prev = block
           block = block_prev.next
           if block is not None:
@@ -287,7 +288,8 @@ def yylex():
     i = match_len - len(match)
     if i:
       assert block is not None
-      match += block.text[block_pos - i:]
+      match += block.text[block_pos - i:block_pos]
+    assert len(match) == match_len
 
     for i in yy_dfa_states[state][2]:
       yy_group_text = match