Improve the Python skeleton to implement REJECT(), unput(), yyless() and so on

diff --git a/skel/skel_py.py b/skel/skel_py.py
index 3c19297..9430b22 100644 (file)
--- a/skel/skel_py.py
+++ b/skel/skel_py.py
@@ -5,6 +5,9 @@ import sys
 
 # GENERATE STARTCONDDECL
 
+class YYReject(Exception):
+  pass
+
 class YYContinue(Exception):
   pass
 
@@ -35,24 +38,62 @@ yy_threads0 = [None]
 yy_threads1 = [None]
 yy_prefix_slop = 1
 
-yytext = None
+yytext = ''
+len_yytext = 0
+
+def REJECT():
+  raise YYReject()
 
 def yyterminate():
   raise YYTerminate()
 
+def yyless(i):
+  assert len_yytext >= i
+  yytext = yytext[:i]
+  len_yytext = i
+
+def unput(text):
+  while len_yytext:
+    block = yy_buffer_stack[-1].next
+    while block is None or block.pos >= len(block.text):
+      if block is None:
+        yy_buffer_stack.pop()
+        block = yy_buffer_stack[-1].next
+        yyin = yy_buffer_stack[-1].file_in
+      else:
+        block = block.next
+        yy_buffer_stack[-1].next = block
+    i = min(len_yytext, len(block.text) - block.pos)
+    block.pos += i
+    len_yytext -= i
+  yy_buffer_stack[-1].next = YYBufferBlock(yy_buffer_stack[-1].next, 0, text)
+
 # GENERATE SECTION2
 
 def yylex():
-  global yyin, yy_threads0, yy_threads1, yy_prefix_slop, yytext
+  global yyin, yy_threads0, yy_threads1, yy_prefix_slop, yytext, len_yytext
 
   # GENERATE SECTION2INITIAL
 
   while True:
-    assert len(yy_threads0) == yy_prefix_slop
-    assert len(yy_threads1) == yy_prefix_slop
+    while len_yytext:
+      block = yy_buffer_stack[-1].next
+      while block is None or block.pos >= len(block.text):
+        if block is None:
+          yy_buffer_stack.pop()
+          block = yy_buffer_stack[-1].next
+          yyin = yy_buffer_stack[-1].file_in
+        else:
+          block = block.next
+          yy_buffer_stack[-1].next = block
+      i = min(len_yytext, len(block.text) - block.pos)
+      block.pos += i
+      len_yytext -= i
+    yytext = ''
+
+    del yy_threads0[yy_prefix_slop:]
     yy_threads0.append(None)
 
-    i = 0
     buffer_ptr = len(yy_buffer_stack) - 1
     block_prev = yy_buffer_stack[buffer_ptr]
     block = block_prev.next
@@ -65,31 +106,32 @@ def yylex():
       state, transition = yy_dfa_actions[action]
       #print('i', i, 'action', action, 'state', state, 'transition', transition)
 
-      j = yy_prefix_slop
+      i = yy_prefix_slop
+      assert len(yy_threads1) == yy_prefix_slop
       for trans in transition:
         if trans[0] == 0: #DFA.TRANSITION_POP:
-          j += trans[1]
+          i += trans[1]
         elif trans[0] == 1: #DFA.TRANSITION_DUP:
-          while j < trans[1]:
+          while i < trans[1]:
             yy_threads0[:0] = [None] * yy_prefix_slop
             yy_threads1[:0] = [None] * yy_prefix_slop
-            j += yy_prefix_slop
+            i += yy_prefix_slop
             yy_prefix_slop *= 2
-          yy_threads0[j - trans[1]:j] = yy_threads0[j:j + trans[1]]
-          j -= trans[1]
+          yy_threads0[i - trans[1]:i] = yy_threads0[i:i + trans[1]]
+          i -= trans[1]
         elif trans[0] == 2: #DFA.TRANSITION_MARK:
-          yy_threads0[j:j + trans[1]] = [
-            (i, trans[2], thread)
-            for thread in yy_threads0[j:j + trans[1]]
+          yy_threads0[i:i + trans[1]] = [
+            (len_yytext, trans[2], thread)
+            for thread in yy_threads0[i:i + trans[1]]
           ]
         elif trans[0] == 3: #DFA.TRANSITION_MOVE:
-          yy_threads1.extend(yy_threads0[j:j + trans[1]])
-          j += trans[1]
+          yy_threads1.extend(yy_threads0[i:i + trans[1]])
+          i += trans[1]
         #elif trans[0] == DFA.TRANSITION_DEL:
         #  del yy_threads1[-trans[1]:]
         else:
           assert False
-      assert j == len(yy_threads0)
+      assert i == len(yy_threads0)
       yy_threads0, yy_threads1 = yy_threads1, yy_threads0
       del yy_threads1[yy_prefix_slop:]
 
@@ -116,6 +158,9 @@ def yylex():
               block_pos = block.pos
             file_in = yy_buffer_stack[buffer_ptr].file_in
         else:
+          i = len_yytext - len(yytext)
+          if i:
+            yytext += block.text[block_pos - i:]
           block_prev = block
           block = block_prev.next
           if block is not None:
@@ -128,49 +173,41 @@ def yylex():
             ord(block.text[block_pos])
           )
         ]
-        i += 1
         block_pos += 1
+        len_yytext += 1
         continue
       # EOF
       if i == 0:
-        del yy_threads0[yy_prefix_slop:]
         try:
           return yy_eof_actions[yystart]()
         except YYTerminate:
           return 0
       break
 
-    accept = yy_dfa_states[state][2]
-    if len(accept) == 0:
-      del yy_threads0[yy_prefix_slop:]
+    i = len_yytext - len(yytext)
+    if i:
+      assert block is not None
+      yytext += block.text[block_pos - i:]
+
+    for i in yy_dfa_states[state][2]:
+      _, _, thread = yy_threads0[yy_prefix_slop + i]
+      #print('thread', thread)
+      len_yytext, mark, thread = thread
+      assert thread is None
+
+      assert len(yytext) >= len_yytext
+      yytext = yytext[:len_yytext]
+      #print('yytext', yytext)
+
+      try:
+        return yy_actions[mark >> 1]()
+      except YYReject:
+        pass
+      except YYContinue:
+        break
+      except YYTerminate:
+        return 0
+    else:
       raise Exception('scanner jammed')
-    _, _, thread = yy_threads0[yy_prefix_slop + accept[0]]
-    del yy_threads0[yy_prefix_slop:]
-    #print('thread', thread)
-    i, mark, thread = thread
-    assert thread is None
-
-    yytext = ''
-    while len(yytext) < i:
-      block = yy_buffer_stack[-1].next
-      while block is None or block.pos >= len(block.text):
-        if block is None:
-          yy_buffer_stack.pop()
-          block = yy_buffer_stack[-1].next
-          yyin = yy_buffer_stack[-1].file_in
-        else:
-          block = block.next
-          yy_buffer_stack[-1].next = block
-      j = min(i - len(yytext), len(block.text) - block.pos)
-      yytext += block.text[block.pos:block.pos + j]
-      block.pos += j
-    #print('yytext', yytext)
-
-    try:
-      return yy_actions[mark >> 1]()
-    except YYContinue:
-      pass
-    except YYTerminate:
-      return 0
 
 # GENERATE SECTION3

diff --git a/tests/cal_py.l b/tests/cal_py.l
index 2922915..0479367 100644 (file)
--- a/tests/cal_py.l
+++ b/tests/cal_py.l
@@ -1,5 +1,4 @@
 %{
-# this is section 1
 NUM = 0x100 
 yylval = None
 %}
@@ -10,8 +9,6 @@ DIGIT [0-9]+\.?|[0-9]*\.[0-9]+
 
 %%
 
-       # this is section 2 initial
-
 [ ]
 {DIGIT}        {
   global yylval
@@ -24,7 +21,6 @@ DIGIT [0-9]+\.?|[0-9]*\.[0-9]+
 
 %%
 
-# this is section 3
 if __name__ == '__main__':
   token = yylex()
   while token != 0: