Implement re-scanning of action text to insert ValueReference, StackReference etc...
authorNick Downing <nick@ndcode.org>
Fri, 18 Jan 2019 01:58:01 +0000 (12:58 +1100)
committerNick Downing <nick@ndcode.org>
Fri, 18 Jan 2019 01:58:30 +0000 (12:58 +1100)
.gitignore
bootstrap/Makefile
bootstrap/parse-gram.y
bootstrap/scan-code.l [new file with mode: 0644]
bootstrap/scan-gram.l
bootstrap/skel_lex_yy_code.py [new file with mode: 0644]

index 19a1d26..3d4aaf5 100644 (file)
@@ -1,6 +1,7 @@
 __pycache__
 bootstrap/*.xml
 bootstrap/lex_yy.py
+bootstrap/lex_yy_code.py
 bootstrap/y_tab.py
 lex-yacc-examples/*.c
 lex-yacc-examples/*.h
index 38fb1d6..903afa5 100644 (file)
@@ -1,12 +1,16 @@
-all: lex_yy.py y_tab.py
+all: lex_yy.py lex_yy_code.py y_tab.py
 
 lex_yy.py: scan-gram.l skel_lex_yy.py
        ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
        ../../pilex.git/pilex.py --python --skel skel_lex_yy.py $<.xml
 
+lex_yy_code.py: scan-code.l skel_lex_yy_code.py
+       ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
+       ../../pilex.git/pilex.py --python --skel skel_lex_yy_code.py $<.xml
+
 y_tab.py: parse-gram.y skel_y_tab.py
        ../../bootstrap_bison.git/src/bison -o /dev/null $< 2>$<.xml
        ../piyacc.py --python --skel skel_y_tab.py $<.xml
 
 clean:
-       rm -f lex_yy.py y_tab.py *.xml
+       rm -f lex_yy.py lex_yy_code.py y_tab.py *.xml
index fa57397..8daa104 100644 (file)
@@ -19,6 +19,7 @@
 
 %code requires
 {
+  import lex_yy_code
   import state
 
   YYLTYPE = state.location
@@ -270,13 +271,18 @@ prologue_declaration
   : grammar_declaration
   | "%{...%}"
     {
-      #code_piece_append('<AST_Section1_Prologue>%{<AST_Text>')
+      lex_yy_code.code_piece_append('<AST_Section1_Prologue>%{<AST_Text>')
       #muscle_code_grow('post_prologue' if union_seen else 'pre_prologue', translate_code($1, @1, True), @1)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($1)
+      lex_yy_code.sc_context = lex_yy_code.INITIAL # CODE_PROPS_PLAIN
+      lex_yy_code.yylex()
       #code_scanner_last_string_free()
-      #code_piece_append('</AST_Text>%}</AST_Section1_Prologue>')
-      #code_piece_pack()
-      #lex_yy.gram_piece[gram_piece2] = code_piece[code_piece0 - 1]
+      lex_yy_code.code_piece_append('</AST_Text>%}</AST_Section1_Prologue>')
+      lex_yy_code.code_piece_pack()
+      lex_yy.gram_piece[gram_piece2] = lex_yy_code.code_piece[lex_yy_code.code_piece0 - 1]
     }
   | "%<flag>"
     {
@@ -345,13 +351,18 @@ prologue_declaration
     }
   | "%initial-action" "{...}"
     {
-      #code_piece_append('<AST_Text>')
+      lex_yy_code.code_piece_append('<AST_Text>')
       #muscle_code_grow('initial_action', translate_code($2, @2, False), @2)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($2)
+      lex_yy_code.sc_context = lex_yy_code.SC_SYMBOL_ACTION # CODE_PROPS_SYMBOL_ACTION
+      lex_yy_code.yylex()
       #code_scanner_last_string_free()
-      #code_piece_append('</AST_Text>')
-      #code_piece_pack()
-      #lex_yy.gram_piece[gram_piece2 + 2] = code_piece[code_piece0 - 1]
+      lex_yy_code.code_piece_append('</AST_Text>')
+      lex_yy_code.code_piece_pack()
+      lex_yy.gram_piece[gram_piece2 + 2] = lex_yy_code.code_piece[lex_yy_code.code_piece0 - 1]
       insert_after(1, '</AST_Section1_InitialAction>')
       insert_before(0, '<AST_Section1_InitialAction>')
     }
@@ -447,12 +458,10 @@ params
   : params "{...}"
     {
       #add_param(current_param, $2, @2)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
     }
   | "{...}"
     {
       #add_param(current_param, $1, @1)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
     }
   ;
 
@@ -475,12 +484,16 @@ grammar_declaration
     {
       #code = None
       #code_props_symbol_action_init(&code, $2, @2)
-      #code_piece_append('<AST_Text>')
+      lex_yy_code.code_piece_append('<AST_Text>')
       #code_props_translate_code(&code)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
-      #code_piece_append('</AST_Text>')
-      #code_piece_pack()
-      #lex_yy.gram_piece[gram_piece2 + 2] = code_piece[code_piece0 - 1]
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($2)
+      lex_yy_code.yylex()
+      lex_yy_code.code_piece_append('</AST_Text>')
+      lex_yy_code.code_piece_pack()
+      lex_yy.gram_piece[gram_piece2 + 2] = lex_yy_code.code_piece[lex_yy_code.code_piece0 - 1]
       #list = None
       #list = $3
       #while list:
@@ -504,25 +517,35 @@ grammar_declaration
     }
   | "%code" "{...}"
     {
-      #code_piece_append('<AST_BracedCode>{<AST_Text>')
+      lex_yy_code.code_piece_append('<AST_BracedCode>{<AST_Text>')
       #muscle_code_grow('percent_code()', translate_code_braceless($2, @2), @2)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($2[1:-1])
+      lex_yy_code.sc_context = lex_yy_code.INITIAL # CODE_PROPS_PLAIN
+      lex_yy_code.yylex()
       #code_scanner_last_string_free()
-      #code_piece_append('</AST_Text>}</AST_BracedCode>')
-      #code_piece_pack()
-      #lex_yy.gram_piece[gram_piece2 + 2] = code_piece[code_piece0 - 1]
+      lex_yy_code.code_piece_append('</AST_Text>}</AST_BracedCode>')
+      lex_yy_code.code_piece_pack()
+      lex_yy.gram_piece[gram_piece2 + 2] = lex_yy_code.code_piece[lex_yy_code.code_piece0 - 1]
       insert_after(1, '</AST_Section1Or2_Code>')
       insert_before(0, '<AST_Section1Or2_Code><AST_ID />')
     }
   | "%code" ID "{...}"
     {
-      #code_piece_append('<AST_BracedCode>{<AST_Text>')
+      lex_yy_code.code_piece_append('<AST_BracedCode>{<AST_Text>')
       #muscle_percent_code_grow($2, @2, translate_code_braceless($3, @3), @3)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($3[1:-1])
+      lex_yy_code.sc_context = lex_yy_code.INITIAL # CODE_PROPS_PLAIN
+      lex_yy_code.yylex()
       #code_scanner_last_string_free()
-      #code_piece_append('</AST_Text>}</AST_BracedCode>')
-      #code_piece_pack()
-      #lex_yy.gram_piece[gram_piece2 + 4] = code_piece[code_piece0 - 1]
+      lex_yy_code.code_piece_append('</AST_Text>}</AST_BracedCode>')
+      lex_yy_code.code_piece_pack()
+      lex_yy.gram_piece[gram_piece2 + 4] = lex_yy_code.code_piece[lex_yy_code.code_piece0 - 1]
       insert_after(2, '</AST_Section1Or2_Code>')
       insert_before(0, '<AST_Section1Or2_Code>')
     }
@@ -568,13 +591,18 @@ grammar_declaration
   : "%union" union_name "{...}"
     {
       #union_seen = True
-      #code_piece_append('<AST_BracedCode>{<AST_Text>')
+      lex_yy_code.code_piece_append('<AST_BracedCode>{<AST_Text>')
       #muscle_code_grow('union_members', translate_code_braceless($3, @3), @3)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($3[1:-1])
+      lex_yy_code.sc_context = lex_yy_code.INITIAL # CODE_PROPS_PLAIN
+      lex_yy_code.yylex()
       #code_scanner_last_string_free()
-      #code_piece_append('</AST_Text>}</AST_BracedCode>')
-      #code_piece_pack()
-      #lex_yy.gram_piece[gram_piece2 + 4] = code_piece[code_piece0 - 1]
+      lex_yy_code.code_piece_append('</AST_Text>}</AST_BracedCode>')
+      lex_yy_code.code_piece_pack()
+      lex_yy.gram_piece[gram_piece2 + 4] = lex_yy_code.code_piece[lex_yy_code.code_piece0 - 1]
       insert_after(2, '</AST_Section1Or2_Union>')
       insert_before(0, '<AST_Section1Or2_Union>')
     }
@@ -873,11 +901,18 @@ rhs
     }
   | rhs "{...}" named_ref.opt
     {
-      #code_piece_append('<AST_Text>')
+      lex_yy_code.trap = True
+      lex_yy_code.code_piece_append('<AST_Text>')
       #grammar_current_rule_action_append($2, @2, $3, False)
-      #code_piece_append('</AST_Text>')
-      #code_piece_pack()
-      #lex_yy.gram_piece[gram_piece2 + 2] = code_piece[code_piece0 - 1]
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($2)
+      lex_yy_code.sc_context = lex_yy_code.SC_RULE_ACTION # CODE_PROPS_RULE_ACTION
+      lex_yy_code.yylex()
+      lex_yy_code.code_piece_append('</AST_Text>')
+      lex_yy_code.code_piece_pack()
+      lex_yy.gram_piece[gram_piece2 + 2] = lex_yy_code.code_piece[lex_yy_code.code_piece0 - 1]
       insert_after(2, '</AST_Production_Action>')
       insert_before(1, '<AST_Production_Action>')
     }
@@ -1044,7 +1079,12 @@ epilogue.opt
   | "%%" EPILOGUE
     {
       #muscle_code_grow('epilogue', translate_code($2, @2, True), @2)
-      lex_yy.gram_scanner_last_string_free() # normally done in above
+      lex_yy_code.yyin = None
+      lex_yy_code.yy_buffer_stack = [lex_yy_code.YYBufferState()]
+      lex_yy_code.yytext = '' # fool unput()
+      lex_yy_code.unput($2)
+      lex_yy_code.sc_context = lex_yy_code.INITIAL # CODE_PROPS_PLAIN
+      lex_yy_code.yylex()
       #code_scanner_last_string_free()
       insert_after(1, '</AST_Section3>')
       insert_after(0, '<AST_Section3>')
diff --git a/bootstrap/scan-code.l b/bootstrap/scan-code.l
new file mode 100644 (file)
index 0000000..fe8d657
--- /dev/null
@@ -0,0 +1,705 @@
+/* Bison Action Scanner                             -*- C -*-
+
+   Copyright (C) 2006-2015, 2018 Free Software Foundation, Inc.
+
+   This file is part of Bison, the GNU Compiler Compiler.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+%option debug nodefault noinput nounput noyywrap never-interactive
+%option prefix="code_" outfile="lex_yy_code.py"
+
+%{
+  #ref_tail_fields = ''
+  #the_location = 0
+  #loc = &the_location
+  #code_last_string = ''
+  #untyped_var_seen = 0
+
+  code_piece = []
+  code_piece0 = 0
+  code_piece1 = 0
+  #no_handle_action = 0
+  sc_context = -1 # usually an argument to code_lex()
+%}
+
+%x SC_COMMENT SC_LINE_COMMENT
+%x SC_STRING SC_CHARACTER
+%x SC_RULE_ACTION SC_SYMBOL_ACTION
+
+/* POSIX says that a tag must be both an id and a C union member, but
+   historically almost any character is allowed in a tag.  We disallow
+   NUL and newline, as this simplifies our implementation.  We allow
+   "->" as a means to dereference a pointer.  */
+tag      ([^\0\n>]|->)+
+
+/* Zero or more instances of backslash-newline.  Following GCC, allow
+   white space between the backslash and the newline.  */
+splice   (\\[ \f\t\v]*\n)*
+
+/* C style identifier. Must start with letter. Will be used for
+   named symbol references. Shall be kept synchronized with
+   scan-gram.l "letter" and "id". */
+letter    [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]
+id        {letter}({letter}|[-0-9])*
+ref      -?[0-9]+|{id}|"["{id}"]"|"$"
+
+%%
+
+
+%{
+  assert sc_context == SC_SYMBOL_ACTION or sc_context == SC_RULE_ACTION or sc_context == INITIAL
+  BEGIN(sc_context)
+%}
+
+
+
+<SC_COMMENT>
+{
+  "*"{splice}"/" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(sc_context)
+  }
+}
+
+
+  /*--------------------------------------------------------------.
+  | Scanning a line comment.  The initial '//' is already eaten.  |
+  `--------------------------------------------------------------*/
+
+<SC_LINE_COMMENT>
+{
+  "\n" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(sc_context)
+  }
+  {splice}                     #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+}
+
+
+  /*--------------------------------------------.
+  | Scanning user-code characters and strings.  |
+  `--------------------------------------------*/
+
+<SC_CHARACTER,SC_STRING>
+{
+  {splice}|\\{splice}.         #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+}
+
+<SC_CHARACTER>
+{
+  "'" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(sc_context)
+  }
+}
+
+<SC_STRING>
+{
+  "\"" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(sc_context)
+  }
+}
+
+
+<SC_RULE_ACTION,SC_SYMBOL_ACTION>
+{
+  "'" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(SC_CHARACTER)
+  }
+  "\"" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(SC_STRING)
+  }
+  "/"{splice}"*" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(SC_COMMENT)
+  }
+  "/"{splice}"/" {
+    #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    BEGIN(SC_LINE_COMMENT)
+  }
+
+  [$@] {
+    state.complain(yylloc, state.Wother, 'stray \'{0:s}\''.format(yytext))
+    #p__ = None
+    #p__ = yytext
+    #while *p__:
+    #  if *p__ == ord('$'):
+    #    (&obstack_for_string)->temp.i = len('$]['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '$][', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  elif *p__ == ord('@'):
+    #    (&obstack_for_string)->temp.i = len('@@'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@@', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  elif *p__ == ord('['):
+    #    (&obstack_for_string)->temp.i = len('@{'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@{', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  elif *p__ == ord(']'):
+    #    (&obstack_for_string)->temp.i = len('@}'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@}', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  else:
+    #    (_obstack_newchunk(&obstack_for_string, 1), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < 1 else 0, NoneType(*(&obstack_for_string)->next_free++ = *p__)
+    #    break
+    #  p__ += 1
+  }
+}
+
+<SC_RULE_ACTION>
+{
+  "$"("<"{tag}">")?{ref} {
+    #ref_tail_fields = None
+    handle_action_dollar(yytext) #(self->rule, yytext, *loc)
+    #if ref_tail_fields:
+    #  (&obstack_for_string)->temp.i = len(ref_tail_fields), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ref_tail_fields, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  }
+  "@"{ref} {
+    #ref_tail_fields = None
+    handle_action_at(yytext) #(self->rule, yytext, *loc)
+    #if ref_tail_fields:
+    #  (&obstack_for_string)->temp.i = len(ref_tail_fields), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ref_tail_fields, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  }
+}
+
+<SC_SYMBOL_ACTION>
+{
+  "$"("<"{tag}">")?"$" {
+    i = len(yytext) - 1
+    while i >= 2 and yytext[i] != '>':
+      i -= 1
+    i += 1
+    code_piece_append('<AST_Text_ValueReference tag_name="{0:s}">'.format(yytext[2:i]))
+    code_piece_flush(len(yytext)) #escape(yytext)
+    code_piece_append('</AST_Text_ValueReference>')
+    #type_name = None
+    #fetch_type_name(yytext + 1, &type_name, *loc)[-1] = 0
+    #(&obstack_for_string)->temp.i = len(']b4_dollar_dollar('), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ']b4_dollar_dollar(', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #obstack_quote_p = type_name
+    #if obstack_quote_p and obstack_quote_p[0]:
+    #  (&obstack_for_string)->temp.i = len('[['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '[[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #  p__ = None
+    #  p__ = obstack_quote_p
+    #  while *p__:
+    #    if *p__ == ord('$'):
+    #      (&obstack_for_string)->temp.i = len('$]['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '$][', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #      break
+    #    elif *p__ == ord('@'):
+    #      (&obstack_for_string)->temp.i = len('@@'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@@', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #      break
+    #    elif *p__ == ord('['):
+    #      (&obstack_for_string)->temp.i = len('@{'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@{', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #      break
+    #    elif *p__ == ord(']'):
+    #      (&obstack_for_string)->temp.i = len('@}'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@}', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #      break
+    #    else:
+    #      (_obstack_newchunk(&obstack_for_string, 1), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < 1 else 0, NoneType(*(&obstack_for_string)->next_free++ = *p__)
+    #      break
+    #    p__ += 1
+    #  (&obstack_for_string)->temp.i = len(']]'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ']]', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #else:
+    #  (&obstack_for_string)->temp.i = len('[]'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '[]', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    ##(&obstack_for_string)->temp.i = len(')['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ')[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #self->is_value_used = True
+    #yytext += i
+  }
+  "@$" {
+    #(&obstack_for_string)->temp.i = len(']b4_at_dollar['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ']b4_at_dollar[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #muscle_percent_define_ensure('locations', the_location, True)
+    code_piece_append('<AST_Text_ValueLocation>')
+    code_piece_flush(len(yytext))
+    code_piece_append('</AST_Text_ValueLocation>')
+  }
+}
+
+
+<*>
+{
+  /* Escape M4 quoting characters in C code.  */
+  [$@\[\]] {
+    #p__ = None
+    #p__ = yytext
+    #while *p__:
+    #  if *p__ == ord('$'):
+    #    (&obstack_for_string)->temp.i = len('$]['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '$][', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  elif *p__ == ord('@'):
+    #    (&obstack_for_string)->temp.i = len('@@'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@@', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  elif *p__ == ord('['):
+    #    (&obstack_for_string)->temp.i = len('@{'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@{', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  elif *p__ == ord(']'):
+    #    (&obstack_for_string)->temp.i = len('@}'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@}', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+    #    break
+    #  else:
+    #    (_obstack_newchunk(&obstack_for_string, 1), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < 1 else 0, NoneType(*(&obstack_for_string)->next_free++ = *p__)
+    #    break
+    #  p__ += 1
+  }
+
+  /* By default, grow the string obstack with the input.  */
+  .|\n                         #(&obstack_for_string)->temp.i = code_leng, (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, yytext, (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+
+ /* End of processing. */
+  <<EOF>> {
+    #code_last_string = ((_obstack_newchunk(&obstack_for_string, 1), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < 1 else 0, NoneType(*(&obstack_for_string)->next_free++ = ord('\0')), str(((&obstack_for_string)->maybe_empty_object = 1, 0) if (&obstack_for_string)->next_free == (&obstack_for_string)->object_base else 0, (&obstack_for_string)->temp.p = (&obstack_for_string)->object_base, (&obstack_for_string)->next_free = ((&obstack_for_string)->object_base if sizeof ptrdiff_t < 4 else str(0)) + ((&obstack_for_string)->next_free - ((&obstack_for_string)->object_base if sizeof ptrdiff_t < 4 else str(0)) + (&obstack_for_string)->alignment_mask & ~(&obstack_for_string)->alignment_mask), ((&obstack_for_string)->next_free = (&obstack_for_string)->chunk_limit) if int((&obstack_for_string)->next_free - str((&obstack_for_string)->chunk)) > int((&obstack_for_string)->chunk_limit - str((&obstack_for_string)->chunk)) else 0, (&obstack_for_string)->object_base = (&obstack_for_string)->next_free, (&obstack_for_string)->temp.p))
+    #return code_last_string
+  }
+}
+
+%%
+
+#def is_dot_or_dash(ch):
+#  return ch == ord('.') or ch == ord('-')
+#
+#def contains_dot_or_dash(p):
+#  pass
+#  while *p:
+#    if is_dot_or_dash(*p):
+#      return True
+#    p += 1
+#  return False
+#variant_table = None
+#variant_table_size = 0
+#variant_count = 0
+#
+#def variant_table_grow():
+#  variant_count += 1
+#  if variant_count > variant_table_size:
+#    while variant_count > variant_table_size:
+#      variant_table_size = 2 * variant_table_size + 3
+#    variant_table = xnrealloc(variant_table, variant_table_size, sizeof *variant_table)
+#  return &variant_table[variant_count - 1]
+#
+#def variant_table_free():
+#  free(variant_table)
+#  variant_table = None
+#  variant_table_size = variant_count = 0
+#
+#def find_prefix_end(prefix, begin, end):
+#  ptr = begin
+#  pass
+#  while *prefix and ptr != end:
+#    if *prefix != *ptr:
+#      return 0
+#    ++prefix, ++ptr
+#  if *prefix:
+#    return 0
+#  return ptr
+#
+#def variant_add(id, id_loc, symbol_index, cp, cp_end, explicit_bracketing):
+#  prefix_end = None
+#  prefix_end = find_prefix_end(id, cp, cp_end)
+#  if prefix_end and (prefix_end == cp_end or not explicit_bracketing and is_dot_or_dash(*prefix_end)):
+#    r = variant_table_grow()
+#    r->symbol_index = symbol_index
+#    r->id = id
+#    r->loc = id_loc
+#    r->hidden_by = None
+#    r->err = 0
+#    return r
+#  else:
+#    return None
+#
+#def get_at_spec(symbol_index):
+#  at_buf = [None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None]
+#  if symbol_index == 0:
+#    strcpy(at_buf, '$$')
+#  else:
+#    rpl_snprintf(at_buf, sizeof at_buf, '$%u', symbol_index)
+#  return at_buf
+#
+#def show_sub_message(warning, cp, explicit_bracketing, midrule_rhs_index, dollar_or_at, indent, var):
+#  at_spec = get_at_spec(var->symbol_index)
+#  if var->err == 0:
+#    complain_indent(&var->loc, warning, &indent, gettext('refers to: %c%s at %s'), dollar_or_at, var->id, at_spec)
+#  else:
+#    msg_buf = None
+#    tail = '' if explicit_bracketing else cp + len(var->id)
+#    id = None
+#    id_loc = None
+#    if var->hidden_by:
+#      id = var->hidden_by->id
+#      id_loc = var->hidden_by->loc
+#    else:
+#      id = var->id
+#      id_loc = var->loc
+#    _obstack_begin(&msg_buf, 0, 0, xmalloc, free)
+#    obstack_printf(&msg_buf, gettext('possibly meant: %c'), dollar_or_at)
+#    if contains_dot_or_dash(id):
+#      obstack_printf(&msg_buf, '[%s]', id)
+#    else:
+#      (&msg_buf)->temp.i = len(id), (_obstack_newchunk(&msg_buf, (&msg_buf)->temp.i), 0) if int((&msg_buf)->chunk_limit - (&msg_buf)->next_free) < (&msg_buf)->temp.i else 0, memcpy((&msg_buf)->next_free, id, (&msg_buf)->temp.i), (&msg_buf)->next_free += (&msg_buf)->temp.i, NoneType(0)
+#    (&msg_buf)->temp.i = len(tail), (_obstack_newchunk(&msg_buf, (&msg_buf)->temp.i), 0) if int((&msg_buf)->chunk_limit - (&msg_buf)->next_free) < (&msg_buf)->temp.i else 0, memcpy((&msg_buf)->next_free, tail, (&msg_buf)->temp.i), (&msg_buf)->next_free += (&msg_buf)->temp.i, NoneType(0)
+#    if var->err & 1 << 0:
+#      obstack_printf(&msg_buf, gettext(', hiding %c'), dollar_or_at)
+#      if contains_dot_or_dash(var->id):
+#        obstack_printf(&msg_buf, '[%s]', var->id)
+#      else:
+#        (&msg_buf)->temp.i = len(var->id), (_obstack_newchunk(&msg_buf, (&msg_buf)->temp.i), 0) if int((&msg_buf)->chunk_limit - (&msg_buf)->next_free) < (&msg_buf)->temp.i else 0, memcpy((&msg_buf)->next_free, var->id, (&msg_buf)->temp.i), (&msg_buf)->next_free += (&msg_buf)->temp.i, NoneType(0)
+#      (&msg_buf)->temp.i = len(tail), (_obstack_newchunk(&msg_buf, (&msg_buf)->temp.i), 0) if int((&msg_buf)->chunk_limit - (&msg_buf)->next_free) < (&msg_buf)->temp.i else 0, memcpy((&msg_buf)->next_free, tail, (&msg_buf)->temp.i), (&msg_buf)->next_free += (&msg_buf)->temp.i, NoneType(0)
+#    obstack_printf(&msg_buf, gettext(' at %s'), at_spec)
+#    if var->err & 1 << 2:
+#      obstack_printf(&msg_buf, gettext(', cannot be accessed from mid-rule action at $%d'), midrule_rhs_index)
+#    complain_indent(&id_loc, warning, &indent, '%s', ((_obstack_newchunk(&msg_buf, 1), 0) if int((&msg_buf)->chunk_limit - (&msg_buf)->next_free) < 1 else 0, NoneType(*(&msg_buf)->next_free++ = ord('\0')), str(((&msg_buf)->maybe_empty_object = 1, 0) if (&msg_buf)->next_free == (&msg_buf)->object_base else 0, (&msg_buf)->temp.p = (&msg_buf)->object_base, (&msg_buf)->next_free = ((&msg_buf)->object_base if sizeof ptrdiff_t < 4 else str(0)) + ((&msg_buf)->next_free - ((&msg_buf)->object_base if sizeof ptrdiff_t < 4 else str(0)) + (&msg_buf)->alignment_mask & ~(&msg_buf)->alignment_mask), ((&msg_buf)->next_free = (&msg_buf)->chunk_limit) if int((&msg_buf)->next_free - str((&msg_buf)->chunk)) > int((&msg_buf)->chunk_limit - str((&msg_buf)->chunk)) else 0, (&msg_buf)->object_base = (&msg_buf)->next_free, (&msg_buf)->temp.p)))
+#    (&msg_buf)->temp.p = list(0), NoneType((&msg_buf)->next_free = (&msg_buf)->object_base = str((&msg_buf)->temp.p)) if (&msg_buf)->temp.p > list((&msg_buf)->chunk) and (&msg_buf)->temp.p < list((&msg_buf)->chunk_limit) else _obstack_free(&msg_buf, (&msg_buf)->temp.p)
+#
+#def show_sub_messages(warning, cp, explicit_bracketing, midrule_rhs_index, dollar_or_at, indent):
+#  i = None
+#  i = 0
+#  while i < variant_count:
+#    show_sub_message(warning | silent, cp, explicit_bracketing, midrule_rhs_index, dollar_or_at, indent, &variant_table[i])
+#    i += 1
+
+def parse_ref(cp): #, rule, rule_length, midrule_rhs_index, text, text_loc, dollar_or_at):
+  #l = None
+  #cp_end = None
+  #explicit_bracketing = None
+  #i = None
+  #valid_variants = 0
+  #valid_variant_index = 0
+  if '$' == cp[0]:
+    return 0x80000001
+  if len(cp) >= 1 and cp[0] in '0123456789' or len(cp) >= 2 and cp[0] == '-' and cp[1] in '0123456789':
+    num = int(cp, 10)
+    if True: #0x80000002 + rule_length <= num and num <= rule_length:
+      return num
+    #else:
+    #  complain(&text_loc, complaint, gettext('integer out of range: %s'), quote(text))
+    #  return 0x80000000
+  #if ord('[') == *cp:
+  #  p = None
+  #  p = ++cp
+  #  while *p != ord(']'):
+  #    p += 1
+  #    continue
+  #    p += 1
+  #  cp_end = p
+  #  explicit_bracketing = True
+  #else:
+  #  p = None
+  #  p = cp
+  #  while *p:
+  #    if is_dot_or_dash(*p):
+  #      ref_tail_fields = p
+  #      break
+  #    p += 1
+  #  p = cp
+  #  while *p:
+  #    p += 1
+  #    continue
+  #    p += 1
+  #  cp_end = p
+  #  explicit_bracketing = False
+  #symbol_index = None
+  #variant_count = 0
+  #symbol_index = 0, l = rule
+  #while not symbol_list_null(l):
+  #  var = None
+  #  if l->content_type != SYMLIST_SYMBOL:
+  #    ++symbol_index, l = l->next
+  #    continue
+  #  var = variant_add(l->content.sym->tag, l->sym_loc, symbol_index, cp, cp_end, explicit_bracketing)
+  #  if var and l->named_refXXX:
+  #    var->hidden_by = l->named_refXXX
+  #  if l->named_refXXX:
+  #    variant_add(l->named_refXXX->id, l->named_refXXX->loc, symbol_index, cp, cp_end, explicit_bracketing)
+  #  ++symbol_index, l = l->next
+  #i = 0
+  #while i < variant_count:
+  #  var = &variant_table[i]
+  #  symbol_index = var->symbol_index
+  #  if midrule_rhs_index != 0 and (symbol_index == 0 or midrule_rhs_index < symbol_index):
+  #    var->err |= 1 << 2
+  #  if not explicit_bracketing and contains_dot_or_dash(var->id):
+  #    var->err |= 1 << 1
+  #  if var->hidden_by:
+  #    var->err |= 1 << 0
+  #  if not var->err:
+  #    valid_variant_index = i
+  #    valid_variants += 1
+  #  i += 1
+  #if valid_variants == 0:
+  #  len = cp_end - cp if explicit_bracketing or not ref_tail_fields else ref_tail_fields - cp
+  #  indent = 0
+  #  complain_indent(&text_loc, complaint, &indent, gettext('invalid reference: %s'), quote(text))
+  #  indent += 4
+  #  if len == 0:
+  #    sym_loc = text_loc
+  #    sym_loc.start.column += 1
+  #    sym_loc.end = sym_loc.start
+  #    complain_indent(&sym_loc, complaint, &indent, gettext('syntax error after \'%c\', expecting integer, ' 'letter, \'_\', \'[\', or \'$\''), dollar_or_at)
+  #  else:
+  #    if midrule_rhs_index:
+  #      complain_indent(&rule->locationXXX, complaint, &indent, gettext('symbol not found in production before $%d: ' '%.*s'), midrule_rhs_index, len, cp)
+  #    else:
+  #      complain_indent(&rule->locationXXX, complaint, &indent, gettext('symbol not found in production: %.*s'), len, cp)
+  #  if variant_count > 0:
+  #    show_sub_messages(complaint, cp, explicit_bracketing, midrule_rhs_index, dollar_or_at, indent)
+  #  return 0x80000000
+  #elif valid_variants == 1:
+  #  indent = 0
+  #  if variant_count > 1:
+  #    complain_indent(&text_loc, Wother, &indent, gettext('misleading reference: %s'), quote(text))
+  #    show_sub_messages(Wother, cp, explicit_bracketing, midrule_rhs_index, dollar_or_at, indent + 4)
+  #  symbol_index = variant_table[valid_variant_index].symbol_index
+  #  return 0x80000001 if symbol_index == midrule_rhs_index else symbol_index
+  #else:
+  #  indent = 0
+  #  complain_indent(&text_loc, complaint, &indent, gettext('ambiguous reference: %s'), quote(text))
+  #  show_sub_messages(complaint, cp, explicit_bracketing, midrule_rhs_index, dollar_or_at, indent + 4)
+  #  return 0x80000000
+  return 0x80000000 # Nick added, normally unreachable
+
+#max_left_semantic_context = 0
+#def fetch_type_name(cp, type_name, dollar_loc):
+#  if *cp == ord('<'):
+#    *type_name = ++cp
+#    while *cp != ord('>') or cp[-1] == ord('-'):
+#      cp += 1
+#    cp += 1
+#    if untyped_var_seen:
+#      complain(&dollar_loc, complaint, gettext('explicit type given in untyped grammar'))
+#    tag_seen = True
+#  return cp
+
+def handle_action_dollar(text): #(rule, text, dollar_loc):
+  #type_name = None
+  cp = text[1:]
+  #effective_rule = None
+  #effective_rule_length = None
+  #n = None
+  #if rule->midrule_parent_rule:
+  #  effective_rule = rule->midrule_parent_rule
+  #  effective_rule_length = rule->midrule_parent_rhs_index - 1
+  #else:
+  #  effective_rule = rule
+  #  effective_rule_length = symbol_list_length(rule->next)
+  #cp = fetch_type_name(cp, &type_name, dollar_loc)
+  n = parse_ref(cp) #, effective_rule, effective_rule_length, rule->midrule_parent_rhs_index, text, dollar_loc, ord('$'))
+  if n == 0x80000001:
+    i = len(yytext) - 1
+    while i >= 2 and yytext[i] != '>':
+      i -= 1
+    i += 1
+    code_piece_append('<AST_Text_ValueReference tag_name="{0:s}">'.format(yytext[2:i]))
+    code_piece_flush(len(yytext))
+    code_piece_append('</AST_Text_ValueReference>')
+  else:
+    i = len(yytext) - 1
+    while i >= 2 and yytext[i] != '>':
+      i -= 1
+    i += 1
+    code_piece_append('<AST_Text_StackReference tag_name="{0:s}" index="{1:d}">'.format(yytext[2:i], n))
+    code_piece_flush(len(yytext))
+    code_piece_append('</AST_Text_StackReference>')
+  #if no_handle_action:
+  #  return True
+  #if type_name:
+  #  cp[-1] = ord('\0')
+  #if n == 0x80000000:
+  #  break
+  #elif n == 0x80000001:
+  #  if not type_name:
+  #    type_name = symbol_list_n_type_name_get(rule, 0)
+  #  if not type_name:
+  #    if union_seen | tag_seen:
+  #      if rule->midrule_parent_rule:
+  #        complain(&dollar_loc, complaint, gettext('$$ for the midrule at $%d of %s' ' has no declared type'), rule->midrule_parent_rhs_index, quote(effective_rule->content.sym->tag))
+  #      else:
+  #        complain(&dollar_loc, complaint, gettext('$$ of %s has no declared type'), quote(rule->content.sym->tag))
+  #    else:
+  #      untyped_var_seen = True
+  #  (&obstack_for_string)->temp.i = len(']b4_lhs_value('), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ']b4_lhs_value(', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  obstack_quote_p = type_name
+  #  if obstack_quote_p and obstack_quote_p[0]:
+  #    (&obstack_for_string)->temp.i = len('[['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '[[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #    p__ = None
+  #    p__ = obstack_quote_p
+  #    while *p__:
+  #      if *p__ == ord('$'):
+  #        (&obstack_for_string)->temp.i = len('$]['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '$][', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      elif *p__ == ord('@'):
+  #        (&obstack_for_string)->temp.i = len('@@'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@@', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      elif *p__ == ord('['):
+  #        (&obstack_for_string)->temp.i = len('@{'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@{', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      elif *p__ == ord(']'):
+  #        (&obstack_for_string)->temp.i = len('@}'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@}', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      else:
+  #        (_obstack_newchunk(&obstack_for_string, 1), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < 1 else 0, NoneType(*(&obstack_for_string)->next_free++ = *p__)
+  #        break
+  #      p__ += 1
+  #    (&obstack_for_string)->temp.i = len(']]'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ']]', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  else:
+  #    (&obstack_for_string)->temp.i = len('[]'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '[]', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  (&obstack_for_string)->temp.i = len(')['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ')[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  rule->action_props.is_value_used = True
+  #  break
+  #else:
+  #  if max_left_semantic_context < 1 - n:
+  #    max_left_semantic_context = 1 - n
+  #  if not type_name and 0 < n:
+  #    type_name = symbol_list_n_type_name_get(effective_rule, n)
+  #  if not type_name:
+  #    if union_seen | tag_seen:
+  #      complain(&dollar_loc, complaint, gettext('$%s of %s has no declared type'), cp, quote(effective_rule->content.sym->tag))
+  #    else:
+  #      untyped_var_seen = True
+  #  obstack_printf(&obstack_for_string, ']b4_rhs_value(%d, %d, ', effective_rule_length, n)
+  #  obstack_quote_p = type_name
+  #  if obstack_quote_p and obstack_quote_p[0]:
+  #    (&obstack_for_string)->temp.i = len('[['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '[[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #    p__ = None
+  #    p__ = obstack_quote_p
+  #    while *p__:
+  #      if *p__ == ord('$'):
+  #        (&obstack_for_string)->temp.i = len('$]['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '$][', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      elif *p__ == ord('@'):
+  #        (&obstack_for_string)->temp.i = len('@@'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@@', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      elif *p__ == ord('['):
+  #        (&obstack_for_string)->temp.i = len('@{'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@{', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      elif *p__ == ord(']'):
+  #        (&obstack_for_string)->temp.i = len('@}'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '@}', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #        break
+  #      else:
+  #        (_obstack_newchunk(&obstack_for_string, 1), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < 1 else 0, NoneType(*(&obstack_for_string)->next_free++ = *p__)
+  #        break
+  #      p__ += 1
+  #    (&obstack_for_string)->temp.i = len(']]'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ']]', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  else:
+  #    (&obstack_for_string)->temp.i = len('[]'), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, '[]', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  (&obstack_for_string)->temp.i = len(')['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ')[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  if 0 < n:
+  #    symbol_list_n_get(effective_rule, n)->action_props.is_value_used = True
+  #  break
+
+def handle_action_at(text): #(rule, text, at_loc):
+  cp = text[1:]
+  #effective_rule = None
+  #effective_rule_length = None
+  #n = None
+  #if rule->midrule_parent_rule:
+  #  effective_rule = rule->midrule_parent_rule
+  #  effective_rule_length = rule->midrule_parent_rhs_index - 1
+  #else:
+  #  effective_rule = rule
+  #  effective_rule_length = symbol_list_length(rule->next)
+  #muscle_percent_define_ensure('locations', at_loc, True)
+  n = parse_ref(cp) #, effective_rule, effective_rule_length, rule->midrule_parent_rhs_index, text, at_loc, ord('@'))
+  if n == 0x80000001:
+    code_piece_append('<AST_Text_ValueLocation>')
+    code_piece_flush(len(yytext))
+    code_piece_append('</AST_Text_ValueLocation>')
+  else:
+    code_piece_append('<AST_Text_StackLocation index="{0:d}">'.format(n))
+    code_piece_flush(len(yytext))
+    code_piece_append('</AST_Text_StackLocation>')
+  #if no_handle_action:
+  #  return True
+  #if n == 0x80000000:
+  #  break
+  #elif n == 0x80000001:
+  #  (&obstack_for_string)->temp.i = len(']b4_lhs_location['), (_obstack_newchunk(&obstack_for_string, (&obstack_for_string)->temp.i), 0) if int((&obstack_for_string)->chunk_limit - (&obstack_for_string)->next_free) < (&obstack_for_string)->temp.i else 0, memcpy((&obstack_for_string)->next_free, ']b4_lhs_location[', (&obstack_for_string)->temp.i), (&obstack_for_string)->next_free += (&obstack_for_string)->temp.i, NoneType(0)
+  #  break
+  #else:
+  #  obstack_printf(&obstack_for_string, ']b4_rhs_location(%d, %d)[', effective_rule_length, n)
+  #  break
+
+#def translate_action(self, sc_context):
+#  res = None
+#  initialized = False
+#  if not initialized:
+#    _obstack_begin(&obstack_for_string, 0, 0, xmalloc, free)
+#    yy_flex_debug = 0
+#    initialized = True
+#  loc->start = loc->end = self->locationXXX.start
+#  yy_switch_to_buffer(yy_scan_string(self->code))
+#  res = code_lex(self, sc_context)
+#  yy_delete_buffer(YY_CURRENT_BUFFER)
+#  return res
+#
+#def code_props_none_init(self):
+#  *self = code_props_none
+#code_props_none = [CODE_PROPS_NONE, None, [[None, 0, 0], [None, 0, 0]], False, False, False, None, None]
+#
+#def code_props_plain_init(self, code, code_loc):
+#  code_props_none_init(self)
+#  self->kind = CODE_PROPS_PLAIN
+#  self->code = code
+#  self->locationXXX = code_loc
+#
+#def code_props_symbol_action_init(self, code, code_loc):
+#  code_props_none_init(self)
+#  self->kind = CODE_PROPS_SYMBOL_ACTION
+#  self->code = code
+#  self->locationXXX = code_loc
+#
+#def code_props_rule_action_init(self, code, code_loc, rule, name, is_predicate):
+#  code_props_none_init(self)
+#  self->kind = CODE_PROPS_RULE_ACTION
+#  self->code = code
+#  self->locationXXX = code_loc
+#  self->rule = rule
+#  self->named_refXXX = name
+#  self->is_predicate = is_predicate
+#
+#def code_props_translate_code(self):
+#  if self->kind == CODE_PROPS_NONE:
+#    break
+#  elif self->kind == CODE_PROPS_PLAIN:
+#    self->code = translate_action(self, INITIAL)
+#    break
+#  elif self->kind == CODE_PROPS_SYMBOL_ACTION:
+#    self->code = translate_action(self, SC_SYMBOL_ACTION)
+#    break
+#  elif self->kind == CODE_PROPS_RULE_ACTION:
+#    self->code = translate_action(self, SC_RULE_ACTION)
+#    break
+#
+#def code_scanner_last_string_free():
+#  (&obstack_for_string)->temp.p = list(code_last_string), NoneType((&obstack_for_string)->next_free = (&obstack_for_string)->object_base = str((&obstack_for_string)->temp.p)) if (&obstack_for_string)->temp.p > list((&obstack_for_string)->chunk) and (&obstack_for_string)->temp.p < list((&obstack_for_string)->chunk_limit) else _obstack_free(&obstack_for_string, (&obstack_for_string)->temp.p)
+#
+#def code_scanner_free():
+#  (&obstack_for_string)->temp.p = list(0), NoneType((&obstack_for_string)->next_free = (&obstack_for_string)->object_base = str((&obstack_for_string)->temp.p)) if (&obstack_for_string)->temp.p > list((&obstack_for_string)->chunk) and (&obstack_for_string)->temp.p < list((&obstack_for_string)->chunk_limit) else _obstack_free(&obstack_for_string, (&obstack_for_string)->temp.p)
+#  variant_table_free()
+#  yy_delete_buffer(YY_CURRENT_BUFFER)
+
+def code_piece_append(str):
+  code_piece.append(str)
+
+def code_piece_insert(n, str):
+  code_piece[n:n] = [str]
+
+xml_escape = {'<': '&lt;', '>': '&gt;', '&': '&amp;'}
+def code_piece_escape(str):
+  code_piece.append(''.join([xml_escape.get(i, i) for i in str]))
+
+def code_piece_flush(n):
+  global yytext
+  code_piece_escape(yytext[:n])
+  yytext = yytext[n:]
+
+def code_piece_pack():
+  global code_piece0
+  code_piece[code_piece0:] = [''.join(code_piece[code_piece0:])]
+  code_piece0 += 1
index da36550..af28282 100644 (file)
@@ -32,8 +32,8 @@
   bracketed_id_context_state = -1
 
   obstack_for_string = []
-  def gram_scanner_last_string_free():
-    del obstack_for_string[:]
+  #def gram_scanner_last_string_free():
+  #  del obstack_for_string[:]
 
   gram_piece = []
   gram_piece0 = 0
diff --git a/bootstrap/skel_lex_yy_code.py b/bootstrap/skel_lex_yy_code.py
new file mode 100644 (file)
index 0000000..4e15cf1
--- /dev/null
@@ -0,0 +1,287 @@
+import bisect
+import sys
+
+# GENERATE SECTION1
+
+# GENERATE STARTCONDDECL
+
+class YYReject(Exception):
+  pass
+
+class YYContinue(Exception):
+  pass
+
+class YYTerminate(Exception):
+  pass
+
+class YYBufferList:
+  def __init__(self, next = None):
+    self.next = next
+
+class YYBufferBlock(YYBufferList):
+  def __init__(self, next = None, pos = 0, text = ''):
+    YYBufferList.__init__(self, next)
+    self.pos = pos
+    self.text = text
+
+class YYBufferState(YYBufferList):
+  def __init__(self, next = None, file_in = None, at_bol = True):
+    YYBufferList.__init__(self, next)
+    self.file_in = file_in
+    self.at_bol = at_bol
+
+yyin = sys.stdin
+yyout = sys.stdout
+yy_buffer_stack = [YYBufferState()]
+
+yystart = INITIAL
+yystart_stack = []
+yy_threads0 = [None]
+yy_threads1 = [None]
+yy_prefix_slop = 1
+
+yy_group_text = None
+yy_group_stack = None
+yy_groups = None
+yy_groups_by_name = None
+yy_action = None
+yytext = ''
+yytext_len = 0
+
+YY_NULL = 0
+
+def REJECT():
+  raise YYReject()
+
+def yyterminate():
+  raise YYTerminate()
+
+def yyless(i):
+  global yytext, yytext_len
+  assert yytext_len >= i
+  yytext = yytext[:i]
+  yytext_len = i
+
+def unput(text):
+  global yyin, yytext_len
+  code_piece_flush(len(yytext))
+  while yytext_len:
+    block = yy_buffer_stack[-1].next
+    while block is None or block.pos >= len(block.text):
+      if block is None:
+        yy_buffer_stack.pop()
+        block = yy_buffer_stack[-1].next
+        yyin = yy_buffer_stack[-1].file_in
+      else:
+        block = block.next
+        yy_buffer_stack[-1].next = block
+    i = min(yytext_len, len(block.text) - block.pos)
+    block.pos += i
+    yytext_len -= i
+  yy_buffer_stack[-1].next = YYBufferBlock(yy_buffer_stack[-1].next, 0, text)
+
+def ECHO():
+  yyout.write(yytext)
+
+def yy_rule_start():
+  global yytext, yytext_len
+  yytext = yy_group_text[:yy_group_stack[-1]]
+  yytext_len = yy_group_stack[-1]
+  del yy_group_stack[-2:]
+  # note that this should also be done after yyless() and REJECT(),
+  # and state should be saved in case they result in a null string,
+  # however, it doesn't seem to be in flex, maintain compatibility:
+  if len(yytext):
+    yy_buffer_stack[-1].at_bol = yytext[-1] == '\n'
+
+def yy_group_end():
+  pass
+
+def BEGIN(start):
+  global yystart
+  yystart = start
+
+def YY_START():
+  return yystart
+
+def yy_push_state(start):
+  global yystart
+  yystart_stack.append(yystart)
+  yystart = start
+
+def yy_pop_state():
+  global yystart
+  yystart = yystart_stack.pop()
+
+def YY_AT_BOL():
+  return yy_buffer_stack[-1].at_bol
+
+def yy_set_bol(at_bol):
+  yy_buffer_stack[-1].at_bol = at_bol
+
+# GENERATE SECTION2
+
+def yylex():
+  global \
+    yyin, \
+    yy_threads0, \
+    yy_threads1, \
+    yy_prefix_slop, \
+    yy_group_text, \
+    yy_group_stack, \
+    yy_action, \
+    yytext, \
+    yytext_len
+
+  # GENERATE SECTION2INITIAL
+
+  while True:
+    while yytext_len:
+      block = yy_buffer_stack[-1].next
+      while block is None or block.pos >= len(block.text):
+        if block is None:
+          yy_buffer_stack.pop()
+          block = yy_buffer_stack[-1].next
+          yyin = yy_buffer_stack[-1].file_in
+        else:
+          block = block.next
+          yy_buffer_stack[-1].next = block
+      i = min(yytext_len, len(block.text) - block.pos)
+      block.pos += i
+      yytext_len -= i
+
+    match = ''
+    match_len = 0
+
+    del yy_threads0[yy_prefix_slop:]
+    yy_threads0.append(None)
+
+    buffer_ptr = len(yy_buffer_stack) - 1
+    block_prev = yy_buffer_stack[buffer_ptr]
+    block = block_prev.next
+    if block is not None:
+      block_pos = block.pos
+
+    action = yy_dfa_start_action[
+      yystart * 2 + int(yy_buffer_stack[-1].at_bol)
+    ]
+    while action != -1:
+      state, transition = yy_dfa_actions[action]
+      #print('i', i, 'action', action, 'state', state, 'transition', transition)
+
+      i = yy_prefix_slop
+      assert len(yy_threads1) == yy_prefix_slop
+      for trans in transition:
+        if trans[0] == 0: #DFA.TRANSITION_POP:
+          i += trans[1]
+        elif trans[0] == 1: #DFA.TRANSITION_DUP:
+          while i < trans[1]:
+            yy_threads0[:0] = [None] * yy_prefix_slop
+            yy_threads1[:0] = [None] * yy_prefix_slop
+            i += yy_prefix_slop
+            yy_prefix_slop *= 2
+          yy_threads0[i - trans[1]:i] = yy_threads0[i:i + trans[1]]
+          i -= trans[1]
+        elif trans[0] == 2: #DFA.TRANSITION_MARK:
+          yy_threads0[i:i + trans[1]] = [
+            (match_len, trans[2], thread)
+            for thread in yy_threads0[i:i + trans[1]]
+          ]
+        elif trans[0] == 3: #DFA.TRANSITION_MOVE:
+          yy_threads1.extend(yy_threads0[i:i + trans[1]])
+          i += trans[1]
+        #elif trans[0] == DFA.TRANSITION_DEL:
+        #  del yy_threads1[-trans[1]:]
+        else:
+          assert False
+      assert i == len(yy_threads0)
+      yy_threads0, yy_threads1 = yy_threads1, yy_threads0
+      del yy_threads1[yy_prefix_slop:]
+
+      if state == 0:
+        # there is only one match, which is complete
+        assert len(yy_threads0) == yy_prefix_slop + 1
+        assert yy_dfa_states[state][2] == [0]
+        break
+
+      yy_buffer_stack[-1].file_in = yyin
+      while block is None or block_pos >= len(block.text):
+        if block is None:
+          file_in = yy_buffer_stack[buffer_ptr].file_in
+          text = '' if file_in is None else file_in.readline()
+          if len(text):
+            block = YYBufferBlock(None, 0, text)
+            block_pos = 0
+            block_prev.next = block
+          else:
+            # do not re-attempt read once EOF is reached
+            yy_buffer_stack[buffer_ptr].file_in = None
+            yyin = yy_buffer_stack[-1].file_in
+            buffer_ptr -= 1
+            if buffer_ptr < 0:
+              break # EOF
+            block_prev = yy_buffer_stack[buffer_ptr]
+            block = block_prev.next
+            if block is not None:
+              block_pos = block.pos
+        else:
+          i = match_len - len(match)
+          if i:
+            match += block.text[block_pos - i:]
+          block_prev = block
+          block = block_prev.next
+          if block is not None:
+            block_pos = block.pos
+      else: 
+        #print('block_pos', block_pos, 'block.text', block.text)
+        action = yy_dfa_states[state][1][
+          bisect.bisect_right(
+            yy_dfa_states[state][0],
+            ord(block.text[block_pos])
+          )
+        ]
+        block_pos += 1
+        match_len += 1
+        continue
+      # EOF
+      if i == 0:
+        try:
+          return yy_eof_actions[yystart]()
+        except YYTerminate:
+          return 0
+      break
+
+    i = match_len - len(match)
+    if i:
+      assert block is not None
+      match += block.text[block_pos - i:]
+
+    for i in yy_dfa_states[state][2]:
+      yy_group_text = match
+      yy_group_stack = []
+      yy_groups = None
+      yy_groups_by_name = None
+      yy_action = None
+      yytext = None
+      yytext_len = None
+
+      thread = yy_threads0[yy_prefix_slop + i]
+      #print('thread', thread)
+      while thread is not None:
+        pos, ref_data, thread = thread
+        yy_group_stack.append(pos)
+        ref_data()
+
+      try:
+        return yy_action()
+      except YYReject:
+        pass
+      except YYContinue:
+        code_piece_escape(yytext)
+        break
+      except YYTerminate:
+        return 0
+    else:
+      raise Exception('scanner jammed')
+
+# GENERATE SECTION3