Make non-element scanner ignore take_space_left which is always True for empty produc...

diff --git a/generate_py.py b/generate_py.py
index e961392..9ab835d 100644 (file)
--- a/generate_py.py
+++ b/generate_py.py
@@ -162,7 +162,7 @@ YYERROR_VERBOSE = {1:s}
                           str(pos1) if pos1 else '',
                           ast_text_to_python(factory_text, '          ')
                         )
-                        for pos0, pos1, factory_text in actions[i][1]
+                        for pos0, pos1, factory_text in actions[i][1] # groups
                       ]
                     ),
                     (
@@ -181,7 +181,10 @@ YYERROR_VERBOSE = {1:s}
                           ''
                         )
                       )
-                    if actions[i][2] else # take_space_left
+                    # note that null productions always have take_space_left ==
+                    # True, so that the action generates a dummy space and then
+                    # the skeleton generates a dummy AST, ignore if not element
+                    if _element and actions[i][2] else # take_space_left
                       ''
                     ),
                     (

diff --git a/skel/skel_py.py b/skel/skel_py.py
index 22a72a4..cb6d892 100644 (file)
--- a/skel/skel_py.py
+++ b/skel/skel_py.py
@@ -30,7 +30,8 @@ import bisect
 # GENERATE TOKENS
 
 yystack = None
-yytoken = None
+yychar = None
+YYEMPTY = -1
 
 yyval = None
 yyloc = None
@@ -41,25 +42,25 @@ yylloc = None
 # GENERATE SECTION2
 
 def yyparse():
-  global yystack, yytoken, yyval, yyloc, yylval, yylloc
+  global yystack, yychar, yyval, yyloc, yylval, yylloc
 
   # GENERATE INITIALACTION
 
   state = 0
   yystack = []
   yylval = None
-  yytoken = -1
+  yychar = -1
   while True:
     #print('state', state, 'yystack', yystack)
     reduce = yy_lr1dfa_states[state][4]
     if reduce == -1:
-      if yytoken == -1:
+      if yychar == -1:
         yylval = None
         yylloc = None
-        yytoken = yylex()
-        #print('yytoken', yytoken, 'yylval', yylval, 'yylloc', yylloc)
+        yychar = yylex()
+        print('yychar', yychar, 'yylval', yylval, 'yylloc', yylloc)
       action = yy_lr1dfa_states[state][1][
-        bisect.bisect_right(yy_lr1dfa_states[state][0], yytoken)
+        bisect.bisect_right(yy_lr1dfa_states[state][0], yychar)
       ]
       if action == -1:
         raise Exception('syntax error')
@@ -67,7 +68,7 @@ def yyparse():
         yystack.append((state, yylval, yylloc))
         state = action >> 1
         #print('shift', state)
-        yytoken = -1
+        yychar = -1
         continue
       reduce = action >> 1
     #print('reduce', reduce)

diff --git a/skel/skel_py_element.py b/skel/skel_py_element.py
index e614553..56b57e2 100644 (file)
--- a/skel/skel_py_element.py
+++ b/skel/skel_py_element.py
@@ -33,7 +33,8 @@ import lex_yy
 # GENERATE TOKENS
 
 yystack = None
-yytoken = None
+yychar = None
+YYEMPTY = -1
 
 yyval = None
 yyloc = None
@@ -46,31 +47,31 @@ yy_element_stack = None
 # GENERATE SECTION2
 
 def yyparse(factory, *args, **kwargs):
-  global yystack, yytoken, yyval, yyloc, yylval, yylloc, yy_element_stack
+  global yystack, yychar, yyval, yyloc, yylval, yylloc, yy_element_stack
 
   # GENERATE INITIALACTION
 
   state = 0
   yystack = []
   yylval = None
-  yytoken = -1
+  yychar = -1
   yy_element_stack = []
   while True:
     #print('state', state, 'yystack', yystack)
     assert len(yy_element_stack) == len(yystack) * 2
     reduce = yy_lr1dfa_states[state][4]
     if reduce == -1:
-      if yytoken == -1:
+      if yychar == -1:
         yylval = None
         yylloc = None
-        yytoken = lex_yy.yylex()
-        #print('yytoken', yytoken, 'yylval', yylval, 'yylloc', yylloc)
+        yychar = lex_yy.yylex()
+        #print('yychar', yychar, 'yylval', yylval, 'yylloc', yylloc)
         #print('lex_yy.yy_element_space')
         #xml.etree.ElementTree.dump(lex_yy.yy_element_space)
         #print('lex_yy.yy_element_token')
         #xml.etree.ElementTree.dump(lex_yy.yy_element_token)
       action = yy_lr1dfa_states[state][1][
-        bisect.bisect_right(yy_lr1dfa_states[state][0], yytoken)
+        bisect.bisect_right(yy_lr1dfa_states[state][0], yychar)
       ]
       if action == -1:
         raise Exception('syntax error')
@@ -85,7 +86,7 @@ def yyparse(factory, *args, **kwargs):
 
         state = action >> 1
         #print('shift', state)
-        yytoken = -1
+        yychar = -1
         continue
       reduce = action >> 1
     #print('reduce', reduce)