Add YYLLOC_DEFAULT() hook for location tracking in Python generator/skeletons

diff --git a/generate_bison.py b/generate_bison.py
index 600b77d..f9e5aac 100644 (file)
--- a/generate_bison.py
+++ b/generate_bison.py
@@ -305,7 +305,7 @@ typedef union YYSTYPE YYSTYPE;
               '''/* GENERATE YYLTYPE BEGIN */
 {0:s}/* GENERATE END */
 '''.format(
-                  '''#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
+                '''#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
 typedef struct YYLTYPE YYLTYPE;
 struct YYLTYPE
 {
@@ -318,8 +318,8 @@ struct YYLTYPE
 # define YYLTYPE_IS_TRIVIAL 1
 #endif
 '''
-                if _ast[0].locations else
-                  ''
+              if _ast[0].locations else
+                ''
               ).replace('YY', type_prefix).replace('yy', name_prefix) # hack
             )
           elif line == '/* GENERATE SECTION1AFTERUNION */\n':

diff --git a/generate_py.py b/generate_py.py
index 90dfe38..1323cc0 100644 (file)
--- a/generate_py.py
+++ b/generate_py.py
@@ -224,7 +224,18 @@ YYERROR_VERBOSE = {1:s}
                   ast_text_to_python(i, '  ')
                   for i in _ast.initial_action_text
                 ]
-              ).replace('(yyval)', '(yylval').replace('(yyloc)', '(yylloc)') # hack
+              ).replace('(yyval)', '(yylval)').replace('(yyloc)', '(yylloc)') # hack
+            )
+          )
+        elif line == '    # GENERATE YYLLOC_DEFAULT\n':
+          fout.write(
+            '''    # GENERATE YYLLOC_DEFAULT BEGIN
+{0:s}    # GENERATE END
+'''.format(
+              '''    YYLLOC_DEFAULT(yyloc, yystack, len_symbols)
+'''
+            if _ast.children[0].locations else
+              ''
             )
           )
         elif line == '# GENERATE SECTION3\n':

diff --git a/skel/skel_py.py b/skel/skel_py.py
index d926a29..e2dfbed 100644 (file)
--- a/skel/skel_py.py
+++ b/skel/skel_py.py
@@ -39,6 +39,23 @@ class YYLTYPE:
     self.first_column = first_column
     self.last_line = last_line
     self.last_column = last_column
+  def copy(self):
+    return YYLTYPE(
+       self.first_line,
+       self.first_column,
+       self.last_line,
+       self.last_column
+    )
+
+def YYLLOC_DEFAULT(current, stack, n):
+  if n:
+    current.first_line = stack[-n][2].first_line
+    current.first_column = stack[-n][2].first_column
+    current.last_line = stack[-1][2].last_line
+    current.last_column = stack[-1][2].last_column
+  else:
+    current.first_line = current.last_line = stack[-1][2].last_line
+    current.first_column = current.last_column = stack[-1][2].last_column
 
 # GENERATE SECTION1
 
@@ -49,7 +66,7 @@ yychar = None
 YYEMPTY = -1
 
 yyval = None
-yyloc = None
+yyloc = YYLTYPE()
 
 yylval = None
 yylloc = YYLTYPE()
@@ -62,16 +79,13 @@ def yyparse():
   # GENERATE INITIALACTION
 
   state = 0
-  yystack = []
-  yylval = None
+  yystack = [(-1, yylval, yylloc.copy())] # kludge for bison compatibility
   yychar = -1
   while True:
     #print('state', state, 'yystack', yystack)
     reduce = yy_lr1dfa_states[state][4]
     if reduce == -1:
       if yychar == -1:
-        yylval = None
-        yylloc = YYLTYPE() # temporary until lex_yy updated, should be None
         yychar = lex_yy.yylex()
         #print('yychar', yychar, 'yylval', yylval, 'yylloc', yylloc, 'lex_yy.yytext', lex_yy.yytext)
       action = yy_lr1dfa_states[state][1][
@@ -80,7 +94,7 @@ def yyparse():
       if action == -1:
         raise Exception('syntax error')
       if (action & 1) == 0:
-        yystack.append((state, yylval, yylloc))
+        yystack.append((state, yylval, yylloc.copy()))
         state = action >> 1
         #print('shift', state)
         yychar = -1
@@ -88,15 +102,16 @@ def yyparse():
       reduce = action >> 1
     #print('reduce', reduce)
     len_symbols, ref_data = yy_lr1dfa_productions[reduce]
+    # GENERATE YYLLOC_DEFAULT
     base = len(yystack) - len_symbols
-    yystack.append((state, None, None))
-    state, yyval, yyloc = yystack[base]
+    yystack.append((state, None, None)) # only has effect if len_symbols == 0
+    state, yyval, _ = yystack[base]
     ref_data()
     del yystack[base:]
     if reduce == 0:
-      assert base == 0
+      assert base == 1
       break
-    yystack.append((state, yyval, yyloc))
+    yystack.append((state, yyval, yyloc.copy()))
     state = yy_lr1dfa_states[state][3][
       bisect.bisect_right(yy_lr1dfa_states[state][2], reduce)
     ]

diff --git a/skel/skel_py_element.py b/skel/skel_py_element.py
index fd6191b..1ef590f 100644 (file)
--- a/skel/skel_py_element.py
+++ b/skel/skel_py_element.py
@@ -41,7 +41,24 @@ class YYLTYPE:
     self.first_column = first_column
     self.last_line = last_line
     self.last_column = last_column
- 
+  def copy(self):
+    return YYLTYPE(
+       self.first_line,
+       self.first_column,
+       self.last_line,
+       self.last_column
+    )
+
+def YYLLOC_DEFAULT(current, stack, n):
+  if n:
+    current.first_line = stack[-n][2].first_line
+    current.first_column = stack[-n][2].first_column
+    current.last_line = stack[-1][2].last_line
+    current.last_column = stack[-1][2].last_column
+  else:
+    current.first_line = current.last_line = stack[-1][2].last_line
+    current.first_column = current.last_column = stack[-1][2].last_column
+
 # GENERATE SECTION1
 
 # GENERATE TOKENS
@@ -51,7 +68,7 @@ yychar = None
 YYEMPTY = -1
 
 yyval = None
-yyloc = None
+yyloc = YYLTYPE()
 
 yylval = None
 yylloc = YYLTYPE()
@@ -66,18 +83,14 @@ def yyparse(factory, *args, **kwargs):
   # GENERATE INITIALACTION
 
   state = 0
-  yystack = []
-  yylval = None
+  yystack = [(-1, yylval, yylloc.copy())] # kludge for bison compatibility
   yychar = -1
   yy_element_stack = []
   while True:
     #print('state', state, 'yystack', yystack)
-    assert len(yy_element_stack) == len(yystack) * 2
     reduce = yy_lr1dfa_states[state][4]
     if reduce == -1:
       if yychar == -1:
-        yylval = None
-        yylloc = YYLTYPE() # temporary until lex_yy updated, should be None
         yychar = lex_yy.yylex()
         #print('yychar', yychar, 'yylval', yylval, 'yylloc', yylloc, 'lex_yy.yytext', lex_yy.yytext)
         #print('lex_yy.yy_element_space')
@@ -90,7 +103,7 @@ def yyparse(factory, *args, **kwargs):
       if action == -1:
         raise Exception('syntax error')
       if (action & 1) == 0:
-        yystack.append((state, yylval, yylloc))
+        yystack.append((state, yylval, yylloc.copy()))
 
         # push space then AST element contiguously onto yy_element_stack
         # even numbered elements are spaces, odd numbered elements are AST
@@ -105,23 +118,24 @@ def yyparse(factory, *args, **kwargs):
       reduce = action >> 1
     #print('reduce', reduce)
     len_symbols, ref_data = yy_lr1dfa_productions[reduce]
+    # GENERATE YYLLOC_DEFAULT
     base = len(yystack) - len_symbols
-    yystack.append((state, None, None))
-    state, yyval, yyloc = yystack[base]
+    yystack.append((state, None, None)) # only has effect if len_symbols == 0
+    state, yyval, _ = yystack[base]
     ref_data()
     del yystack[base:]
     if reduce == 0:
-      assert base == 0
+      assert base == 1
       break
-    yystack.append((state, yyval, yyloc))
+    yystack.append((state, yyval, yyloc.copy()))
 
     # action creates empty space in yy_element_stack[base * 2] if needed
-    assert len(yy_element_stack) > base * 2
+    assert len(yy_element_stack) >= base * 2 - 1
 
-    # concatenate yy_element_stack[base * 2 + 1:] to a single AST element
-    yy_element_stack[base * 2 + 1:] = [
+    # concatenate yy_element_stack[base * 2 - 1:] to a single AST element
+    yy_element_stack[base * 2 - 1:] = [
       element.concatenate(
-        yy_element_stack[base * 2 + 1:],
+        yy_element_stack[base * 2 - 1:],
         element.Element
       )
     ]

diff --git a/tests_ast/cal_py.y b/tests_ast/cal_py.y
index fbc503b..4ec2399 100644 (file)
--- a/tests_ast/cal_py.y
+++ b/tests_ast/cal_py.y
@@ -20,6 +20,9 @@
 import t_def
 import sys
 %}
+
+%locations
+
 %token NUM
 
 %left '+' '-'