Add location tracking, syntax error and invalid character reporting

diff --git a/ndcode/pitree/cli.py b/ndcode/pitree/cli.py
index b3fead9..fe108a1 100755 (executable)
--- a/ndcode/pitree/cli.py
+++ b/ndcode/pitree/cli.py
@@ -79,6 +79,7 @@ def main():
 
   with open(in_file) as fin:
     lex_yy.yyin = fin
+    y_tab.in_file = in_file
     ast = y_tab.yyparse(t_def.AST)
   #element.serialize(ast, 'a.xml', 'utf-8')
   #ast = element.deserialize('a.xml', t_def.factory, 'utf-8')

diff --git a/ndcode/pitree/pitree.l b/ndcode/pitree/pitree.l
index e167942..63a4d2c 100644 (file)
--- a/ndcode/pitree/pitree.l
+++ b/ndcode/pitree/pitree.l
@@ -20,6 +20,27 @@
   from ndcode.pitree import element
   from ndcode.pitree import t_def
   from ndcode.pitree import y_tab
+
+  def YY_USER_ACTION():
+    line = y_tab.yylloc.last_line
+    column = y_tab.yylloc.last_column
+    y_tab.yylloc.first_line = line
+    y_tab.yylloc.first_column = column
+    p0 = 0
+    p = 0
+    while p < len(yytext):
+      if yytext[p] == '\n':
+        line += 1
+        column = 1
+        p0 = p + 1
+      elif yytext[p] == '\t':
+        column += p - p0
+        column += 8 - ((column - 1) & 7)
+        p0 = p + 1
+      p += 1
+    column += p - p0
+    y_tab.yylloc.last_line = line
+    y_tab.yylloc.last_column = column
 %}
 
 %option nodefault
@@ -143,3 +164,9 @@
 <SECTION3>{
   .*\n?
 }
+
+<INITIAL,SECTION2>{
+  .|\n {
+    y_tab.yyerror(y_tab.yylloc, 'invalid character')
+  }
+}

diff --git a/ndcode/pitree/pitree.y b/ndcode/pitree/pitree.y
index a1b5768..c22c037 100644 (file)
--- a/ndcode/pitree/pitree.y
+++ b/ndcode/pitree/pitree.y
@@ -17,9 +17,13 @@
  */
 
 %{
+  import sys
   from ndcode.pitree import element
   from ndcode.pitree import lex_yy
   from ndcode.pitree import t_def
+
+  # set this before calling yyparse(), for error messages
+  in_file = None
 %}
 
 %token CODEBLOCK_START CODEBLOCK_END SECTION2_START SECTION3_START
@@ -123,3 +127,9 @@ section3_opt
   :
   | SECTION3_START (?E{t_def.AST.Text}%space)
   ;
+
+%%
+
+def yyerror(loc, msg):
+  print(f'{in_file:s}({loc.first_line:d},{loc.first_column:d}..{loc.last_line:d},{loc.last_column:d}): {msg:s}')
+  sys.exit(1)

diff --git a/skel_lex_yy.py b/skel_lex_yy.py
index 5d45b69..ad1fa13 100644 (file)
--- a/skel_lex_yy.py
+++ b/skel_lex_yy.py
@@ -25,8 +25,26 @@
 # special exception.
 
 import bisect
-#import element
 import sys
+from ndcode.piyacc import element
+
+# this can be redefined in SECTION1
+def YY_AT_BOL():
+  return yy_buffer_stack[-1].at_bol
+
+def yy_set_bol(at_bol):
+  yy_buffer_stack[-1].at_bol = at_bol
+
+def YY_USER_ACTION():
+  pass
+
+def YY_RULE_START():
+  # note that this should also be done after yyless() and REJECT(),
+  # and state should be saved in case they result in a null string,
+  # however, it doesn't seem to be in flex, maintain compatibility:
+  if len(yytext):
+    yy_set_bol(yytext[-1] == '\n')
+  YY_USER_ACTION()
 
 # GENERATE SECTION1
 
@@ -118,11 +136,7 @@ def yy_rule_start():
   yytext = yy_group_text[:yy_group_stack[-1]]
   yytext_len = yy_group_stack[-1]
   del yy_group_stack[-2:]
-  # note that this should also be done after yyless() and REJECT(),
-  # and state should be saved in case they result in a null string,
-  # however, it doesn't seem to be in flex, maintain compatibility:
-  if len(yytext):
-    yy_buffer_stack[-1].at_bol = yytext[-1] == '\n'
+  YY_RULE_START()
   yy_element_stack.append([])
 
 def yy_group_end():
@@ -157,12 +171,6 @@ def yy_pop_state():
   global yystart
   yystart = yystart_stack.pop()
 
-def YY_AT_BOL():
-  return yy_buffer_stack[-1].at_bol
-
-def yy_set_bol(at_bol):
-  yy_buffer_stack[-1].at_bol = at_bol
-
 # GENERATE SECTION2
 
 def yylex(_class = element.Element, *args, **kwargs):

diff --git a/skel_t_def.py b/skel_t_def.py
index fb3b973..009fa52 100644 (file)
--- a/skel_t_def.py
+++ b/skel_t_def.py
@@ -24,8 +24,8 @@
 # files to be licensed under the GNU General Public License without this
 # special exception.
 
-#import element
 import json
+from ndcode.piyacc import element
 
 # GENERATE SECTION1
 

diff --git a/skel_y_tab.py b/skel_y_tab.py
index d78e0e8..e33be3a 100644 (file)
--- a/skel_y_tab.py
+++ b/skel_y_tab.py
@@ -24,24 +24,41 @@
 # License without this special exception.
 
 import bisect
-#import element
-#import lex_yy
 #import xml.etree.ElementTree
+from ndcode.piyacc import element
+from ndcode.piyacc import lex_yy
 
 # this can be redefined in SECTION1
 class YYLTYPE:
   def __init__(
     self,
-    first_line = 0,
-    first_column = 0,
-    last_line = 0,
-    last_column = 0
+    first_line = 1,
+    first_column = 1,
+    last_line = 1,
+    last_column = 1
   ):
     self.first_line = first_line
     self.first_column = first_column
     self.last_line = last_line
     self.last_column = last_column
- 
+  def copy(self):
+    return YYLTYPE(
+       self.first_line,
+       self.first_column,
+       self.last_line,
+       self.last_column
+    )
+
+def YYLLOC_DEFAULT(current, stack, n):
+  if n:
+    current.first_line = stack[-n][2].first_line
+    current.first_column = stack[-n][2].first_column
+    current.last_line = stack[-1][2].last_line
+    current.last_column = stack[-1][2].last_column
+  else:
+    current.first_line = current.last_line = stack[-1][2].last_line
+    current.first_column = current.last_column = stack[-1][2].last_column
+
 # GENERATE SECTION1
 
 # GENERATE TOKENS
@@ -51,7 +68,7 @@ yychar = None
 YYEMPTY = -1
 
 yyval = None
-yyloc = None
+yyloc = YYLTYPE()
 
 yylval = None
 yylloc = YYLTYPE()
@@ -66,18 +83,14 @@ def yyparse(factory, *args, **kwargs):
   # GENERATE INITIALACTION
 
   state = 0
-  yystack = []
-  yylval = None
+  yystack = [(-1, yylval, yylloc.copy())] # kludge for bison compatibility
   yychar = -1
   yy_element_stack = []
   while True:
     #print('state', state, 'yystack', yystack)
-    assert len(yy_element_stack) == len(yystack) * 2
     reduce = yy_lr1dfa_states[state][4]
     if reduce == -1:
       if yychar == -1:
-        yylval = None
-        yylloc = YYLTYPE() # temporary until lex_yy updated, should be None
         yychar = lex_yy.yylex()
         #print('yychar', yychar, 'yylval', yylval, 'yylloc', yylloc, 'lex_yy.yytext', lex_yy.yytext)
         #print('lex_yy.yy_element_space')
@@ -88,9 +101,10 @@ def yyparse(factory, *args, **kwargs):
         bisect.bisect_right(yy_lr1dfa_states[state][0], yychar)
       ]
       if action == -1:
-        raise Exception('syntax error')
+        yyerror(yylloc, 'syntax error')
+        assert False # error recovery is not implemented yet
       if (action & 1) == 0:
-        yystack.append((state, yylval, yylloc))
+        yystack.append((state, yylval, yylloc.copy()))
 
         # push space then AST element contiguously onto yy_element_stack
         # even numbered elements are spaces, odd numbered elements are AST
@@ -105,23 +119,26 @@ def yyparse(factory, *args, **kwargs):
       reduce = action >> 1
     #print('reduce', reduce)
     len_symbols, ref_data = yy_lr1dfa_productions[reduce]
+    # GENERATE YYLLOC_DEFAULT BEGIN
+    YYLLOC_DEFAULT(yyloc, yystack, len_symbols)
+    # GENERATE END
     base = len(yystack) - len_symbols
-    yystack.append((state, None, None))
-    state, yyval, yyloc = yystack[base]
+    yystack.append((state, None, None)) # only has effect if len_symbols == 0
+    state, yyval, _ = yystack[base]
     ref_data()
     del yystack[base:]
     if reduce == 0:
-      assert base == 0
+      assert base == 1
       break
-    yystack.append((state, yyval, yyloc))
+    yystack.append((state, yyval, yyloc.copy()))
 
     # action creates empty space in yy_element_stack[base * 2] if needed
-    assert len(yy_element_stack) > base * 2
+    assert len(yy_element_stack) >= base * 2 - 1
 
-    # concatenate yy_element_stack[base * 2 + 1:] to a single AST element
-    yy_element_stack[base * 2 + 1:] = [
+    # concatenate yy_element_stack[base * 2 - 1:] to a single AST element
+    yy_element_stack[base * 2 - 1:] = [
       element.concatenate(
-        yy_element_stack[base * 2 + 1:],
+        yy_element_stack[base * 2 - 1:],
         element.Element
       )
     ]