Implement element groups with support from piyacc (not via user code)

diff --git a/ast.py b/ast.py
index 71cf50e..e505ff1 100644 (file)
--- a/ast.py
+++ b/ast.py
@@ -371,6 +371,16 @@ class AST(element.Element):
         tag_names
       ):
         return last_action
+      def add_to_groups(
+        self,
+        _ast,
+        production,
+        first_action,
+        add_space,
+        groups,
+        pos
+      ):
+        return first_action, add_space, pos
 
     class Action(Item):
       # GENERATE ELEMENT() BEGIN
@@ -434,7 +444,7 @@ class AST(element.Element):
               # lookaheads (list of initial_set, can_be_empty)
               [([], True)],
               # ref_data
-              last_action
+              ([], None, last_action)
             )
           )
         assert isinstance(self[0], AST.Text) # temporary
@@ -450,6 +460,18 @@ class AST(element.Element):
               i.tag_name = tag_names[i.index - 1]
             i.offset = -len(symbols)
         return self[0]
+      def add_to_groups(
+        self,
+        _ast,
+        production,
+        first_action,
+        add_space,
+        groups,
+        pos
+      ):
+        if first_action: # had first action, treat this as symbol
+          return True, False, pos - 2
+        return True, add_space, pos
 
     class DPrec(Item):
       # GENERATE ELEMENT(int value) BEGIN
@@ -550,6 +572,17 @@ class AST(element.Element):
       ):
         # just skip %empty for now (fix this later)
         return last_action
+      def add_to_groups(
+        self,
+        _ast,
+        production,
+        first_action,
+        add_space,
+        groups,
+        pos
+      ):
+        # just skip %empty for now (fix this later)
+        return first_action, add_space, pos
 
     class Merge(Item):
       # GENERATE ELEMENT() BEGIN
@@ -744,7 +777,7 @@ class AST(element.Element):
               # lookaheads (list of initial_set, can_be_empty)
               [([], True)],
               # ref_data
-              last_action
+              ([], None, last_action)
             )
           )
         symbols.append(
@@ -758,6 +791,16 @@ class AST(element.Element):
           _ast.tags[_ast.symbols[self.symbol]._tag].name
         )
         return None
+      def add_to_groups(
+        self,
+        _ast,
+        production,
+        first_action,
+        add_space,
+        groups,
+        pos
+      ):
+        return True, False, pos - 2
 
     class GroupElement(Item):
       # GENERATE ELEMENT() BEGIN
@@ -828,6 +871,27 @@ class AST(element.Element):
             tag_names
           )
         return last_action
+      def add_to_groups(
+        self,
+        _ast,
+        production,
+        first_action,
+        add_space,
+        groups,
+        pos
+      ):
+        pos0 = pos
+        for i in self[:0:-1]:
+          first_action, add_space, pos0 = i.add_to_groups(
+            _ast,
+            production,
+            add_space,
+            first_action,
+            groups,
+            pos0
+          )
+        groups.append((pos0, pos, self[0]))
+        return first_action, True, pos - 1
  
     # GENERATE ELEMENT(int lhs_nonterminal, int n_symbols, int last_terminal, int precedence_terminal) BEGIN
     def __init__(
@@ -977,6 +1041,22 @@ class AST(element.Element):
               _ast.tags[_ast.symbols[self.lhs_nonterminal]._tag].name
             )
 
+      # go backwards collecting negative indices of element group start/end
+      # here we ignore the first action, rather than holding the most recent
+      first_action = False # have not had first action yet
+      add_space = False # didn't absorb inter-token space
+      groups = []
+      pos = 0
+      for i in self[::-1]:
+        first_action, add_space, pos = i.add_to_groups(
+          _ast,
+          self,
+          first_action,
+          add_space,
+          groups,
+          pos
+        )
+
       _lr1.productions.append(
         (
           # symbols (list of terminal_set, nonterminal_set)
@@ -984,7 +1064,7 @@ class AST(element.Element):
           # lookaheads (list of initial_set, can_be_empty)
           [([], False) for i in range(len(symbols))] + [([], True)],
           # ref_data
-          last_action
+          (groups, pos if add_space else None, last_action)
         )
       )
 
@@ -3680,7 +3760,7 @@ class AST(element.Element):
           # lookaheads (list of initial_set, can_be_empty)
           [([], False), ([], True)],
           # ref_data
-          None # temporary
+          ([], None, None) # temporary
         )
       ],
       # precedences

diff --git a/generate_py.py b/generate_py.py
index 577b0a0..cde45c5 100644 (file)
--- a/generate_py.py
+++ b/generate_py.py
@@ -42,6 +42,7 @@ def generate_py(
   _lr1dfa = _ast.to_lr1().to_lalr1()
   assert _lr1dfa.eof_terminal == 0
   actions = [i for _, i in _lr1dfa.productions]
+  #print('actions', actions)
   _lr1dfa.productions = [
     (_lr1dfa.productions[i][0], 'yy_action{0:d}'.format(i))
     for i in range(len(_lr1dfa.productions))
@@ -108,12 +109,53 @@ YYERROR_VERBOSE = {1:s}
                 [
                   '''def yy_action{0:d}():
   global yyval, yyloc
-{1:s}'''.format(
+{1:s}{2:s}{3:s}'''.format(
                     i,
+                    ''.join(
+                      [
+                        '''  yy_element_stack[{0:s}:{1:s}] = [
+    element.Element(
+      'root',
+      children = [
+        element.concatenate(
+          yy_element_stack[{2:s}:{3:s}],
+{4:s}        )
+      ]
+    )
+  ]
+'''.format(
+                          str(pos0) if pos0 else 'len(yy_element_stack)',
+                          str(pos1) if pos1 else '',
+                          str(pos0) if pos0 else 'len(yy_element_stack)',
+                          str(pos1) if pos1 else '',
+                          ast_text_to_python(factory_text, '          ')
+                        )
+                        for pos0, pos1, factory_text in actions[i][0]
+                      ]
+                    ),
+                    (
+                      ''
+                    if actions[i][1] is None else
+                      '''  yy_element_stack[{0:s}:{1:s}] = [
+    element.Element('root')
+  ]
+'''.format(
+                        (
+                          str(actions[i][1])
+                        if actions[i][1] else
+                          'len(yy_element_stack)'
+                        ),
+                        (
+                          str(actions[i][1])
+                        if actions[i][1] else
+                          ''
+                        )
+                      )
+                    ),
                     (
                       '  pass\n'
-                    if actions[i] is None else
-                      ast_text_to_python(actions[i], '  ')
+                    if actions[i][2] is None else
+                      ast_text_to_python(actions[i][2], '  ')
                     )
                   )
                   for i in range(len(actions))

diff --git a/tests_ast/cal_py.y b/tests_ast/cal_py.y
index 9c29550..0b574a2 100644 (file)
--- a/tests_ast/cal_py.y
+++ b/tests_ast/cal_py.y
@@ -21,62 +21,22 @@ S : S E '\n' {
 }
   ;
 E : (?E{ast.AST.Add}E '+' E) {
-    yy_element_stack[-5:] = [
-      element.Element(
-        'root',
-        children = [
-          element.concatenate(yy_element_stack[-5:], ast.AST.Add)
-        ]
-      )
-    ]
     $$ = $1 + $3
   }
-  | E '-' E {
-    yy_element_stack[-5:] = [
-      element.Element(
-        'root',
-        children = [
-          element.concatenate(yy_element_stack[-5:], ast.AST.Sub)
-        ]
-      )
-    ]
+  | (?E{ast.AST.Sub}E '-' E) {
     $$ = $1 - $3
   }
-  | E '*' E {
-    yy_element_stack[-5:] = [
-      element.Element(
-        'root',
-        children = [
-          element.concatenate(yy_element_stack[-5:], ast.AST.Mul)
-        ]
-      )
-    ]
+  | (?E{ast.AST.Mul}E '*' E) {
     $$ = $1 * $3
   }
-  | E '/' E {
-    yy_element_stack[-5:] = [
-      element.Element(
-        'root',
-        children = [
-          element.concatenate(yy_element_stack[-5:], ast.AST.Div)
-        ]
-      )
-    ]
+  | (?E{ast.AST.Div}E '/' E) {
     $$ = $1 / $3
   }
   | '(' E ')' {
     $$ = $2
   }
-  | '-' E %prec UMINUS {
+  | (?E{ast.AST.Neg}'-' E) %prec UMINUS {
     $$ = -$2
-    yy_element_stack[-3:] = [
-      element.Element(
-        'root',
-        children = [
-          element.concatenate(yy_element_stack[-3:], ast.AST.Neg)
-        ]
-      )
-    ]
   }
   | NUM
   ;