Change bracketing of action text so that delimiters are captured within markup

diff --git a/.gitignore b/.gitignore
index cf141e2..c841339 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,7 @@
 __pycache__
-lex.yy.c
 tests/cal
 tests/cal.l.xml
+tests/cal.y.xml
 tests/flex0
 tests/flex0.c
 tests/flex0.l.xml
@@ -11,4 +11,5 @@ tests/flex1.l.xml
 tests/lex.yy.c.orig
 tests/lex.yy.c
 tests/y.tab.c
+skel/lex.yy.c
 skel/lex.yy.c.orig

diff --git a/ast.py b/ast.py
index f8417ac..718c80b 100644 (file)
--- a/ast.py
+++ b/ast.py
@@ -108,6 +108,34 @@ class PLex(element.Element):
       return 'ast.PLex.StartCondition({0:s})'.format(', '.join(params))
     # GENERATE END
 
+  class Name(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'PLex_Name',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        Name if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.PLex.Name({0:s})'.format(', '.join(params))
+    # GENERATE END
+
   class Section(element.Element):
     # GENERATE ELEMENT() BEGIN
     def __init__(
@@ -151,11 +179,11 @@ class PLex(element.Element):
           inclusive_start_conditions
         )
 
-  class Name(element.Element):
+  class Text(element.Element):
     # GENERATE ELEMENT() BEGIN
     def __init__(
       self,
-      tag = 'PLex_Name',
+      tag = 'PLex_Text',
       attrib = {},
       text = '',
       children = []
@@ -170,13 +198,13 @@ class PLex(element.Element):
     def copy(self, factory = None):
       result = element.Element.copy(
         self,
-        Name if factory is None else factory
+        Text if factory is None else factory
       )
       return result
     def __repr__(self):
       params = []
       self.repr_serialize(params)
-      return 'ast.PLex.Name({0:s})'.format(', '.join(params))
+      return 'ast.PLex.Text({0:s})'.format(', '.join(params))
     # GENERATE END
 
   class Section1Or2(Section):
@@ -215,16 +243,16 @@ class PLex(element.Element):
         all_start_conditions,
         inclusive_start_conditions
       ):
-        section.code_blocks.append(self)
+        section.code_blocks_text.append(self[0])
 
-    # GENERATE ELEMENT(list(ref) code_blocks) BEGIN
+    # GENERATE ELEMENT(list(ref) code_blocks_text) BEGIN
     def __init__(
       self,
       tag = 'PLex_Section1Or2',
       attrib = {},
       text = '',
       children = [],
-      code_blocks = []
+      code_blocks_text = []
     ):
       PLex.Section.__init__(
         self,
@@ -233,32 +261,32 @@ class PLex(element.Element):
         text,
         children
       )
-      self.code_blocks = code_blocks
+      self.code_blocks_text = code_blocks_text
     def serialize(self, ref_list, indent = 0):
       PLex.Section.serialize(self, ref_list, indent)
       self.set(
-        'code_blocks',
-        ' '.join([element.serialize_ref(i, ref_list) for i in self.code_blocks])
+        'code_blocks_text',
+        ' '.join([element.serialize_ref(i, ref_list) for i in self.code_blocks_text])
       )
     def deserialize(self, ref_list):
       PLex.Section.deserialize(self, ref_list)
-      self.code_blocks = [
+      self.code_blocks_text = [
         element.deserialize_ref(i, ref_list)
-        for i in self.get('code_blocks', '').split()
+        for i in self.get('code_blocks_text', '').split()
       ]
     def copy(self, factory = None):
       result = PLex.Section.copy(
         self,
         Section1Or2 if factory is None else factory
       )
-      result.code_blocks = self.code_blocks
+      result.code_blocks_text = self.code_blocks_text
       return result
     def repr_serialize(self, params):
       PLex.Section.repr_serialize(self, params)
-      if len(self.code_blocks):
+      if len(self.code_blocks_text):
         params.append(
-          'code_blocks = [{0:s}]'.format(
-            ', '.join([repr(i) for i in self.code_blocks])
+          'code_blocks_text = [{0:s}]'.format(
+            ', '.join([repr(i) for i in self.code_blocks_text])
           )
         )
     def __repr__(self):
@@ -621,7 +649,7 @@ class PLex(element.Element):
       attrib = {},
       text = '',
       children = [],
-      code_blocks = [],
+      code_blocks_text = [],
       ecs = False,
       meta_ecs = False,
       reject = False,
@@ -634,7 +662,7 @@ class PLex(element.Element):
         attrib,
         text,
         children,
-        code_blocks
+        code_blocks_text
       )
       self.ecs = (
         element.deserialize_bool(ecs)
@@ -926,8 +954,8 @@ class PLex(element.Element):
             if default and plex.start_conditions[i].eof_action != 0:
               continue # rule applies to start conditions with no EOF rule yet
             assert plex.start_conditions[i].eof_action == 0
-            plex.start_conditions[i].eof_action = len(plex.eof_actions)
-          plex.eof_actions.append(action)
+            plex.start_conditions[i].eof_action = len(plex.eof_actions_text)
+          plex.eof_actions_text.append(action[0])
         else:
           if isinstance(expr, PLex.Section2.Rule.BOLRule):
             bol_rule = True
@@ -945,7 +973,7 @@ class PLex(element.Element):
               )
             ]
           )
-          expr.post_process(len(plex.actions))
+          expr.post_process(len(plex.actions_text))
           for j in start_condition_set:
             for k in range(int(bol_rule), 2):
               plex.start_conditions[j][k] = regex.RegexOr(
@@ -954,7 +982,7 @@ class PLex(element.Element):
                   expr
                 ]
               )
-          plex.actions.append(action)
+          plex.actions_text.append(action[0])
 
     # GENERATE ELEMENT() BEGIN
     def __init__(
@@ -963,7 +991,7 @@ class PLex(element.Element):
       attrib = {},
       text = '',
       children = [],
-      code_blocks = []
+      code_blocks_text = []
     ):
       PLex.Section1Or2.__init__(
         self,
@@ -971,7 +999,7 @@ class PLex(element.Element):
         attrib,
         text,
         children,
-        code_blocks
+        code_blocks_text
       )
     def copy(self, factory = None):
       result = PLex.Section1Or2.copy(
@@ -1013,7 +1041,7 @@ class PLex(element.Element):
       return 'ast.PLex.Section3({0:s})'.format(', '.join(params))
     # GENERATE END
 
-  # GENERATE ELEMENT(list(ref) start_conditions, list(ref) actions, list(ref) eof_actions) BEGIN
+  # GENERATE ELEMENT(list(ref) start_conditions, list(ref) actions_text, list(ref) eof_actions_text) BEGIN
   def __init__(
     self,
     tag = 'PLex',
@@ -1021,8 +1049,8 @@ class PLex(element.Element):
     text = '',
     children = [],
     start_conditions = [],
-    actions = [],
-    eof_actions = []
+    actions_text = [],
+    eof_actions_text = []
   ):
     element.Element.__init__(
       self,
@@ -1032,8 +1060,8 @@ class PLex(element.Element):
       children
     )
     self.start_conditions = start_conditions
-    self.actions = actions
-    self.eof_actions = eof_actions
+    self.actions_text = actions_text
+    self.eof_actions_text = eof_actions_text
   def serialize(self, ref_list, indent = 0):
     element.Element.serialize(self, ref_list, indent)
     self.set(
@@ -1041,12 +1069,12 @@ class PLex(element.Element):
       ' '.join([element.serialize_ref(i, ref_list) for i in self.start_conditions])
     )
     self.set(
-      'actions',
-      ' '.join([element.serialize_ref(i, ref_list) for i in self.actions])
+      'actions_text',
+      ' '.join([element.serialize_ref(i, ref_list) for i in self.actions_text])
     )
     self.set(
-      'eof_actions',
-      ' '.join([element.serialize_ref(i, ref_list) for i in self.eof_actions])
+      'eof_actions_text',
+      ' '.join([element.serialize_ref(i, ref_list) for i in self.eof_actions_text])
     )
   def deserialize(self, ref_list):
     element.Element.deserialize(self, ref_list)
@@ -1054,13 +1082,13 @@ class PLex(element.Element):
       element.deserialize_ref(i, ref_list)
       for i in self.get('start_conditions', '').split()
     ]
-    self.actions = [
+    self.actions_text = [
       element.deserialize_ref(i, ref_list)
-      for i in self.get('actions', '').split()
+      for i in self.get('actions_text', '').split()
     ]
-    self.eof_actions = [
+    self.eof_actions_text = [
       element.deserialize_ref(i, ref_list)
-      for i in self.get('eof_actions', '').split()
+      for i in self.get('eof_actions_text', '').split()
     ]
   def copy(self, factory = None):
     result = element.Element.copy(
@@ -1068,8 +1096,8 @@ class PLex(element.Element):
       PLex if factory is None else factory
     )
     result.start_conditions = self.start_conditions
-    result.actions = self.actions
-    result.eof_actions = self.eof_actions
+    result.actions_text = self.actions_text
+    result.eof_actions_text = self.eof_actions_text
     return result
   def repr_serialize(self, params):
     element.Element.repr_serialize(self, params)
@@ -1079,16 +1107,16 @@ class PLex(element.Element):
           ', '.join([repr(i) for i in self.start_conditions])
         )
       )
-    if len(self.actions):
+    if len(self.actions_text):
       params.append(
-        'actions = [{0:s}]'.format(
-          ', '.join([repr(i) for i in self.actions])
+        'actions_text = [{0:s}]'.format(
+          ', '.join([repr(i) for i in self.actions_text])
         )
       )
-    if len(self.eof_actions):
+    if len(self.eof_actions_text):
       params.append(
-        'eof_actions = [{0:s}]'.format(
-          ', '.join([repr(i) for i in self.eof_actions])
+        'eof_actions_text = [{0:s}]'.format(
+          ', '.join([repr(i) for i in self.eof_actions_text])
         )
       )
   def __repr__(self):
@@ -1106,8 +1134,8 @@ class PLex(element.Element):
         eof_action = 0
       )
     ]
-    self.actions = []
-    self.eof_actions = [
+    self.actions_text = []
+    self.eof_actions_text = [
       PLex.Section2.Rule.Action(text = '\t\t\t\tyyterminate();\n')
     ]
 
@@ -1146,7 +1174,7 @@ class PLex(element.Element):
                       char_set = [0, 0x100]
                     ),
                     regex.RegexGroup(
-                      group_index = len(self.actions),
+                      group_index = len(self.actions_text),
                       children = [
                         regex.RegexEmpty()
                       ]
@@ -1157,7 +1185,7 @@ class PLex(element.Element):
             )
           ]
         )
-    self.actions.append(PLex.Section2.Rule.Action(text = 'ECHO;\n'))
+    self.actions_text.append(PLex.Text(text = 'ECHO;\n'))
   def to_nfa(self):
     nfa = regex.NFA()
     for i in self.start_conditions:
@@ -1170,8 +1198,9 @@ tag_to_class = {
   'Item': Item,
   'PLex': PLex,
   'PLex_StartCondition': PLex.StartCondition,
-  'PLex_Section': PLex.Section,
   'PLex_Name': PLex.Name,
+  'PLex_Section': PLex.Section,
+  'PLex_Text': PLex.Text,
   'PLex_Section1Or2': PLex.Section1Or2,
   'PLex_Section1Or2_CodeBlock': PLex.Section1Or2.CodeBlock,
   'PLex_Section1': PLex.Section1,

diff --git a/flex_dfa.py b/flex_dfa.py
index 04bbe3e..51ae564 100644 (file)
--- a/flex_dfa.py
+++ b/flex_dfa.py
@@ -197,7 +197,7 @@ class FlexDFA:
 def generate(plex, skel_file, out_file):
   nfa = plex.to_nfa()
   eob_expr = regex.RegexGroup(children = [regex.RegexEmpty()])
-  eob_expr.post_process(len(plex.actions))
+  eob_expr.post_process(len(plex.actions_text))
   eob_expr.add_to_nfa(nfa)
   flex_dfa = FlexDFA(nfa.to_dfa())
 
@@ -240,7 +240,7 @@ static const flex_int16_t yy_chk[] = {{{6:s}
 }};
 /* GENERATE TABLES END */
 '''.format(
-              len(plex.actions),
+              len(plex.actions_text),
               ','.join(
                 [
                   '\n\t{0:s}'.format(
@@ -326,7 +326,7 @@ static const flex_int16_t yy_chk[] = {{{6:s}
             '''/* GENERATE SECTION1 BEGIN */
 {0:s}/* GENERATE SECTION1 END */
 '''.format(
-              ''.join([element.get_text(i, 0) for i in plex[0].code_blocks])
+              ''.join([element.get_text(i, 0) for i in plex[0].code_blocks_text])
             )
           )
         elif line == '/* GENERATE STARTCONDDECL */\n':
@@ -350,7 +350,7 @@ static const flex_int16_t yy_chk[] = {{{6:s}
             '''/* GENERATE SECTION2INITIAL BEGIN */
 {0:s}/* GENERATE SECTION2INITIAL END */
 '''.format(
-              ''.join([element.get_text(i, 0) for i in plex[1].code_blocks])
+              ''.join([element.get_text(i, 0) for i in plex[1].code_blocks_text])
             )
           )
         elif line == '/* GENERATE SECTION2 */\n':
@@ -360,7 +360,7 @@ static const flex_int16_t yy_chk[] = {{{6:s}
               for j in range(len(plex.start_conditions))
               if plex.start_conditions[i].eof_action == j
             ]
-            for i in range(len(plex.eof_actions))
+            for i in range(len(plex.eof_actions_text))
           ]
           #print('eof_action_to_start_conditions', eof_action_to_start_conditions)
           fout.write(
@@ -374,9 +374,9 @@ YY_RULE_SETUP
 {1:s}  YY_BREAK
 '''.format(
                     i,
-                    element.get_text(plex.actions[i], 0)
+                    element.get_text(plex.actions_text[i], 0)
                   )
-                  for i in range(len(plex.actions))
+                  for i in range(len(plex.actions_text))
                 ]
               ),
               ''.join(
@@ -390,9 +390,9 @@ YY_RULE_SETUP
                         for j in eof_action_to_start_conditions[i]
                       ]
                     ),
-                    element.get_text(plex.eof_actions[i], 0)
+                    element.get_text(plex.eof_actions_text[i], 0)
                   )
-                  for i in range(len(plex.eof_actions))
+                  for i in range(len(plex.eof_actions_text))
                   if len(eof_action_to_start_conditions[i]) > 0
                 ]
               )