Further rearrangement to put everything inside the PLex class, and move the entire...

diff --git a/ast.py b/ast.py
index 255c966..af8f59d 100644 (file)
--- a/ast.py
+++ b/ast.py
@@ -1,69 +1,6 @@
 import element
 import regex
 
-class PLexSpecification(element.Element):
-  # GENERATE ELEMENT() BEGIN
-  def __init__(
-    self,
-    tag = 'PLexSpecification',
-    attrib = {},
-    text = '',
-    children = []
-  ):
-    element.Element.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
-    )
-  def copy(self, factory = None):
-    result = element.Element.copy(
-      self,
-      PLexSpecification if factory is None else factory
-    )
-    return result
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.PLexSpecification({0:s})'.format(', '.join(params))
-  # GENERATE END
-  def post_process(self):
-    for i in self:
-      i.post_process(self)
-
-class Section(element.Element):
-  # GENERATE ELEMENT() BEGIN
-  def __init__(
-    self,
-    tag = 'Section',
-    attrib = {},
-    text = '',
-    children = []
-  ):
-    element.Element.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
-    )
-  def copy(self, factory = None):
-    result = element.Element.copy(
-      self,
-      Section if factory is None else factory
-    )
-    return result
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.Section({0:s})'.format(', '.join(params))
-  # GENERATE END
-  def post_process(self, plex_specification):
-    for i in self:
-      #print('item', i.tag)
-      i.post_process(plex_specification, self)
-
 class Item(element.Element):
   # GENERATE ELEMENT() BEGIN
   def __init__(
@@ -91,366 +28,139 @@ class Item(element.Element):
     self.repr_serialize(params)
     return 'ast.Item({0:s})'.format(', '.join(params))
   # GENERATE END
-  def post_process(self, plex_specification, section):
-    raise NotImplementedException
-
-class Name(element.Element):
-  # GENERATE ELEMENT() BEGIN
-  def __init__(
-    self,
-    tag = 'Name',
-    attrib = {},
-    text = '',
-    children = []
-  ):
-    element.Element.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
-    )
-  def copy(self, factory = None):
-    result = element.Element.copy(
-      self,
-      Name if factory is None else factory
-    )
-    return result
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.Name({0:s})'.format(', '.join(params))
-  # GENERATE END
-
-class SectionCodeBlocks(Section):
-  # GENERATE ELEMENT(list(ref) code_blocks) BEGIN
-  def __init__(
+  def post_process(
     self,
-    tag = 'SectionCodeBlocks',
-    attrib = {},
-    text = '',
-    children = [],
-    code_blocks = []
+    plex,
+    section,
+    name_to_start_condition,
+    all_start_conditions,
+    inclusive_start_conditions
   ):
-    Section.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
-    )
-    self.code_blocks = code_blocks
-  def serialize(self, ref_list, indent = 0):
-    Section.serialize(self, ref_list, indent)
-    self.set(
-      'code_blocks',
-      ' '.join([element.serialize_ref(i, ref_list) for i in self.code_blocks])
-    )
-  def deserialize(self, ref_list):
-    Section.deserialize(self, ref_list)
-    self.code_blocks = [
-      element.deserialize_ref(i, ref_list)
-      for i in self.get('code_blocks', '').split()
-    ]
-  def copy(self, factory = None):
-    result = Section.copy(
-      self,
-      SectionCodeBlocks if factory is None else factory
-    )
-    result.code_blocks = self.code_blocks
-    return result
-  def repr_serialize(self, params):
-    Section.repr_serialize(self, params)
-    if len(self.code_blocks):
-      params.append(
-        'code_blocks = [{0:s}]'.format(
-          ', '.join([repr(i) for i in self.code_blocks])
-        )
-      )
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.SectionCodeBlocks({0:s})'.format(', '.join(params))
-  # GENERATE END
-
-class CodeBlock(Item):
-  # GENERATE ELEMENT() BEGIN
-  def __init__(
-    self,
-    tag = 'CodeBlock',
-    attrib = {},
-    text = '',
-    children = []
-  ):
-    Item.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
-    )
-  def copy(self, factory = None):
-    result = Item.copy(
-      self,
-      CodeBlock if factory is None else factory
-    )
-    return result
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.CodeBlock({0:s})'.format(', '.join(params))
-  # GENERATE END
-  def post_process(self, plex_specification, section):
-    section.code_blocks.append(self)
-
-class Option(element.Element):
-  # GENERATE ELEMENT() BEGIN
-  def __init__(
-    self,
-    tag = 'Option',
-    attrib = {},
-    text = '',
-    children = []
-  ):
-    element.Element.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
-    )
-  def copy(self, factory = None):
-    result = element.Element.copy(
-      self,
-      Option if factory is None else factory
-    )
-    return result
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.Option({0:s})'.format(', '.join(params))
-  # GENERATE END
-  def post_process(self, section):
     raise NotImplementedException
 
-class BoolOption(Option):
-  # GENERATE ELEMENT(bool value) BEGIN
-  def __init__(
-    self,
-    tag = 'BoolOption',
-    attrib = {},
-    text = '',
-    children = [],
-    value = False
-  ):
-    Option.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
-    )
-    self.value = (
-      element.deserialize_bool(value)
-    if isinstance(value, str) else
-      value
-    )
-  def serialize(self, ref_list, indent = 0):
-    Option.serialize(self, ref_list, indent)
-    self.set('value', element.serialize_bool(self.value))
-  def deserialize(self, ref_list):
-    Option.deserialize(self, ref_list)
-    self.value = element.deserialize_bool(self.get('value', 'false'))
-  def copy(self, factory = None):
-    result = Option.copy(
+class PLex(element.Element):
+  class StartCondition(element.Element):
+    # GENERATE ELEMENT(str name, bool exclusive, int eof_action) BEGIN
+    def __init__(
       self,
-      BoolOption if factory is None else factory
-    )
-    result.value = self.value
-    return result
-  def repr_serialize(self, params):
-    Option.repr_serialize(self, params)
-    if self.value != False:
-      params.append(
-        'value = {0:s}'.format(repr(self.value))
-      )
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.BoolOption({0:s})'.format(', '.join(params))
-  # GENERATE END
-
-class Section1(SectionCodeBlocks):
-  class Options(Item):
-    class ECS(BoolOption):
-      # GENERATE ELEMENT() BEGIN
-      def __init__(
+      tag = 'PLex_StartCondition',
+      attrib = {},
+      text = '',
+      children = [],
+      name = '',
+      exclusive = False,
+      eof_action = -1
+    ):
+      element.Element.__init__(
         self,
-        tag = 'Section1_Options_ECS',
-        attrib = {},
-        text = '',
-        children = [],
-        value = False
-      ):
-        BoolOption.__init__(
-          self,
-          tag,
-          attrib,
-          text,
-          children,
-          value
-        )
-      def copy(self, factory = None):
-        result = BoolOption.copy(
-          self,
-          ECS if factory is None else factory
-        )
-        return result
-      def __repr__(self):
-        params = []
-        self.repr_serialize(params)
-        return 'ast.Section1.Options.ECS({0:s})'.format(', '.join(params))
-      # GENERATE END
-      def post_process(self, section):
-        section.ecs = self.value
-
-    class MetaECS(BoolOption):
-      # GENERATE ELEMENT() BEGIN
-      def __init__(
+        tag,
+        attrib,
+        text,
+        children
+      )
+      self.name = name
+      self.exclusive = (
+        element.deserialize_bool(exclusive)
+      if isinstance(exclusive, str) else
+        exclusive
+      )
+      self.eof_action = (
+        element.deserialize_int(eof_action)
+      if isinstance(eof_action, str) else
+        eof_action
+      )
+    def serialize(self, ref_list, indent = 0):
+      element.Element.serialize(self, ref_list, indent)
+      self.set('name', element.serialize_str(self.name))
+      self.set('exclusive', element.serialize_bool(self.exclusive))
+      self.set('eof_action', element.serialize_int(self.eof_action))
+    def deserialize(self, ref_list):
+      element.Element.deserialize(self, ref_list)
+      self.name = element.deserialize_str(self.get('name', ''))
+      self.exclusive = element.deserialize_bool(self.get('exclusive', 'false'))
+      self.eof_action = element.deserialize_int(self.get('eof_action', '-1'))
+    def copy(self, factory = None):
+      result = element.Element.copy(
         self,
-        tag = 'Section1_Options_MetaECS',
-        attrib = {},
-        text = '',
-        children = [],
-        value = False
-      ):
-        BoolOption.__init__(
-          self,
-          tag,
-          attrib,
-          text,
-          children,
-          value
-        )
-      def copy(self, factory = None):
-        result = BoolOption.copy(
-          self,
-          MetaECS if factory is None else factory
+        StartCondition if factory is None else factory
+      )
+      result.name = self.name
+      result.exclusive = self.exclusive
+      result.eof_action = self.eof_action
+      return result
+    def repr_serialize(self, params):
+      element.Element.repr_serialize(self, params)
+      if self.name != '':
+        params.append(
+          'name = {0:s}'.format(repr(self.name))
         )
-        return result
-      def __repr__(self):
-        params = []
-        self.repr_serialize(params)
-        return 'ast.Section1.Options.MetaECS({0:s})'.format(', '.join(params))
-      # GENERATE END
-      def post_process(self, section):
-        section.meta_ecs = self.value
-
-    class Reject(BoolOption):
-      # GENERATE ELEMENT() BEGIN
-      def __init__(
-        self,
-        tag = 'Section1_Options_Reject',
-        attrib = {},
-        text = '',
-        children = [],
-        value = False
-      ):
-        BoolOption.__init__(
-          self,
-          tag,
-          attrib,
-          text,
-          children,
-          value
+      if self.exclusive != False:
+        params.append(
+          'exclusive = {0:s}'.format(repr(self.exclusive))
         )
-      def copy(self, factory = None):
-        result = BoolOption.copy(
-          self,
-          Reject if factory is None else factory
+      if self.eof_action != -1:
+        params.append(
+          'eof_action = {0:s}'.format(repr(self.eof_action))
         )
-        return result
-      def __repr__(self):
-        params = []
-        self.repr_serialize(params)
-        return 'ast.Section1.Options.Reject({0:s})'.format(', '.join(params))
-      # GENERATE END
-      def post_process(self, section):
-        section.reject = self.value
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.PLex.StartCondition({0:s})'.format(', '.join(params))
+    # GENERATE END
 
-    class YYMore(BoolOption):
-      # GENERATE ELEMENT() BEGIN
-      def __init__(
+  class Section(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'PLex_Section',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
         self,
-        tag = 'Section1_Options_YYMore',
-        attrib = {},
-        text = '',
-        children = [],
-        value = False
-      ):
-        BoolOption.__init__(
-          self,
-          tag,
-          attrib,
-          text,
-          children,
-          value
-        )
-      def copy(self, factory = None):
-        result = BoolOption.copy(
-          self,
-          YYMore if factory is None else factory
-        )
-        return result
-      def __repr__(self):
-        params = []
-        self.repr_serialize(params)
-        return 'ast.Section1.Options.YYMore({0:s})'.format(', '.join(params))
-      # GENERATE END
-      def post_process(self, section):
-        section.yymore = self.value
-
-    class YYWrap(BoolOption):
-      # GENERATE ELEMENT() BEGIN
-      def __init__(
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
         self,
-        tag = 'Section1_Options_YYWrap',
-        attrib = {},
-        text = '',
-        children = [],
-        value = False
-      ):
-        BoolOption.__init__(
-          self,
-          tag,
-          attrib,
-          text,
-          children,
-          value
-        )
-      def copy(self, factory = None):
-        result = BoolOption.copy(
-          self,
-          YYWrap if factory is None else factory
-        )
-        return result
-      def __repr__(self):
-        params = []
-        self.repr_serialize(params)
-        return 'ast.Section1.Options.YYWrap({0:s})'.format(', '.join(params))
-      # GENERATE END
-      def post_process(self, section):
-        section.yywrap = self.value
+        Section if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.PLex.Section({0:s})'.format(', '.join(params))
+    # GENERATE END
+    def post_process(
+      self,
+      plex,
+      name_to_start_condition,
+      all_start_conditions,
+      inclusive_start_conditions
+    ):
+      for i in self:
+        i.post_process(
+          plex,
+          self,
+          name_to_start_condition,
+          all_start_conditions,
+          inclusive_start_conditions
+        )
 
+  class Name(element.Element):
     # GENERATE ELEMENT() BEGIN
     def __init__(
       self,
-      tag = 'Section1_Options',
+      tag = 'PLex_Name',
       attrib = {},
       text = '',
       children = []
     ):
-      Item.__init__(
+      element.Element.__init__(
         self,
         tag,
         attrib,
@@ -458,236 +168,353 @@ class Section1(SectionCodeBlocks):
         children
       )
     def copy(self, factory = None):
-      result = Item.copy(
+      result = element.Element.copy(
         self,
-        Options if factory is None else factory
+        Name if factory is None else factory
       )
       return result
     def __repr__(self):
       params = []
       self.repr_serialize(params)
-      return 'ast.Section1.Options({0:s})'.format(', '.join(params))
+      return 'ast.PLex.Name({0:s})'.format(', '.join(params))
     # GENERATE END
-    def post_process(self, plex_specification, section):
-      for i in self:
-        #print('option', i.tag)
-        i.post_process(section)
 
-  class StartConditions(Item):
-    # GENERATE ELEMENT(bool exclusive) BEGIN
+  class SectionCodeBlocks(Section):
+    # GENERATE ELEMENT(list(ref) code_blocks) BEGIN
     def __init__(
       self,
-      tag = 'Section1_StartConditions',
+      tag = 'PLex_SectionCodeBlocks',
       attrib = {},
       text = '',
       children = [],
-      exclusive = False
+      code_blocks = []
     ):
-      Item.__init__(
+      PLex.Section.__init__(
         self,
         tag,
         attrib,
         text,
         children
       )
-      self.exclusive = (
-        element.deserialize_bool(exclusive)
-      if isinstance(exclusive, str) else
-        exclusive
-      )
+      self.code_blocks = code_blocks
     def serialize(self, ref_list, indent = 0):
-      Item.serialize(self, ref_list, indent)
-      self.set('exclusive', element.serialize_bool(self.exclusive))
+      PLex.Section.serialize(self, ref_list, indent)
+      self.set(
+        'code_blocks',
+        ' '.join([element.serialize_ref(i, ref_list) for i in self.code_blocks])
+      )
     def deserialize(self, ref_list):
-      Item.deserialize(self, ref_list)
-      self.exclusive = element.deserialize_bool(self.get('exclusive', 'false'))
+      PLex.Section.deserialize(self, ref_list)
+      self.code_blocks = [
+        element.deserialize_ref(i, ref_list)
+        for i in self.get('code_blocks', '').split()
+      ]
     def copy(self, factory = None):
-      result = Item.copy(
+      result = PLex.Section.copy(
         self,
-        StartConditions if factory is None else factory
+        SectionCodeBlocks if factory is None else factory
       )
-      result.exclusive = self.exclusive
+      result.code_blocks = self.code_blocks
       return result
     def repr_serialize(self, params):
-      Item.repr_serialize(self, params)
-      if self.exclusive != False:
+      PLex.Section.repr_serialize(self, params)
+      if len(self.code_blocks):
         params.append(
-          'exclusive = {0:s}'.format(repr(self.exclusive))
+          'code_blocks = [{0:s}]'.format(
+            ', '.join([repr(i) for i in self.code_blocks])
+          )
         )
     def __repr__(self):
       params = []
       self.repr_serialize(params)
-      return 'ast.Section1.StartConditions({0:s})'.format(', '.join(params))
+      return 'ast.PLex.SectionCodeBlocks({0:s})'.format(', '.join(params))
     # GENERATE END
-    def post_process(self, plex_specification, section):
-      pass # do something here eventually
-
-  # GENERATE ELEMENT(bool ecs, bool meta_ecs, bool reject, bool yymore, bool yywrap) BEGIN
-  def __init__(
-    self,
-    tag = 'Section1',
-    attrib = {},
-    text = '',
-    children = [],
-    code_blocks = [],
-    ecs = False,
-    meta_ecs = False,
-    reject = False,
-    yymore = False,
-    yywrap = False
-  ):
-    SectionCodeBlocks.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children,
-      code_blocks
-    )
-    self.ecs = (
-      element.deserialize_bool(ecs)
-    if isinstance(ecs, str) else
-      ecs
-    )
-    self.meta_ecs = (
-      element.deserialize_bool(meta_ecs)
-    if isinstance(meta_ecs, str) else
-      meta_ecs
-    )
-    self.reject = (
-      element.deserialize_bool(reject)
-    if isinstance(reject, str) else
-      reject
-    )
-    self.yymore = (
-      element.deserialize_bool(yymore)
-    if isinstance(yymore, str) else
-      yymore
-    )
-    self.yywrap = (
-      element.deserialize_bool(yywrap)
-    if isinstance(yywrap, str) else
-      yywrap
-    )
-  def serialize(self, ref_list, indent = 0):
-    SectionCodeBlocks.serialize(self, ref_list, indent)
-    self.set('ecs', element.serialize_bool(self.ecs))
-    self.set('meta_ecs', element.serialize_bool(self.meta_ecs))
-    self.set('reject', element.serialize_bool(self.reject))
-    self.set('yymore', element.serialize_bool(self.yymore))
-    self.set('yywrap', element.serialize_bool(self.yywrap))
-  def deserialize(self, ref_list):
-    SectionCodeBlocks.deserialize(self, ref_list)
-    self.ecs = element.deserialize_bool(self.get('ecs', 'false'))
-    self.meta_ecs = element.deserialize_bool(self.get('meta_ecs', 'false'))
-    self.reject = element.deserialize_bool(self.get('reject', 'false'))
-    self.yymore = element.deserialize_bool(self.get('yymore', 'false'))
-    self.yywrap = element.deserialize_bool(self.get('yywrap', 'false'))
-  def copy(self, factory = None):
-    result = SectionCodeBlocks.copy(
-      self,
-      Section1 if factory is None else factory
-    )
-    result.ecs = self.ecs
-    result.meta_ecs = self.meta_ecs
-    result.reject = self.reject
-    result.yymore = self.yymore
-    result.yywrap = self.yywrap
-    return result
-  def repr_serialize(self, params):
-    SectionCodeBlocks.repr_serialize(self, params)
-    if self.ecs != False:
-      params.append(
-        'ecs = {0:s}'.format(repr(self.ecs))
-      )
-    if self.meta_ecs != False:
-      params.append(
-        'meta_ecs = {0:s}'.format(repr(self.meta_ecs))
-      )
-    if self.reject != False:
-      params.append(
-        'reject = {0:s}'.format(repr(self.reject))
-      )
-    if self.yymore != False:
-      params.append(
-        'yymore = {0:s}'.format(repr(self.yymore))
-      )
-    if self.yywrap != False:
-      params.append(
-        'yywrap = {0:s}'.format(repr(self.yywrap))
-      )
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.Section1({0:s})'.format(', '.join(params))
-  # GENERATE END
-  def post_process(self, plex_specification):
-    self.ecs = False
-    self.meta_ecs = False
-    self.reject = True
-    self.yymore = True
-    self.yywrap = True
-    Section.post_process(self, plex_specification)
 
-class Section2(SectionCodeBlocks):
-  class StartConditions(element.Element):
-    # GENERATE ELEMENT(bool wildcard) BEGIN
+  class CodeBlock(Item):
+    # GENERATE ELEMENT() BEGIN
     def __init__(
       self,
-      tag = 'Section2_StartConditions',
+      tag = 'PLex_CodeBlock',
       attrib = {},
       text = '',
-      children = [],
-      wildcard = False
+      children = []
     ):
-      element.Element.__init__(
+      Item.__init__(
         self,
         tag,
         attrib,
         text,
         children
       )
-      self.wildcard = (
-        element.deserialize_bool(wildcard)
-      if isinstance(wildcard, str) else
-        wildcard
-      )
-    def serialize(self, ref_list, indent = 0):
-      element.Element.serialize(self, ref_list, indent)
-      self.set('wildcard', element.serialize_bool(self.wildcard))
-    def deserialize(self, ref_list):
-      element.Element.deserialize(self, ref_list)
-      self.wildcard = element.deserialize_bool(self.get('wildcard', 'false'))
     def copy(self, factory = None):
-      result = element.Element.copy(
+      result = Item.copy(
         self,
-        StartConditions if factory is None else factory
+        CodeBlock if factory is None else factory
       )
-      result.wildcard = self.wildcard
       return result
-    def repr_serialize(self, params):
-      element.Element.repr_serialize(self, params)
-      if self.wildcard != False:
-        params.append(
-          'wildcard = {0:s}'.format(repr(self.wildcard))
-        )
     def __repr__(self):
       params = []
       self.repr_serialize(params)
-      return 'ast.Section2.StartConditions({0:s})'.format(', '.join(params))
+      return 'ast.PLex.CodeBlock({0:s})'.format(', '.join(params))
     # GENERATE END
+    def post_process(
+      self,
+      plex,
+      section,
+      name_to_start_condition,
+      all_start_conditions,
+      inclusive_start_conditions
+    ):
+      section.code_blocks.append(self)
+
+  class Section1(SectionCodeBlocks):
+    class Options(Item):
+      class Option(element.Element):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section1_Options_Option',
+          attrib = {},
+          text = '',
+          children = []
+        ):
+          element.Element.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children
+          )
+        def copy(self, factory = None):
+          result = element.Element.copy(
+            self,
+            Option if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section1.Options.Option({0:s})'.format(', '.join(params))
+        # GENERATE END
+        def post_process(self, section):
+          raise NotImplementedException
+
+      class BoolOption(Option):
+        # GENERATE ELEMENT(bool value) BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section1_Options_BoolOption',
+          attrib = {},
+          text = '',
+          children = [],
+          value = False
+        ):
+          PLex.Section1.Options.Option.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children
+          )
+          self.value = (
+            element.deserialize_bool(value)
+          if isinstance(value, str) else
+            value
+          )
+        def serialize(self, ref_list, indent = 0):
+          PLex.Section1.Options.Option.serialize(self, ref_list, indent)
+          self.set('value', element.serialize_bool(self.value))
+        def deserialize(self, ref_list):
+          PLex.Section1.Options.Option.deserialize(self, ref_list)
+          self.value = element.deserialize_bool(self.get('value', 'false'))
+        def copy(self, factory = None):
+          result = PLex.Section1.Options.Option.copy(
+            self,
+            BoolOption if factory is None else factory
+          )
+          result.value = self.value
+          return result
+        def repr_serialize(self, params):
+          PLex.Section1.Options.Option.repr_serialize(self, params)
+          if self.value != False:
+            params.append(
+              'value = {0:s}'.format(repr(self.value))
+            )
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section1.Options.BoolOption({0:s})'.format(', '.join(params))
+        # GENERATE END
+
+      class ECS(BoolOption):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section1_Options_ECS',
+          attrib = {},
+          text = '',
+          children = [],
+          value = False
+        ):
+          PLex.Section1.Options.BoolOption.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children,
+            value
+          )
+        def copy(self, factory = None):
+          result = PLex.Section1.Options.BoolOption.copy(
+            self,
+            ECS if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section1.Options.ECS({0:s})'.format(', '.join(params))
+        # GENERATE END
+        def post_process(self, section):
+          section.ecs = self.value
+
+      class MetaECS(BoolOption):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section1_Options_MetaECS',
+          attrib = {},
+          text = '',
+          children = [],
+          value = False
+        ):
+          PLex.Section1.Options.BoolOption.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children,
+            value
+          )
+        def copy(self, factory = None):
+          result = PLex.Section1.Options.BoolOption.copy(
+            self,
+            MetaECS if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section1.Options.MetaECS({0:s})'.format(', '.join(params))
+        # GENERATE END
+        def post_process(self, section):
+          section.meta_ecs = self.value
+
+      class Reject(BoolOption):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section1_Options_Reject',
+          attrib = {},
+          text = '',
+          children = [],
+          value = False
+        ):
+          PLex.Section1.Options.BoolOption.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children,
+            value
+          )
+        def copy(self, factory = None):
+          result = PLex.Section1.Options.BoolOption.copy(
+            self,
+            Reject if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section1.Options.Reject({0:s})'.format(', '.join(params))
+        # GENERATE END
+        def post_process(self, section):
+          section.reject = self.value
+
+      class YYMore(BoolOption):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section1_Options_YYMore',
+          attrib = {},
+          text = '',
+          children = [],
+          value = False
+        ):
+          PLex.Section1.Options.BoolOption.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children,
+            value
+          )
+        def copy(self, factory = None):
+          result = PLex.Section1.Options.BoolOption.copy(
+            self,
+            YYMore if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section1.Options.YYMore({0:s})'.format(', '.join(params))
+        # GENERATE END
+        def post_process(self, section):
+          section.yymore = self.value
+
+      class YYWrap(BoolOption):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section1_Options_YYWrap',
+          attrib = {},
+          text = '',
+          children = [],
+          value = False
+        ):
+          PLex.Section1.Options.BoolOption.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children,
+            value
+          )
+        def copy(self, factory = None):
+          result = PLex.Section1.Options.BoolOption.copy(
+            self,
+            YYWrap if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section1.Options.YYWrap({0:s})'.format(', '.join(params))
+        # GENERATE END
+        def post_process(self, section):
+          section.yywrap = self.value
 
-  class Rule(Item):
-    class BOLRule(element.Element):
       # GENERATE ELEMENT() BEGIN
       def __init__(
         self,
-        tag = 'Section2_Rule_BOLRule',
+        tag = 'PLex_Section1_Options',
         attrib = {},
         text = '',
         children = []
       ):
-        element.Element.__init__(
+        Item.__init__(
           self,
           tag,
           attrib,
@@ -695,25 +522,227 @@ class Section2(SectionCodeBlocks):
           children
         )
       def copy(self, factory = None):
-        result = element.Element.copy(
+        result = Item.copy(
           self,
-          BOLRule if factory is None else factory
+          Options if factory is None else factory
         )
         return result
       def __repr__(self):
         params = []
         self.repr_serialize(params)
-        return 'ast.Section2.Rule.BOLRule({0:s})'.format(', '.join(params))
+        return 'ast.PLex.Section1.Options({0:s})'.format(', '.join(params))
       # GENERATE END
+      def post_process(
+        self,
+        plex,
+        section,
+        name_to_start_condition,
+        all_start_conditions,
+        inclusive_start_conditions
+      ):
+        for i in self:
+          i.post_process(section)
 
-    class EOFRule(element.Element):
-      # GENERATE ELEMENT() BEGIN
+    class StartConditions(Item):
+      # GENERATE ELEMENT(bool exclusive) BEGIN
       def __init__(
         self,
-        tag = 'Section2_Rule_EOFRule',
+        tag = 'PLex_Section1_StartConditions',
         attrib = {},
         text = '',
-        children = []
+        children = [],
+        exclusive = False
+      ):
+        Item.__init__(
+          self,
+          tag,
+          attrib,
+          text,
+          children
+        )
+        self.exclusive = (
+          element.deserialize_bool(exclusive)
+        if isinstance(exclusive, str) else
+          exclusive
+        )
+      def serialize(self, ref_list, indent = 0):
+        Item.serialize(self, ref_list, indent)
+        self.set('exclusive', element.serialize_bool(self.exclusive))
+      def deserialize(self, ref_list):
+        Item.deserialize(self, ref_list)
+        self.exclusive = element.deserialize_bool(self.get('exclusive', 'false'))
+      def copy(self, factory = None):
+        result = Item.copy(
+          self,
+          StartConditions if factory is None else factory
+        )
+        result.exclusive = self.exclusive
+        return result
+      def repr_serialize(self, params):
+        Item.repr_serialize(self, params)
+        if self.exclusive != False:
+          params.append(
+            'exclusive = {0:s}'.format(repr(self.exclusive))
+          )
+      def __repr__(self):
+        params = []
+        self.repr_serialize(params)
+        return 'ast.PLex.Section1.StartConditions({0:s})'.format(', '.join(params))
+      # GENERATE END
+      def post_process(
+        self,
+        plex,
+        section,
+        name_to_start_condition,
+        all_start_conditions,
+        inclusive_start_conditions
+      ):
+        for i in self:
+          assert isinstance(i, PLex.Name)
+          name = element.get_text(i, 0)
+          assert name not in name_to_start_condition
+          name_to_start_condition[name] = len(plex.start_conditions)
+          all_start_conditions.add(len(plex.start_conditions))
+          if not self.exclusive:
+            inclusive_start_conditions.add(len(plex.start_conditions))
+          plex.start_conditions.append(
+            PLex.StartCondition(
+              children = [regex.RegexNone(), regex.RegexNone()], # [normal, BOL]
+              name = name,
+              exclusive = self.exclusive,
+              eof_action = 0
+            )
+          )
+
+    # GENERATE ELEMENT(bool ecs, bool meta_ecs, bool reject, bool yymore, bool yywrap) BEGIN
+    def __init__(
+      self,
+      tag = 'PLex_Section1',
+      attrib = {},
+      text = '',
+      children = [],
+      code_blocks = [],
+      ecs = False,
+      meta_ecs = False,
+      reject = False,
+      yymore = False,
+      yywrap = False
+    ):
+      PLex.SectionCodeBlocks.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children,
+        code_blocks
+      )
+      self.ecs = (
+        element.deserialize_bool(ecs)
+      if isinstance(ecs, str) else
+        ecs
+      )
+      self.meta_ecs = (
+        element.deserialize_bool(meta_ecs)
+      if isinstance(meta_ecs, str) else
+        meta_ecs
+      )
+      self.reject = (
+        element.deserialize_bool(reject)
+      if isinstance(reject, str) else
+        reject
+      )
+      self.yymore = (
+        element.deserialize_bool(yymore)
+      if isinstance(yymore, str) else
+        yymore
+      )
+      self.yywrap = (
+        element.deserialize_bool(yywrap)
+      if isinstance(yywrap, str) else
+        yywrap
+      )
+    def serialize(self, ref_list, indent = 0):
+      PLex.SectionCodeBlocks.serialize(self, ref_list, indent)
+      self.set('ecs', element.serialize_bool(self.ecs))
+      self.set('meta_ecs', element.serialize_bool(self.meta_ecs))
+      self.set('reject', element.serialize_bool(self.reject))
+      self.set('yymore', element.serialize_bool(self.yymore))
+      self.set('yywrap', element.serialize_bool(self.yywrap))
+    def deserialize(self, ref_list):
+      PLex.SectionCodeBlocks.deserialize(self, ref_list)
+      self.ecs = element.deserialize_bool(self.get('ecs', 'false'))
+      self.meta_ecs = element.deserialize_bool(self.get('meta_ecs', 'false'))
+      self.reject = element.deserialize_bool(self.get('reject', 'false'))
+      self.yymore = element.deserialize_bool(self.get('yymore', 'false'))
+      self.yywrap = element.deserialize_bool(self.get('yywrap', 'false'))
+    def copy(self, factory = None):
+      result = PLex.SectionCodeBlocks.copy(
+        self,
+        Section1 if factory is None else factory
+      )
+      result.ecs = self.ecs
+      result.meta_ecs = self.meta_ecs
+      result.reject = self.reject
+      result.yymore = self.yymore
+      result.yywrap = self.yywrap
+      return result
+    def repr_serialize(self, params):
+      PLex.SectionCodeBlocks.repr_serialize(self, params)
+      if self.ecs != False:
+        params.append(
+          'ecs = {0:s}'.format(repr(self.ecs))
+        )
+      if self.meta_ecs != False:
+        params.append(
+          'meta_ecs = {0:s}'.format(repr(self.meta_ecs))
+        )
+      if self.reject != False:
+        params.append(
+          'reject = {0:s}'.format(repr(self.reject))
+        )
+      if self.yymore != False:
+        params.append(
+          'yymore = {0:s}'.format(repr(self.yymore))
+        )
+      if self.yywrap != False:
+        params.append(
+          'yywrap = {0:s}'.format(repr(self.yywrap))
+        )
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.PLex.Section1({0:s})'.format(', '.join(params))
+    # GENERATE END
+    def post_process(
+      self,
+      plex,
+      name_to_start_condition,
+      all_start_conditions,
+      inclusive_start_conditions
+    ):
+      self.ecs = False
+      self.meta_ecs = False
+      self.reject = True
+      self.yymore = True
+      self.yywrap = True
+      PLex.Section.post_process(
+        self,
+        plex,
+        name_to_start_condition,
+        all_start_conditions,
+        inclusive_start_conditions
+      )
+
+  class Section2(SectionCodeBlocks):
+    class StartConditions(element.Element):
+      # GENERATE ELEMENT(bool wildcard) BEGIN
+      def __init__(
+        self,
+        tag = 'PLex_Section2_StartConditions',
+        attrib = {},
+        text = '',
+        children = [],
+        wildcard = False
       ):
         element.Element.__init__(
           self,
@@ -722,28 +751,130 @@ class Section2(SectionCodeBlocks):
           text,
           children
         )
+        self.wildcard = (
+          element.deserialize_bool(wildcard)
+        if isinstance(wildcard, str) else
+          wildcard
+        )
+      def serialize(self, ref_list, indent = 0):
+        element.Element.serialize(self, ref_list, indent)
+        self.set('wildcard', element.serialize_bool(self.wildcard))
+      def deserialize(self, ref_list):
+        element.Element.deserialize(self, ref_list)
+        self.wildcard = element.deserialize_bool(self.get('wildcard', 'false'))
       def copy(self, factory = None):
         result = element.Element.copy(
           self,
-          EOFRule if factory is None else factory
+          StartConditions if factory is None else factory
         )
+        result.wildcard = self.wildcard
         return result
+      def repr_serialize(self, params):
+        element.Element.repr_serialize(self, params)
+        if self.wildcard != False:
+          params.append(
+            'wildcard = {0:s}'.format(repr(self.wildcard))
+          )
       def __repr__(self):
         params = []
         self.repr_serialize(params)
-        return 'ast.Section2.Rule.EOFRule({0:s})'.format(', '.join(params))
+        return 'ast.PLex.Section2.StartConditions({0:s})'.format(', '.join(params))
       # GENERATE END
 
-    class Action(element.Element):
+    class Rule(Item):
+      class BOLRule(element.Element):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section2_Rule_BOLRule',
+          attrib = {},
+          text = '',
+          children = []
+        ):
+          element.Element.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children
+          )
+        def copy(self, factory = None):
+          result = element.Element.copy(
+            self,
+            BOLRule if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section2.Rule.BOLRule({0:s})'.format(', '.join(params))
+        # GENERATE END
+
+      class EOFRule(element.Element):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section2_Rule_EOFRule',
+          attrib = {},
+          text = '',
+          children = []
+        ):
+          element.Element.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children
+          )
+        def copy(self, factory = None):
+          result = element.Element.copy(
+            self,
+            EOFRule if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section2.Rule.EOFRule({0:s})'.format(', '.join(params))
+        # GENERATE END
+
+      class Action(element.Element):
+        # GENERATE ELEMENT() BEGIN
+        def __init__(
+          self,
+          tag = 'PLex_Section2_Rule_Action',
+          attrib = {},
+          text = '',
+          children = []
+        ):
+          element.Element.__init__(
+            self,
+            tag,
+            attrib,
+            text,
+            children
+          )
+        def copy(self, factory = None):
+          result = element.Element.copy(
+            self,
+            Action if factory is None else factory
+          )
+          return result
+        def __repr__(self):
+          params = []
+          self.repr_serialize(params)
+          return 'ast.PLex.Section2.Rule.Action({0:s})'.format(', '.join(params))
+        # GENERATE END
+
       # GENERATE ELEMENT() BEGIN
       def __init__(
         self,
-        tag = 'Section2_Rule_Action',
+        tag = 'PLex_Section2_Rule',
         attrib = {},
         text = '',
         children = []
       ):
-        element.Element.__init__(
+        Item.__init__(
           self,
           tag,
           attrib,
@@ -751,26 +882,119 @@ class Section2(SectionCodeBlocks):
           children
         )
       def copy(self, factory = None):
-        result = element.Element.copy(
+        result = Item.copy(
           self,
-          Action if factory is None else factory
+          Rule if factory is None else factory
         )
         return result
       def __repr__(self):
         params = []
         self.repr_serialize(params)
-        return 'ast.Section2.Rule.Action({0:s})'.format(', '.join(params))
+        return 'ast.PLex.Section2.Rule({0:s})'.format(', '.join(params))
       # GENERATE END
+      def post_process(
+        self,
+        plex,
+        section,
+        name_to_start_condition,
+        all_start_conditions,
+        inclusive_start_conditions
+      ):
+        start_conditions = self[0]
+        assert isinstance(start_conditions, PLex.Section2.StartConditions)
+        default = False
+        if start_conditions.wildcard:
+          start_condition_set = all_start_conditions
+        elif len(start_conditions) == 0:
+          default = True
+          start_condition_set = inclusive_start_conditions
+        else:
+          start_condition_set = set()
+          for i in start_conditions:
+            assert isinstance(i, PLex.Name)
+            start_condition_set.add(
+              name_to_start_condition[element.get_text(i, 0)]
+            )
+        expr = self[1]
+        trailing_context = self[2]
+        assert isinstance(trailing_context, regex.Regex)
+        action = self[3]
+        assert isinstance(action, PLex.Section2.Rule.Action)
+        if isinstance(expr, PLex.Section2.Rule.EOFRule):
+          assert isinstance(trailing_context, regex.RegexNone)
+          for i in start_condition_set:
+            if default and plex.start_conditions[i].eof_action != 0:
+              continue # rule applies to start conditions with no EOF rule yet
+            assert plex.start_conditions[i].eof_action == 0
+            plex.start_conditions[i].eof_action = len(plex.eof_actions)
+          plex.eof_actions.append(action)
+        else:
+          if isinstance(expr, PLex.Section2.Rule.BOLRule):
+            bol_rule = True
+            expr = expr[0]
+          else:
+            bol_rule = False
+          assert isinstance(expr, regex.Regex)
+          expr = regex.RegexSequence(
+            children = [
+              expr,
+              regex.RegexGroup(
+                children = [
+                  trailing_context
+                ]
+              )
+            ]
+          )
+          expr.post_process(len(plex.actions))
+          for j in start_condition_set:
+            for k in range(int(bol_rule), 2):
+              plex.start_conditions[j][k] = regex.RegexOr(
+                children = [
+                  plex.start_conditions[j][k],
+                  expr
+                ]
+              )
+          plex.actions.append(action)
+
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'PLex_Section2',
+      attrib = {},
+      text = '',
+      children = [],
+      code_blocks = []
+    ):
+      PLex.SectionCodeBlocks.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children,
+        code_blocks
+      )
+    def copy(self, factory = None):
+      result = PLex.SectionCodeBlocks.copy(
+        self,
+        Section2 if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.PLex.Section2({0:s})'.format(', '.join(params))
+    # GENERATE END
 
+  class Section3(Section):
     # GENERATE ELEMENT() BEGIN
     def __init__(
       self,
-      tag = 'Section2_Rule',
+      tag = 'PLex_Section3',
       attrib = {},
       text = '',
       children = []
     ):
-      Item.__init__(
+      PLex.Section.__init__(
         self,
         tag,
         attrib,
@@ -778,101 +1002,195 @@ class Section2(SectionCodeBlocks):
         children
       )
     def copy(self, factory = None):
-      result = Item.copy(
+      result = PLex.Section.copy(
         self,
-        Rule if factory is None else factory
+        Section3 if factory is None else factory
       )
       return result
     def __repr__(self):
       params = []
       self.repr_serialize(params)
-      return 'ast.Section2.Rule({0:s})'.format(', '.join(params))
+      return 'ast.PLex.Section3({0:s})'.format(', '.join(params))
     # GENERATE END
-    def post_process(self, plex_specification, section):
-      pass # do something here eventually
 
-  # GENERATE ELEMENT() BEGIN
+  # GENERATE ELEMENT(list(ref) start_conditions, list(ref) actions, list(ref) eof_actions) BEGIN
   def __init__(
     self,
-    tag = 'Section2',
+    tag = 'PLex',
     attrib = {},
     text = '',
     children = [],
-    code_blocks = []
+    start_conditions = [],
+    actions = [],
+    eof_actions = []
   ):
-    SectionCodeBlocks.__init__(
+    element.Element.__init__(
       self,
       tag,
       attrib,
       text,
-      children,
-      code_blocks
+      children
     )
-  def copy(self, factory = None):
-    result = SectionCodeBlocks.copy(
-      self,
-      Section2 if factory is None else factory
+    self.start_conditions = start_conditions
+    self.actions = actions
+    self.eof_actions = eof_actions
+  def serialize(self, ref_list, indent = 0):
+    element.Element.serialize(self, ref_list, indent)
+    self.set(
+      'start_conditions',
+      ' '.join([element.serialize_ref(i, ref_list) for i in self.start_conditions])
     )
-    return result
-  def __repr__(self):
-    params = []
-    self.repr_serialize(params)
-    return 'ast.Section2({0:s})'.format(', '.join(params))
-  # GENERATE END
-
-class Section3(Section):
-  # GENERATE ELEMENT() BEGIN
-  def __init__(
-    self,
-    tag = 'Section3',
-    attrib = {},
-    text = '',
-    children = []
-  ):
-    Section.__init__(
-      self,
-      tag,
-      attrib,
-      text,
-      children
+    self.set(
+      'actions',
+      ' '.join([element.serialize_ref(i, ref_list) for i in self.actions])
+    )
+    self.set(
+      'eof_actions',
+      ' '.join([element.serialize_ref(i, ref_list) for i in self.eof_actions])
     )
+  def deserialize(self, ref_list):
+    element.Element.deserialize(self, ref_list)
+    self.start_conditions = [
+      element.deserialize_ref(i, ref_list)
+      for i in self.get('start_conditions', '').split()
+    ]
+    self.actions = [
+      element.deserialize_ref(i, ref_list)
+      for i in self.get('actions', '').split()
+    ]
+    self.eof_actions = [
+      element.deserialize_ref(i, ref_list)
+      for i in self.get('eof_actions', '').split()
+    ]
   def copy(self, factory = None):
-    result = Section.copy(
+    result = element.Element.copy(
       self,
-      Section3 if factory is None else factory
+      PLex if factory is None else factory
     )
+    result.start_conditions = self.start_conditions
+    result.actions = self.actions
+    result.eof_actions = self.eof_actions
     return result
+  def repr_serialize(self, params):
+    element.Element.repr_serialize(self, params)
+    if len(self.start_conditions):
+      params.append(
+        'start_conditions = [{0:s}]'.format(
+          ', '.join([repr(i) for i in self.start_conditions])
+        )
+      )
+    if len(self.actions):
+      params.append(
+        'actions = [{0:s}]'.format(
+          ', '.join([repr(i) for i in self.actions])
+        )
+      )
+    if len(self.eof_actions):
+      params.append(
+        'eof_actions = [{0:s}]'.format(
+          ', '.join([repr(i) for i in self.eof_actions])
+        )
+      )
   def __repr__(self):
     params = []
     self.repr_serialize(params)
-    return 'ast.Section3({0:s})'.format(', '.join(params))
+    return 'ast.PLex({0:s})'.format(', '.join(params))
   # GENERATE END
+  def post_process(self):
+    # variables that will be serialized
+    self.start_conditions = [
+      PLex.StartCondition(
+        children = [regex.RegexNone(), regex.RegexNone()], # [normal, BOL]
+        name = 'INITIAL',
+        exclusive = False,
+        eof_action = 0
+      )
+    ]
+    self.actions = []
+    self.eof_actions = [
+      PLex.Section2.Rule.Action(text = '\t\t\t\tyyterminate();\n')
+    ]
+
+    # variables that won't be serialized
+    name_to_start_condition = {'INITIAL': 0}
+    all_start_conditions = set([0])
+    inclusive_start_conditions = set([0])
+
+    for i in self:
+      i.post_process(
+        self,
+        name_to_start_condition,
+        all_start_conditions,
+        inclusive_start_conditions
+      )
+
+    # add default rule to each expr, then make it greedy
+    for i in range(len(self.start_conditions)):
+      for j in range(2):
+        self.start_conditions[i][j] = regex.RegexAnd(
+          children = [
+            regex.RegexRepeat(
+              count0 = 0,
+              children = [
+                regex.RegexCharacter(
+                  char_set = [0, 0x100]
+                )
+              ]
+            ),
+            regex.RegexOr(
+              children = [
+                self.start_conditions[i][j],
+                regex.RegexSequence(
+                  children = [
+                    regex.RegexCharacter(
+                      char_set = [0, 0x100]
+                    ),
+                    regex.RegexGroup(
+                      group_index = len(self.actions),
+                      children = [
+                        regex.RegexEmpty()
+                      ]
+                    )
+                  ]
+                )
+              ]
+            )
+          ]
+        )
+    self.actions.append(PLex.Section2.Rule.Action(text = 'ECHO;\n'))
+  def to_nfa(self):
+    nfa = regex.NFA()
+    for i in self.start_conditions:
+      for j in i:
+        j.add_to_nfa(nfa)
+    return nfa
 
 # GENERATE FACTORY(regex.factory) BEGIN
 tag_to_class = {
-  'PLexSpecification': PLexSpecification,
-  'Section': Section,
   'Item': Item,
-  'Name': Name,
-  'SectionCodeBlocks': SectionCodeBlocks,
-  'CodeBlock': CodeBlock,
-  'Option': Option,
-  'BoolOption': BoolOption,
-  'Section1': Section1,
-  'Section1_Options': Section1.Options,
-  'Section1_Options_ECS': Section1.Options.ECS,
-  'Section1_Options_MetaECS': Section1.Options.MetaECS,
-  'Section1_Options_Reject': Section1.Options.Reject,
-  'Section1_Options_YYMore': Section1.Options.YYMore,
-  'Section1_Options_YYWrap': Section1.Options.YYWrap,
-  'Section1_StartConditions': Section1.StartConditions,
-  'Section2': Section2,
-  'Section2_StartConditions': Section2.StartConditions,
-  'Section2_Rule': Section2.Rule,
-  'Section2_Rule_BOLRule': Section2.Rule.BOLRule,
-  'Section2_Rule_EOFRule': Section2.Rule.EOFRule,
-  'Section2_Rule_Action': Section2.Rule.Action,
-  'Section3': Section3
+  'PLex': PLex,
+  'PLex_StartCondition': PLex.StartCondition,
+  'PLex_Section': PLex.Section,
+  'PLex_Name': PLex.Name,
+  'PLex_SectionCodeBlocks': PLex.SectionCodeBlocks,
+  'PLex_CodeBlock': PLex.CodeBlock,
+  'PLex_Section1': PLex.Section1,
+  'PLex_Section1_Options': PLex.Section1.Options,
+  'PLex_Section1_Options_Option': PLex.Section1.Options.Option,
+  'PLex_Section1_Options_BoolOption': PLex.Section1.Options.BoolOption,
+  'PLex_Section1_Options_ECS': PLex.Section1.Options.ECS,
+  'PLex_Section1_Options_MetaECS': PLex.Section1.Options.MetaECS,
+  'PLex_Section1_Options_Reject': PLex.Section1.Options.Reject,
+  'PLex_Section1_Options_YYMore': PLex.Section1.Options.YYMore,
+  'PLex_Section1_Options_YYWrap': PLex.Section1.Options.YYWrap,
+  'PLex_Section1_StartConditions': PLex.Section1.StartConditions,
+  'PLex_Section2': PLex.Section2,
+  'PLex_Section2_StartConditions': PLex.Section2.StartConditions,
+  'PLex_Section2_Rule': PLex.Section2.Rule,
+  'PLex_Section2_Rule_BOLRule': PLex.Section2.Rule.BOLRule,
+  'PLex_Section2_Rule_EOFRule': PLex.Section2.Rule.EOFRule,
+  'PLex_Section2_Rule_Action': PLex.Section2.Rule.Action,
+  'PLex_Section3': PLex.Section3
 }
 def factory(tag, attrib = {}, *args, **kwargs):
   return tag_to_class.get(tag, regex.factory)(tag, attrib, *args, **kwargs)

diff --git a/plex.py b/bootstrap_plex.py
similarity index 71%
rename from plex.py
rename to bootstrap_plex.py
index df4d1d1..9f4e777 100755 (executable)
--- a/plex.py
+++ b/bootstrap_plex.py
@@ -230,163 +230,14 @@ if len(args) < 1:
   sys.exit(1)
 in_file = args[0]
 
-#root = element.Element('root')
-#mark = []
-#macro_dict = {}
-#with open(in_file) as fin:
-#  assert not yacc.yyparse(
-#    root,
-#    mark,
-#    lex.yylex(root, mark, macro_dict, work.yychunk_line(root, fin))
-#  )
-#def post_process(node):
-#  if isinstance(node, regex.Regex):
-#    node.post_process()
-#  else:
-#    for i in node:
-#      post_process(i)
-#post_process(root)
-with open(in_file + '.xml') as fin:
-  root = element.deserialize(fin, ast.factory)
-#xml.etree.ElementTree.dump(root)
+with open(in_file) as fin:
+  plex = element.deserialize(fin, ast.factory)
+plex.post_process()
 
-class StartCondition:
-  def __init__(self, name, eof_action):
-    self.name = name
-    self.eof_action = eof_action
-name_to_start_condition = {'INITIAL': 0}
-inclusive_start_conditions = set([0])
-start_conditions = [StartCondition('INITIAL', 0)]
-
-section1 = root[0]
-assert isinstance(section1, ast.Section1)
-section2 = root[1]
-assert isinstance(section2, ast.Section2)
-if len(root) < 3:
-  section3 = ast.Section3()
-else:
-  section3 = root[2]
-  assert isinstance(section3, ast.Section3)
-
-root.post_process()
-for i in section1:
-  if isinstance(i, ast.Section1.StartConditions):
-    for j in i:
-      assert isinstance(j, ast.Name)
-      name = element.get_text(j, 0)
-      assert name not in name_to_start_condition
-      name_to_start_condition[name] = len(start_conditions)
-      if not i.exclusive:
-        inclusive_start_conditions.add(len(start_conditions))
-      start_conditions.append(StartCondition(name, 0))
-
-actions = []
-eof_actions = [ast.Section2.Rule.Action(text = '\t\t\t\tyyterminate();\n')]
-start_condition_exprs = [
-  regex.RegexNone()
-  for i in range(len(start_conditions) * 2) # normal followed by BOL expr
-]
-all_start_conditions = set(range(len(start_conditions)))
-for i in section2:
-  if isinstance(i, ast.Section2.Rule):
-    rule_start_conditions = i[0]
-    assert isinstance(rule_start_conditions, ast.Section2.StartConditions)
-    default = False
-    if rule_start_conditions.wildcard:
-      start_condition_set = all_start_conditions
-    elif len(rule_start_conditions) == 0:
-      default = True
-      start_condition_set = inclusive_start_conditions
-    else:
-      start_condition_set = set()
-      for j in rule_start_conditions:
-        assert isinstance(j, ast.Name)
-        start_condition_set.add(
-          name_to_start_condition[element.get_text(j, 0)]
-        )
-    rule_expr = i[1]
-    rule_trailing_context = i[2]
-    assert isinstance(rule_trailing_context, regex.Regex)
-    rule_action = i[3]
-    assert isinstance(rule_action, ast.Section2.Rule.Action)
-    if isinstance(rule_expr, ast.Section2.Rule.EOFRule):
-      assert isinstance(rule_trailing_context, regex.RegexNone)
-      for j in start_condition_set:
-        if default and start_conditions[j].eof_action != 0:
-          continue # rule applies to start conditions with no EOF rule yet
-        assert start_conditions[j].eof_action == 0
-        start_conditions[j].eof_action = len(eof_actions)
-      eof_actions.append(rule_action)
-    else:
-      if isinstance(rule_expr, ast.Section2.Rule.BOLRule):
-        bol_rule = True
-        rule_expr = rule_expr[0]
-      else:
-        bol_rule = False
-      assert isinstance(rule_expr, regex.Regex)
-      rule_expr = regex.RegexSequence(
-        children = [
-          rule_expr,
-          regex.RegexGroup(
-            children = [
-              rule_trailing_context
-            ]
-          )
-        ]
-      )
-      rule_expr.post_process(len(actions))
-      for j in start_condition_set:
-        for k in range(j * 2 + int(bol_rule), j * 2 + 2):
-          start_condition_exprs[k] = regex.RegexOr(
-            children = [
-              start_condition_exprs[k],
-              rule_expr
-            ]
-          )
-      actions.append(rule_action)
-
-nfa = regex.NFA()
-for i in range(len(start_condition_exprs)):
-  # make expr match as much as possible
-  # add default rule to match one char
-  start_condition_exprs[i] = regex.RegexAnd(
-    children = [
-      regex.RegexRepeat(
-        count0 = 0,
-        children = [
-          regex.RegexCharacter(
-            char_set = [0, 0x100]
-          )
-        ]
-      ),
-      regex.RegexOr(
-        children = [
-          start_condition_exprs[i],
-          regex.RegexSequence(
-            children = [
-              regex.RegexCharacter(
-                char_set = [0, 0x100]
-              ),
-              regex.RegexGroup(
-                group_index = len(actions),
-                children = [
-                  regex.RegexEmpty()
-                ]
-              )
-            ]
-          )
-        ]
-      )
-    ]
-  )
-  #print('i', i, 'expr', repr(start_condition_exprs[i]))
-  start_condition_exprs[i].add_to_nfa(nfa)
-actions.append(ast.Section2.Rule.Action(text = 'ECHO;\n'))
+nfa = plex.to_nfa()
 eob_expr = regex.RegexGroup(children = [regex.RegexEmpty()])
-eob_expr.post_process(len(actions))
-#print('eob expr', repr(eob_expr))
+eob_expr.post_process(len(plex.actions))
 eob_expr.add_to_nfa(nfa)
-
 dfa = nfa.to_dfa()
 #print(dfa.start_action)
 #print(dfa.actions[2])
@@ -402,13 +253,7 @@ with open(skel_file, 'r') as fin:
           '''/* GENERATE SECTION1 BEGIN */
 {0:s}/* GENERATE SECTION1 END*/
 '''.format(
-            ''.join(
-              [
-                element.get_text(i, 0)
-                for i in section1
-                if isinstance(i, ast.CodeBlock)
-              ]
-            )
+            ''.join([element.get_text(i, 0) for i in plex[0].code_blocks])
           )
         )
       elif line == '/* GENERATE STARTCONDDECL */\n':
@@ -418,8 +263,11 @@ with open(skel_file, 'r') as fin:
 '''.format(
             ''.join(
               [
-                '#define {0:s} {1:d}\n'.format(start_conditions[i].name, i)
-                for i in range(len(start_conditions))
+                '#define {0:s} {1:d}\n'.format(
+                  plex.start_conditions[i].name,
+                  i
+                )
+                for i in range(len(plex.start_conditions))
               ]
             )
           )
@@ -447,7 +295,7 @@ static const flex_int16_t yy_chk[] = {{{6:s}
 }};
 /* GENERATE TABLES END */
 '''.format(
-            len(actions),
+            len(plex.actions),
             ','.join(
               [
                 '\n\t{0:s}'.format(
@@ -533,23 +381,17 @@ static const flex_int16_t yy_chk[] = {{{6:s}
           '''/* GENERATE SECTION2INITIAL BEGIN */
 {0:s}/* GENERATE SECTION2INITIAL END */
 '''.format(
-            ''.join(
-              [
-                element.get_text(i, 0)
-                for i in section2
-                if isinstance(i, ast.CodeBlock)
-              ]
-            )
+            ''.join([element.get_text(i, 0) for i in plex[1].code_blocks])
           )
         )
       elif line == '/* GENERATE SECTION2 */\n':
         eof_action_to_start_conditions = [
           [
             j
-            for j in range(len(start_conditions))
-            if start_conditions[i].eof_action == j
+            for j in range(len(plex.start_conditions))
+            if plex.start_conditions[i].eof_action == j
           ]
-          for i in range(len(eof_actions))
+          for i in range(len(plex.eof_actions))
         ]
         #print('eof_action_to_start_conditions', eof_action_to_start_conditions)
         fout.write(
@@ -563,9 +405,9 @@ YY_RULE_SETUP
 {1:s}  YY_BREAK
 '''.format(
                   i,
-                  element.get_text(actions[i], 0)
+                  element.get_text(plex.actions[i], 0)
                 )
-                for i in range(len(actions))
+                for i in range(len(plex.actions))
               ]
             ),
             ''.join(
@@ -574,14 +416,14 @@ YY_RULE_SETUP
                   ''.join(
                     [
                       '\t\t\tcase YY_STATE_EOF({0:s}):\n'.format(
-                        start_conditions[j].name
+                        plex.start_conditions[j].name
                       )
                       for j in eof_action_to_start_conditions[i]
                     ]
                   ),
-                  element.get_text(eof_actions[i], 0)
+                  element.get_text(plex.eof_actions[i], 0)
                 )
-                for i in range(len(eof_actions))
+                for i in range(len(plex.eof_actions))
                 if len(eof_action_to_start_conditions[i]) > 0
               ]
             )
@@ -592,7 +434,7 @@ YY_RULE_SETUP
           '''/* GENERATE SECTION3 BEGIN */
 {0:s}/*GENERATE SECTION3 END */
 '''.format(
-            element.get_text(section3, 0)
+            '' if len(plex) < 3 else element.get_text(plex[2], 0)
           )
         )
       else:

diff --git a/generate.py b/generate.py
index 434da3f..169c348 100755 (executable)
--- a/generate.py
+++ b/generate.py
@@ -16,7 +16,11 @@ default_value = {
   'list(bool)': '[]',
   'list(int)': '[]',
   'list(ref)': '[]',
-  'list(str)': '[]'
+  'list(str)': '[]',
+  'set(bool)': 'set()',
+  'set(int)': 'set()',
+  'set(ref)': 'set()',
+  'set(str)': 'set()'
 }
 default_value_str = {
   'bool': 'false',
@@ -36,7 +40,7 @@ re_factory = re.compile(
 )
 stack = []
 classes = []
-base_classes = {'element.Element': []} # params
+base_classes = [{'element.Element': []}] # params
 
 line = sys.stdin.readline()
 while len(line):
@@ -46,19 +50,26 @@ while len(line):
     indent = match.group(1)
     class_name = match.group(2)
     base_class = match.group(3)
-    for i in range(len(stack)):
-      if stack[i][0][:len(indent)] == indent:
-        del stack[i:]
+    while len(stack) and stack[-1][0][:len(indent)] == indent:
+      _, temp_class_name, _, _ = stack.pop()
+      for temp_base_class, temp_fields in base_classes.pop().items():
+        base_classes[-1][
+          '{0:s}.{1:s}'.format(temp_class_name, temp_base_class)
+        ] = temp_fields
+    for i in range(len(base_classes) - 1, -1, -1):
+      if base_class in base_classes[i]:
+        classes.append(
+          '.'.join([j for _, j, _, _ in stack] + [class_name])
+        )
+        full_base_class = '.'.join(
+          [j for _, j, _, _ in stack[:i]] + [base_class]
+        )
+        base_classes[-1][class_name] = list(base_classes[i][base_class])
         break
-    full_name = (
-      '{0:s}.{1:s}'.format(stack[-1][2], class_name)
-    if len(stack) else
-      class_name
-    )
-    stack.append((indent, class_name, full_name, base_class))
-    if base_class in base_classes:
-      classes.append(full_name)
-      base_classes[full_name] = list(base_classes[base_class])
+    else:
+      full_base_class = base_class
+    stack.append((indent, class_name, base_class, full_base_class))
+    base_classes.append({})
   else:
     match = re_element.match(line)
     if match is not None:
@@ -66,18 +77,21 @@ while len(line):
       params = match.group(2)
       begin = match.group(4)
 
-      for i in range(len(stack)):
-        if stack[i][0][:len(indent)] == indent:
-          del stack[i:]
-          break
-      _, class_name, full_name, base_class = stack[-1]
+      while len(stack) and stack[-1][0][:len(indent)] == indent:
+        _, temp_class_name, _, _ = stack.pop()
+        for temp_base_class, temp_fields in base_classes.pop().items():
+          base_classes[-1][
+            '{0:s}.{1:s}'.format(temp_class_name, temp_base_class)
+          ] = temp_fields
+      _, class_name, base_class, full_base_class = stack[-1]
  
       fields = params.split(',')
       if fields[-1] == '':
         del fields[-1:]
       fields = [i.split() for i in fields]
       fields = [(type, name) for [type, name] in fields]
-      base_classes[full_name].extend(fields)
+      i = len(base_classes[-2][class_name])
+      base_classes[-2][class_name].extend(fields)
 
       sys.stdout.write(
         '''{0:s}# GENERATE ELEMENT({1:s}) BEGIN
@@ -101,7 +115,7 @@ while len(line):
           indent,
           indent,
           indent,
-          full_name.replace('.', '_'),
+          '_'.join([i for _, i, _, _ in stack]),
           indent,
           indent,
           indent,
@@ -112,12 +126,12 @@ while len(line):
                 name,
                 default_value[type]
               )
-              for type, name in base_classes[full_name]
+              for type, name in base_classes[-2][class_name]
             ]
           ),
           indent,
           indent,
-          base_class,  
+          full_base_class,
           indent,
           indent,
           indent,
@@ -129,14 +143,14 @@ while len(line):
                 indent,
                 name
               )
-              for type, name in base_classes[base_class]
+              for type, name in base_classes[-2][class_name][:i]
             ]
           ),
           indent
         )
       )
       for type, name in fields:
-        if type == 'ref' or type == 'list(ref)' or type == 'str':
+        if type == 'ref' or type == 'list(ref)' or type == 'set(ref)' or type == 'str':
           sys.stdout.write(
             '''{0:s}  self.{1:s} = {2:s}
 '''.format(indent, name, name)
@@ -149,6 +163,27 @@ while len(line):
 {5:s}  if isinstance({6:s}, str) else
 {7:s}    {8:s}
 {9:s}  )
+'''.format(
+              indent,
+              name,
+              indent,
+              subtype,
+              name,
+              indent,
+              name,
+              indent,
+              name,
+              indent
+            )
+          )
+        elif type[:4] == 'set(' and type[-1:] == ')':
+          subtype = type[4:-1]
+          sys.stdout.write(
+            '''{0:s}  self.{1:s} = (
+{2:s}    set([element.deserialize_{3:s}(i) for i in {4:s}.split()])
+{5:s}  if isinstance({6:s}, str) else
+{7:s}    {8:s}
+{9:s}  )
 '''.format(
               indent,
               name,
@@ -186,7 +221,7 @@ while len(line):
         sys.stdout.write(
           '''{0:s}def serialize(self, ref_list, indent = 0):
 {1:s}  {2:s}.serialize(self, ref_list, indent)
-'''.format(indent, indent, base_class)
+'''.format(indent, indent, full_base_class)
         )
         for type, name in fields:
           if type[:5] == 'list(' and type[-1:] == ')':
@@ -196,6 +231,24 @@ while len(line):
 {1:s}    '{2:s}',
 {3:s}    ' '.join([element.serialize_{4:s}(i{5:s}) for i in self.{6:s}])
 {7:s}  )
+'''.format(
+                indent,
+                indent,
+                name,
+                indent,
+                subtype,
+                ', ref_list' if subtype == 'ref' else '',
+                name,
+                indent
+              )
+            )
+          elif type[:4] == 'set(' and type[-1:] == ')':
+            subtype = type[4:-1]
+            sys.stdout.write(
+              '''{0:s}  self.set(
+{1:s}    '{2:s}',
+{3:s}    ' '.join([element.serialize_{4:s}(i{5:s}) for i in sorted(self.{6:s})])
+{7:s}  )
 '''.format(
                 indent,
                 indent,
@@ -221,7 +274,7 @@ while len(line):
         sys.stdout.write(
           '''{0:s}def deserialize(self, ref_list):
 {1:s}  {2:s}.deserialize(self, ref_list)
-'''.format(indent, indent, base_class)
+'''.format(indent, indent, full_base_class)
         )
         for type, name in fields:
           if type[:5] == 'list(' and type[-1:] == ')':
@@ -242,6 +295,28 @@ while len(line):
                 indent
               )
             )
+          elif type[:4] == 'set(' and type[-1:] == ')':
+            subtype = type[4:-1]
+            sys.stdout.write(
+              '''{0:s}  self.{1:s} = set(
+{2:s}    [
+{3:s}      element.deserialize_{4:s}(i{5:s})
+{6:s}      for i in self.get('{7:s}', '').split()
+{8:s}    ]
+{9:s}  )
+'''.format(
+                indent,
+                name,
+                indent,
+                indent,
+                subtype,
+                ', ref_list' if subtype == 'ref' else '',
+                indent,
+                name,
+                indent,
+                indent
+              )
+            )
           else: 
             sys.stdout.write(
               '''{0:s}  self.{1:s} = element.deserialize_{2:s}(self.get('{3:s}', '{4:s}'){5:s})
@@ -264,7 +339,7 @@ while len(line):
 '''.format(
           indent,
           indent,
-          base_class,
+          full_base_class,
           indent,
           indent,
           class_name,
@@ -289,7 +364,7 @@ while len(line):
 '''.format(
             indent,
             indent,
-            base_class
+            full_base_class
           )
         )
         for type, name in fields:
@@ -302,6 +377,27 @@ while len(line):
 {5:s}        ', '.join([repr(i) for i in self.{6:s}])
 {7:s}      )
 {8:s}    )
+'''.format(
+                indent,
+                name,
+                indent,
+                indent,
+                name,
+                indent,
+                name,
+                indent,
+                indent
+              )
+            )
+          elif type[:4] == 'set(' and type[-1:] == ')':
+            subtype = type[4:-1]
+            sys.stdout.write(
+              '''{0:s}  if len(self.{1:s}):
+{2:s}    params.append(
+{3:s}      '{4:s} = set([{{0:s}}])'.format(
+{5:s}        ', '.join([repr(i) for i in sorted(self.{6:s})])
+{7:s}      )
+{8:s}    )
 '''.format(
                 indent,
                 name,
@@ -343,7 +439,7 @@ while len(line):
           indent,
           indent,
           package_name,
-          full_name,
+          '.'.join([i for _, i, _, _ in stack]),
           indent
         )
       )

diff --git a/tests/Makefile b/tests/Makefile
index 33c6d57..323dce6 100644 (file)
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -6,7 +6,7 @@ cal: y.tab.c lex.yy.c
 
 lex.yy.c: cal.l
        ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
-       ../plex.py $<
+       ../bootstrap_plex.py $<.xml
        # add the patch for state machine diagnostic
        #cp $@ $@.orig
        #patch $@ <$@.patch
@@ -19,11 +19,11 @@ flex0: flex0.c
 
 flex0.c: flex0.l
        ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
-       ../plex.py -o $@ $<
+       ../bootstrap_plex.py -o $@ $<.xml
 
 flex1: flex1.c
        gcc -o $@ $< -ll
 
 flex1.c: flex1.l
        ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
-       ../plex.py -o $@ $<
+       ../bootstrap_plex.py -o $@ $<.xml