Add scanner and parser that can recognize and mark up the test inputs
authorNick Downing <nick@ndcode.org>
Sat, 26 Jan 2019 10:50:17 +0000 (21:50 +1100)
committerNick Downing <nick@ndcode.org>
Sat, 26 Jan 2019 10:50:17 +0000 (21:50 +1100)
.gitignore
Makefile [new file with mode: 0644]
ast.py [new file with mode: 0644]
ast.sh [new file with mode: 0755]
element.py [new file with mode: 0644]
generate_ast.py [new file with mode: 0755]
markup.py [new file with mode: 0755]
pitree.l [new file with mode: 0644]
pitree.y [new file with mode: 0644]

index b2bf8c8..168724a 100644 (file)
@@ -1,3 +1,5 @@
 __pycache__
+/lex_yy.py
 /out
 /tests
+/y_tab.py
diff --git a/Makefile b/Makefile
new file mode 100644 (file)
index 0000000..5637da2
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,7 @@
+all: lex_yy.py y_tab.py
+
+lex_yy.py: pitree.l
+       ../pilex.git/pilex.py --element --python $<
+
+y_tab.py: pitree.y
+       ../piyacc.git/piyacc.py --element --python $<
diff --git a/ast.py b/ast.py
new file mode 100644 (file)
index 0000000..69c0b7e
--- /dev/null
+++ b/ast.py
@@ -0,0 +1,843 @@
+import element
+
+class AST(element.Element):
+  # internal classes:
+  class Expression(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_Expression',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        Expression if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.Expression({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class Type(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_Type',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        Type if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.Type({0:s})'.format(', '.join(params))
+    # GENERATE END
+
+  # syntax classes:
+  class BaseClass(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_BaseClass',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        BaseClass if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.BaseClass({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class ClassBody(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_ClassBody',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        ClassBody if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.ClassBody({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class ClassName(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_ClassName',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        ClassName if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.ClassName({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class DefaultValue(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_DefaultValue',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        DefaultValue if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.DefaultValue({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class Identifier(element.Element):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_Identifier',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        Identifier if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.Identifier({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class LiteralBool(Expression):
+    # GENERATE ELEMENT(bool value) BEGIN
+    def __init__(
+      self,
+      tag = 'AST_LiteralBool',
+      attrib = {},
+      text = '',
+      children = [],
+      value = False
+    ):
+      AST.Expression.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+      self.value = (
+        element.deserialize_bool(value)
+      if isinstance(value, str) else
+        value
+      )
+    def serialize(self, ref_list):
+      AST.Expression.serialize(self, ref_list)
+      self.set('value', element.serialize_bool(self.value))
+    def deserialize(self, ref_list):
+      AST.Expression.deserialize(self, ref_list)
+      self.value = element.deserialize_bool(self.get('value', 'false'))
+    def copy(self, factory = None):
+      result = AST.Expression.copy(
+        self,
+        LiteralBool if factory is None else factory
+      )
+      result.value = self.value
+      return result
+    def repr_serialize(self, params):
+      AST.Expression.repr_serialize(self, params)
+      if self.value != False:
+        params.append(
+          'value = {0:s}'.format(repr(self.value))
+        )
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.LiteralBool({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class LiteralInt(Expression):
+    # GENERATE ELEMENT(str sign, int base, str digits) BEGIN
+    def __init__(
+      self,
+      tag = 'AST_LiteralInt',
+      attrib = {},
+      text = '',
+      children = [],
+      sign = '',
+      base = -1,
+      digits = ''
+    ):
+      AST.Expression.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+      self.sign = sign
+      self.base = (
+        element.deserialize_int(base)
+      if isinstance(base, str) else
+        base
+      )
+      self.digits = digits
+    def serialize(self, ref_list):
+      AST.Expression.serialize(self, ref_list)
+      self.set('sign', element.serialize_str(self.sign))
+      self.set('base', element.serialize_int(self.base))
+      self.set('digits', element.serialize_str(self.digits))
+    def deserialize(self, ref_list):
+      AST.Expression.deserialize(self, ref_list)
+      self.sign = element.deserialize_str(self.get('sign', ''))
+      self.base = element.deserialize_int(self.get('base', '-1'))
+      self.digits = element.deserialize_str(self.get('digits', ''))
+    def copy(self, factory = None):
+      result = AST.Expression.copy(
+        self,
+        LiteralInt if factory is None else factory
+      )
+      result.sign = self.sign
+      result.base = self.base
+      result.digits = self.digits
+      return result
+    def repr_serialize(self, params):
+      AST.Expression.repr_serialize(self, params)
+      if self.sign != '':
+        params.append(
+          'sign = {0:s}'.format(repr(self.sign))
+        )
+      if self.base != -1:
+        params.append(
+          'base = {0:s}'.format(repr(self.base))
+        )
+      if self.digits != '':
+        params.append(
+          'digits = {0:s}'.format(repr(self.digits))
+        )
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.LiteralInt({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class LiteralList(Expression):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_LiteralList',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Expression.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Expression.copy(
+        self,
+        LiteralList if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.LiteralList({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class LiteralRef(Expression):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_LiteralRef',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Expression.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Expression.copy(
+        self,
+        LiteralRef if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.LiteralRef({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class LiteralSet(Expression):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_LiteralSet',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Expression.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Expression.copy(
+        self,
+        LiteralSet if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.LiteralSet({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class LiteralStr(Expression):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_LiteralStr',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Expression.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Expression.copy(
+        self,
+        LiteralStr if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.LiteralStr({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class Text(element.Element):
+    class Escape(element.Element):
+      # GENERATE ELEMENT(int value) BEGIN
+      def __init__(
+        self,
+        tag = 'AST_Text_Escape',
+        attrib = {},
+        text = '',
+        children = [],
+        value = -1
+      ):
+        element.Element.__init__(
+          self,
+          tag,
+          attrib,
+          text,
+          children
+        )
+        self.value = (
+          element.deserialize_int(value)
+        if isinstance(value, str) else
+          value
+        )
+      def serialize(self, ref_list):
+        element.Element.serialize(self, ref_list)
+        self.set('value', element.serialize_int(self.value))
+      def deserialize(self, ref_list):
+        element.Element.deserialize(self, ref_list)
+        self.value = element.deserialize_int(self.get('value', '-1'))
+      def copy(self, factory = None):
+        result = element.Element.copy(
+          self,
+          Escape if factory is None else factory
+        )
+        result.value = self.value
+        return result
+      def repr_serialize(self, params):
+        element.Element.repr_serialize(self, params)
+        if self.value != -1:
+          params.append(
+            'value = {0:s}'.format(repr(self.value))
+          )
+      def __repr__(self):
+        params = []
+        self.repr_serialize(params)
+        return 'ast.AST.Text.Escape({0:s})'.format(', '.join(params))
+      # GENERATE END
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_Text',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        Text if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.Text({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class TypeBool(Type):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_TypeBool',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Type.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Type.copy(
+        self,
+        TypeBool if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.TypeBool({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class TypeInt(Type):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_TypeInt',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Type.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Type.copy(
+        self,
+        TypeInt if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.TypeInt({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class TypeList(Type):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_TypeList',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Type.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Type.copy(
+        self,
+        TypeList if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.TypeList({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class TypeRef(Type):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_TypeRef',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Type.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Type.copy(
+        self,
+        TypeRef if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.TypeRef({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class TypeSet(Type):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_TypeSet',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Type.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Type.copy(
+        self,
+        TypeSet if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.TypeSet({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class TypeStr(Type):
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_TypeStr',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      AST.Type.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = AST.Type.copy(
+        self,
+        TypeStr if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.TypeStr({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class Section1(element.Element):
+    class CodeBlock(element.Element):
+      # GENERATE ELEMENT() BEGIN
+      def __init__(
+        self,
+        tag = 'AST_Section1_CodeBlock',
+        attrib = {},
+        text = '',
+        children = []
+      ):
+        element.Element.__init__(
+          self,
+          tag,
+          attrib,
+          text,
+          children
+        )
+      def copy(self, factory = None):
+        result = element.Element.copy(
+          self,
+          CodeBlock if factory is None else factory
+        )
+        return result
+      def __repr__(self):
+        params = []
+        self.repr_serialize(params)
+        return 'ast.AST.Section1.CodeBlock({0:s})'.format(', '.join(params))
+      # GENERATE END
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_Section1',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        Section1 if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.Section1({0:s})'.format(', '.join(params))
+    # GENERATE END
+  class Section2(element.Element):
+    class ClassDef(element.Element):
+      # GENERATE ELEMENT() BEGIN
+      def __init__(
+        self,
+        tag = 'AST_Section2_ClassDef',
+        attrib = {},
+        text = '',
+        children = []
+      ):
+        element.Element.__init__(
+          self,
+          tag,
+          attrib,
+          text,
+          children
+        )
+      def copy(self, factory = None):
+        result = element.Element.copy(
+          self,
+          ClassDef if factory is None else factory
+        )
+        return result
+      def __repr__(self):
+        params = []
+        self.repr_serialize(params)
+        return 'ast.AST.Section2.ClassDef({0:s})'.format(', '.join(params))
+      # GENERATE END
+    class FieldDef(element.Element):
+      # GENERATE ELEMENT() BEGIN
+      def __init__(
+        self,
+        tag = 'AST_Section2_FieldDef',
+        attrib = {},
+        text = '',
+        children = []
+      ):
+        element.Element.__init__(
+          self,
+          tag,
+          attrib,
+          text,
+          children
+        )
+      def copy(self, factory = None):
+        result = element.Element.copy(
+          self,
+          FieldDef if factory is None else factory
+        )
+        return result
+      def __repr__(self):
+        params = []
+        self.repr_serialize(params)
+        return 'ast.AST.Section2.FieldDef({0:s})'.format(', '.join(params))
+      # GENERATE END
+    # GENERATE ELEMENT() BEGIN
+    def __init__(
+      self,
+      tag = 'AST_Section2',
+      attrib = {},
+      text = '',
+      children = []
+    ):
+      element.Element.__init__(
+        self,
+        tag,
+        attrib,
+        text,
+        children
+      )
+    def copy(self, factory = None):
+      result = element.Element.copy(
+        self,
+        Section2 if factory is None else factory
+      )
+      return result
+    def __repr__(self):
+      params = []
+      self.repr_serialize(params)
+      return 'ast.AST.Section2({0:s})'.format(', '.join(params))
+    # GENERATE END
+  # GENERATE ELEMENT() BEGIN
+  def __init__(
+    self,
+    tag = 'AST',
+    attrib = {},
+    text = '',
+    children = []
+  ):
+    element.Element.__init__(
+      self,
+      tag,
+      attrib,
+      text,
+      children
+    )
+  def copy(self, factory = None):
+    result = element.Element.copy(
+      self,
+      AST if factory is None else factory
+    )
+    return result
+  def __repr__(self):
+    params = []
+    self.repr_serialize(params)
+    return 'ast.AST({0:s})'.format(', '.join(params))
+  # GENERATE END
+
+# GENERATE FACTORY(element.Element) BEGIN
+tag_to_class = {
+  'AST': AST,
+  'AST_Expression': AST.Expression,
+  'AST_Type': AST.Type,
+  'AST_BaseClass': AST.BaseClass,
+  'AST_ClassBody': AST.ClassBody,
+  'AST_ClassName': AST.ClassName,
+  'AST_DefaultValue': AST.DefaultValue,
+  'AST_Identifier': AST.Identifier,
+  'AST_LiteralBool': AST.LiteralBool,
+  'AST_LiteralInt': AST.LiteralInt,
+  'AST_LiteralList': AST.LiteralList,
+  'AST_LiteralRef': AST.LiteralRef,
+  'AST_LiteralSet': AST.LiteralSet,
+  'AST_LiteralStr': AST.LiteralStr,
+  'AST_Text': AST.Text,
+  'AST_Text_Escape': AST.Text.Escape,
+  'AST_TypeBool': AST.TypeBool,
+  'AST_TypeInt': AST.TypeInt,
+  'AST_TypeList': AST.TypeList,
+  'AST_TypeRef': AST.TypeRef,
+  'AST_TypeSet': AST.TypeSet,
+  'AST_TypeStr': AST.TypeStr,
+  'AST_Section1': AST.Section1,
+  'AST_Section1_CodeBlock': AST.Section1.CodeBlock,
+  'AST_Section2': AST.Section2,
+  'AST_Section2_ClassDef': AST.Section2.ClassDef,
+  'AST_Section2_FieldDef': AST.Section2.FieldDef
+}
+def factory(tag, attrib = {}, *args, **kwargs):
+  return tag_to_class.get(tag, element.Element)(tag, attrib, *args, **kwargs)
+# GENERATE END
diff --git a/ast.sh b/ast.sh
new file mode 100755 (executable)
index 0000000..2ba2388
--- /dev/null
+++ b/ast.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+if ./generate_ast.py ast <ast.py >ast.py.new && ! diff -q ast.py ast.py.new
+then
+  mv ast.py.new ast.py
+else
+  rm -f ast.py.new
+fi
diff --git a/element.py b/element.py
new file mode 100644 (file)
index 0000000..2d02217
--- /dev/null
@@ -0,0 +1,179 @@
+# Copyright (C) 2018 Nick Downing <nick@ndcode.org>
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+import xml.etree.ElementTree
+
+class Element(xml.etree.ElementTree._Element_Py):
+  def __init__(self, tag = 'Element', attrib = {}, text = '', children = []):
+    xml.etree.ElementTree._Element_Py.__init__(self, tag, attrib)
+    self.ref = -1
+    self.seen = False
+    set_text(self, 0, text)
+    self[:] = children
+  def serialize(self, ref_list):
+    for i in self:
+      # parented, enforce that child can only be parented at most once
+      # (although there can be unlimited numbers of numeric refs to it)
+      assert not i.seen
+      i.seen = True
+      if i.ref == -1:
+        i.serialize(ref_list)
+  def deserialize(self, ref_list):
+    for i in self:
+      i.deserialize(ref_list)
+  def copy(self, factory = None):
+    result = (Element if factory is None else factory)(self.tag, self.attrib)
+    result.text = self.text
+    result.tail = self.tail
+    result[:] = [i.copy() for i in self]
+    return result
+  def repr_serialize(self, params):
+    if len(self):
+      params.append(
+        'children = [{0:s}]'.format(
+          ', '.join([repr(i) for i in self])
+        )
+      )
+  def __repr__(self):
+    params = []
+    self.repr_serialize(params)
+    return 'element.Element({0:s})'.format(', '.join(params))
+
+bool_to_str = ['false', 'true']
+def serialize_bool(value):
+  return bool_to_str[int(value)]
+
+str_to_bool = {'false': False, 'true': True}
+def deserialize_bool(text):
+  return str_to_bool[text]
+
+def serialize_int(value):
+  return str(value)
+
+def deserialize_int(text):
+  return int(text)
+
+def serialize_ref(value, ref_list):
+  if value is None:
+    ref = -1
+  else:
+    ref = value.ref
+    if ref == -1:
+      ref = len(ref_list)
+      ref_list.append(value)
+      value.ref = ref
+      value.set('ref', str(ref))
+      # this doesn't set the seen flag, so it will be parented by the
+      # root, unless it is already parented or gets parented later on
+      if not value.seen:
+        value.serialize(ref_list)
+  return str(ref)
+
+def deserialize_ref(text, ref_list):
+  ref = int(text)
+  return None if ref < 0 else ref_list[ref]
+
+def serialize_str(value):
+  return value
+
+def deserialize_str(text):
+  return text
+
+def serialize(value, fout, encoding = 'unicode'):
+  ref_list = []
+  serialize_ref(value, ref_list)
+  parents = [i for i in ref_list if not i.seen]
+  root = Element('root', children = parents)
+  for i in range(len(root)):
+    set_text(root, i, '\n  ')
+  set_text(root, len(root), '\n')
+  root.tail = '\n'
+  xml.etree.ElementTree.ElementTree(root).write(fout, encoding)
+  for i in root:
+    i.tail = None
+  for i in ref_list:
+    i.ref = -1
+    del i.attrib['ref']
+  i = 0
+  while i < len(parents):
+    for j in parents[i]:
+      j.seen = False
+      parents.append(j)
+    i += 1
+
+def deserialize(fin, factory = Element, encoding = 'unicode'):
+  root = xml.etree.ElementTree.parse(
+    fin,
+    xml.etree.ElementTree.XMLParser(
+      target = xml.etree.ElementTree.TreeBuilder(factory),
+      encoding = encoding
+    )
+  ).getroot()
+  assert root.tag == 'root'
+  for i in root:
+    i.tail = None
+  i = 0
+  parents = root[:]
+  ref_list = []
+  while i < len(parents):
+    j = parents[i]
+    if 'ref' in j.attrib:
+      ref = int(j.attrib['ref'])
+      del j.attrib['ref']
+      if len(ref_list) < ref + 1:
+        ref_list.extend([None] * (ref + 1 - len(ref_list)))
+      ref_list[ref] = j
+    parents.extend(j[:])
+    i += 1
+  for i in root:
+    i.deserialize(ref_list)
+  return ref_list[0]
+
+# compatibility scheme to access arbitrary xml.etree.ElementTree.Element-like
+# objects (not just Element defined above) using a more consistent interface:
+def get_text(root, i):
+  if i < 0:
+    i += len(root) + 1
+  text = root.text if i == 0 else root[i - 1].tail
+  return '' if text is None else text
+
+def set_text(root, i, text):
+  if i < 0:
+    i += len(root) + 1
+  if len(text) == 0:
+    text = None
+  if i == 0:
+    root.text = text
+  else:
+    root[i - 1].tail = text
+
+def to_text(root):
+  return ''.join(
+    [
+      j
+      for i in range(len(root))
+      for j in [get_text(root, i), to_text(root[i])]
+    ] +
+    [get_text(root, len(root))]
+  )
+
+def concatenate(children, factory = Element, *args, **kwargs):
+  root = factory(*args, **kwargs)
+  for child in children:
+    i = len(root)
+    set_text(root, i, get_text(root, i) + get_text(child, 0))
+    root[i:] = child[:]
+  return root
diff --git a/generate_ast.py b/generate_ast.py
new file mode 100755 (executable)
index 0000000..f9a633b
--- /dev/null
@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+
+if len(sys.argv) >= 2:
+  package_name = '{0:s}.'.format(sys.argv[1])
+else:
+  package_name = ''
+
+default_value = {
+  'bool': 'False',
+  'int': '-1',
+  'ref': 'None',
+  'str': '\'\'',
+  'list(bool)': '[]',
+  'list(int)': '[]',
+  'list(ref)': '[]',
+  'list(str)': '[]',
+  'set(bool)': 'set()',
+  'set(int)': 'set()',
+  'set(ref)': 'set()',
+  'set(str)': 'set()'
+}
+default_value_str = {
+  'bool': 'false',
+  'int': '-1',
+  'ref': '-1',
+  'str': ''
+}
+
+re_class = re.compile(
+  '([\t ]*)class ([A-Za-z_][A-Za-z0-9_]*)\(([A-Za-z_][A-Za-z0-9_.]*)'
+)
+re_element = re.compile(
+  '([\t ]*)# GENERATE ELEMENT\((([^()]|\([^()]*\))*)\)( BEGIN)?'
+)
+re_factory = re.compile(
+  '([\t ]*)# GENERATE FACTORY\(([^()]*)\)( BEGIN)?'
+)
+stack = []
+classes = []
+base_classes = [{'element.Element': []}] # params
+
+line = sys.stdin.readline()
+while len(line):
+  match = re_class.match(line)
+  if match is not None:
+    sys.stdout.write(line)
+    indent = match.group(1)
+    class_name = match.group(2)
+    base_class = match.group(3)
+    while len(stack) and stack[-1][0][:len(indent)] == indent:
+      _, temp_class_name, _, _ = stack.pop()
+      for temp_base_class, temp_fields in base_classes.pop().items():
+        base_classes[-1][
+          '{0:s}.{1:s}'.format(temp_class_name, temp_base_class)
+        ] = temp_fields
+    for i in range(len(base_classes) - 1, -1, -1):
+      if base_class in base_classes[i]:
+        classes.append(
+          '.'.join([j for _, j, _, _ in stack] + [class_name])
+        )
+        full_base_class = '.'.join(
+          [j for _, j, _, _ in stack[:i]] + [base_class]
+        )
+        base_classes[-1][class_name] = list(base_classes[i][base_class])
+        break
+    else:
+      full_base_class = base_class
+    stack.append((indent, class_name, base_class, full_base_class))
+    base_classes.append({})
+  else:
+    match = re_element.match(line)
+    if match is not None:
+      indent = match.group(1)
+      params = match.group(2)
+      begin = match.group(4)
+
+      while len(stack) and stack[-1][0][:len(indent)] == indent:
+        _, temp_class_name, _, _ = stack.pop()
+        for temp_base_class, temp_fields in base_classes.pop().items():
+          base_classes[-1][
+            '{0:s}.{1:s}'.format(temp_class_name, temp_base_class)
+          ] = temp_fields
+      _, class_name, base_class, full_base_class = stack[-1]
+      fields = params.split(',')
+      if fields[-1] == '':
+        del fields[-1:]
+      fields = [i.split() for i in fields]
+      fields = [(type, name) for [type, name] in fields]
+      i = len(base_classes[-2][class_name])
+      base_classes[-2][class_name].extend(fields)
+
+      sys.stdout.write(
+        '''{0:s}# GENERATE ELEMENT({1:s}) BEGIN
+{2:s}def __init__(
+{3:s}  self,
+{4:s}  tag = '{5:s}',
+{6:s}  attrib = {{}},
+{7:s}  text = '',
+{8:s}  children = []{9:s}
+{10:s}):
+{11:s}  {12:s}.__init__(
+{13:s}    self,
+{14:s}    tag,
+{15:s}    attrib,
+{16:s}    text,
+{17:s}    children{18:s}
+{19:s}  )
+'''.format(
+          indent,
+          params,
+          indent,
+          indent,
+          indent,
+          '_'.join([i for _, i, _, _ in stack]),
+          indent,
+          indent,
+          indent,
+          ''.join(
+            [
+              ',\n{0:s}  {1:s} = {2:s}'.format(
+                indent,
+                name,
+                default_value[type]
+              )
+              for type, name in base_classes[-2][class_name]
+            ]
+          ),
+          indent,
+          indent,
+          full_base_class,
+          indent,
+          indent,
+          indent,
+          indent,
+          indent,
+          ''.join(
+            [
+              ',\n{0:s}    {1:s}'.format(
+                indent,
+                name
+              )
+              for type, name in base_classes[-2][class_name][:i]
+            ]
+          ),
+          indent
+        )
+      )
+      for type, name in fields:
+        if type == 'ref' or type == 'list(ref)' or type == 'set(ref)' or type == 'str':
+          sys.stdout.write(
+            '''{0:s}  self.{1:s} = {2:s}
+'''.format(indent, name, name)
+          )
+        elif type[:5] == 'list(' and type[-1:] == ')':
+          subtype = type[5:-1]
+          sys.stdout.write(
+            '''{0:s}  self.{1:s} = (
+{2:s}    [element.deserialize_{3:s}(i) for i in {4:s}.split()]
+{5:s}  if isinstance({6:s}, str) else
+{7:s}    {8:s}
+{9:s}  )
+'''.format(
+              indent,
+              name,
+              indent,
+              subtype,
+              name,
+              indent,
+              name,
+              indent,
+              name,
+              indent
+            )
+          )
+        elif type[:4] == 'set(' and type[-1:] == ')':
+          subtype = type[4:-1]
+          sys.stdout.write(
+            '''{0:s}  self.{1:s} = (
+{2:s}    set([element.deserialize_{3:s}(i) for i in {4:s}.split()])
+{5:s}  if isinstance({6:s}, str) else
+{7:s}    {8:s}
+{9:s}  )
+'''.format(
+              indent,
+              name,
+              indent,
+              subtype,
+              name,
+              indent,
+              name,
+              indent,
+              name,
+              indent
+            )
+          )
+        else:
+          sys.stdout.write(
+            '''{0:s}  self.{1:s} = (
+{2:s}    element.deserialize_{3:s}({4:s})
+{5:s}  if isinstance({6:s}, str) else
+{7:s}    {8:s}
+{9:s}  )
+'''.format(
+              indent,
+              name,
+              indent,
+              type,
+              name,
+              indent,
+              name,
+              indent,
+              name,
+              indent
+            )
+          )
+      if len(fields):
+        sys.stdout.write(
+          '''{0:s}def serialize(self, ref_list):
+{1:s}  {2:s}.serialize(self, ref_list)
+'''.format(indent, indent, full_base_class)
+        )
+        for type, name in fields:
+          if type[:5] == 'list(' and type[-1:] == ')':
+            subtype = type[5:-1]
+            sys.stdout.write(
+              '''{0:s}  self.set(
+{1:s}    '{2:s}',
+{3:s}    ' '.join([element.serialize_{4:s}(i{5:s}) for i in self.{6:s}])
+{7:s}  )
+'''.format(
+                indent,
+                indent,
+                name,
+                indent,
+                subtype,
+                ', ref_list' if subtype == 'ref' else '',
+                name,
+                indent
+              )
+            )
+          elif type[:4] == 'set(' and type[-1:] == ')':
+            subtype = type[4:-1]
+            sys.stdout.write(
+              '''{0:s}  self.set(
+{1:s}    '{2:s}',
+{3:s}    ' '.join([element.serialize_{4:s}(i{5:s}) for i in sorted(self.{6:s})])
+{7:s}  )
+'''.format(
+                indent,
+                indent,
+                name,
+                indent,
+                subtype,
+                ', ref_list' if subtype == 'ref' else '',
+                name,
+                indent
+              )
+            )
+          else:
+            sys.stdout.write(
+              '''{0:s}  self.set('{1:s}', element.serialize_{2:s}(self.{3:s}{4:s}))
+'''.format(
+                indent,
+                name,
+                type,
+                name,
+                ', ref_list' if type == 'ref' else ''
+              )
+            )
+        sys.stdout.write(
+          '''{0:s}def deserialize(self, ref_list):
+{1:s}  {2:s}.deserialize(self, ref_list)
+'''.format(indent, indent, full_base_class)
+        )
+        for type, name in fields:
+          if type[:5] == 'list(' and type[-1:] == ')':
+            subtype = type[5:-1]
+            sys.stdout.write(
+              '''{0:s}  self.{1:s} = [
+{2:s}    element.deserialize_{3:s}(i{4:s})
+{5:s}    for i in self.get('{6:s}', '').split()
+{7:s}  ]
+'''.format(
+                indent,
+                name,
+                indent,
+                subtype,
+                ', ref_list' if subtype == 'ref' else '',
+                indent,
+                name,
+                indent
+              )
+            )
+          elif type[:4] == 'set(' and type[-1:] == ')':
+            subtype = type[4:-1]
+            sys.stdout.write(
+              '''{0:s}  self.{1:s} = set(
+{2:s}    [
+{3:s}      element.deserialize_{4:s}(i{5:s})
+{6:s}      for i in self.get('{7:s}', '').split()
+{8:s}    ]
+{9:s}  )
+'''.format(
+                indent,
+                name,
+                indent,
+                indent,
+                subtype,
+                ', ref_list' if subtype == 'ref' else '',
+                indent,
+                name,
+                indent,
+                indent
+              )
+            )
+          else: 
+            sys.stdout.write(
+              '''{0:s}  self.{1:s} = element.deserialize_{2:s}(self.get('{3:s}', '{4:s}'){5:s})
+'''.format(
+                indent,
+                name,
+                type,
+                name,
+                default_value_str[type],
+                ', ref_list' if type == 'ref' else ''
+              )
+            )
+      sys.stdout.write(
+        '''{0:s}def copy(self, factory = None):
+{1:s}  result = {2:s}.copy(
+{3:s}    self,
+{4:s}    {5:s} if factory is None else factory
+{6:s}  ){7:s}
+{8:s}  return result
+'''.format(
+          indent,
+          indent,
+          full_base_class,
+          indent,
+          indent,
+          class_name,
+          indent,
+          ''.join(
+            [
+              '\n{0:s}  result.{1:s} = self.{2:s}'.format(
+                indent,
+                name,
+                name
+              )
+              for _, name in fields
+            ]
+          ),
+          indent
+        )
+      )
+      if len(fields):
+        sys.stdout.write(
+          '''{0:s}def repr_serialize(self, params):
+{1:s}  {2:s}.repr_serialize(self, params)
+'''.format(
+            indent,
+            indent,
+            full_base_class
+          )
+        )
+        for type, name in fields:
+          if type[:5] == 'list(' and type[-1:] == ')':
+            subtype = type[5:-1]
+            sys.stdout.write(
+              '''{0:s}  if len(self.{1:s}):
+{2:s}    params.append(
+{3:s}      '{4:s} = [{{0:s}}]'.format(
+{5:s}        ', '.join([repr(i) for i in self.{6:s}])
+{7:s}      )
+{8:s}    )
+'''.format(
+                indent,
+                name,
+                indent,
+                indent,
+                name,
+                indent,
+                name,
+                indent,
+                indent
+              )
+            )
+          elif type[:4] == 'set(' and type[-1:] == ')':
+            subtype = type[4:-1]
+            sys.stdout.write(
+              '''{0:s}  if len(self.{1:s}):
+{2:s}    params.append(
+{3:s}      '{4:s} = set([{{0:s}}])'.format(
+{5:s}        ', '.join([repr(i) for i in sorted(self.{6:s})])
+{7:s}      )
+{8:s}    )
+'''.format(
+                indent,
+                name,
+                indent,
+                indent,
+                name,
+                indent,
+                name,
+                indent,
+                indent
+              )
+            )
+          else:
+            sys.stdout.write(
+              '''{0:s}  if self.{1:s} != {2:s}:
+{3:s}    params.append(
+{4:s}      '{5:s} = {{0:s}}'.format(repr(self.{6:s}))
+{7:s}    )
+'''.format(
+                indent,
+                name,
+                default_value[type],
+                indent,
+                indent,
+                name,
+                name,
+                indent
+              )
+            )
+      sys.stdout.write(
+        '''{0:s}def __repr__(self):
+{1:s}  params = []
+{2:s}  self.repr_serialize(params)
+{3:s}  return '{4:s}{5:s}({{0:s}})'.format(', '.join(params))
+{6:s}# GENERATE END
+'''.format(
+          indent,
+          indent,
+          indent,
+          indent,
+          package_name,
+          '.'.join([i for _, i, _, _ in stack]),
+          indent
+        )
+      )
+      if begin is not None:
+        line = sys.stdin.readline()
+        while len(line):
+          if line.strip() == '# GENERATE END':
+            break
+          line = sys.stdin.readline()
+        else:
+          assert False
+    else:
+      match = re_factory.match(line)
+      if match is not None:
+        indent = match.group(1)
+        param = match.group(2)
+        begin = match.group(3)
+
+        sys.stdout.write(
+          '''{0:s}# GENERATE FACTORY({1:s}) BEGIN
+{2:s}tag_to_class = {{{3:s}
+{4:s}}}
+{5:s}def factory(tag, attrib = {{}}, *args, **kwargs):
+{6:s}  return tag_to_class.get(tag, {7:s})(tag, attrib, *args, **kwargs)
+{8:s}# GENERATE END
+'''.format(
+            indent,
+            param,
+            indent,
+            ','.join(
+              [
+                '\n{0:s}  \'{1:s}\': {2:s}'.format(
+                  indent,
+                  i.replace('.', '_'),
+                  i
+                )
+                for i in classes
+              ]
+            ),
+            indent,
+            indent,
+            indent,
+            param,
+            indent
+          )
+        )
+
+        if begin is not None:
+          line = sys.stdin.readline()
+          while len(line):
+            if line.strip() == '# GENERATE END':
+              break
+            line = sys.stdin.readline()
+          else:
+            assert False
+      else:
+        sys.stdout.write(line)
+  line = sys.stdin.readline()
diff --git a/markup.py b/markup.py
new file mode 100755 (executable)
index 0000000..caf5a41
--- /dev/null
+++ b/markup.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import ast
+import element
+import y_tab
+import sys
+
+element.serialize(y_tab.yyparse(ast.AST), sys.stdout)
diff --git a/pitree.l b/pitree.l
new file mode 100644 (file)
index 0000000..2cb01fc
--- /dev/null
+++ b/pitree.l
@@ -0,0 +1,113 @@
+%{
+  import ast
+  import y_tab
+%}
+
+%option nodefault
+
+%x CODEBLOCK SECTION2 COMMENT SINGLE_QUOTED SECTION3
+
+%%
+
+<INITIAL>{
+  [[:blank:]]*"%{"[[:blank:]]*\n? {
+    BEGIN(CODEBLOCK)
+    return y_tab.CODEBLOCK_START
+  }
+  [[:blank:]]*"%%"[[:blank:]]*\n? {
+    BEGIN(SECTION2)
+    return y_tab.SECTION2_START
+  }
+  [[:blank:]]*\n?
+}
+
+<CODEBLOCK>{
+  [[:blank:]]*"%}"[[:blank:]]*\n? {
+    BEGIN(INITIAL)
+    return y_tab.CODEBLOCK_END
+  }
+  .*\n?
+}
+
+<SECTION2>{
+  "."                          return ord('.')
+  "="                          return ord('=')
+  ":"                          return ord(':')
+  ";"                          return ord(';')
+  "("                          return ord('(')
+  ")"                          return ord(')')
+  "["                          return ord('[')
+  "]"                          return ord(']')
+  "{"                          return ord('{')
+  "}"                          return ord('}')
+  "bool"                       return y_tab.KEYWORD_BOOL
+  "class"                      return y_tab.KEYWORD_CLASS
+  "int"                                return y_tab.KEYWORD_INT
+  "list"                       return y_tab.KEYWORD_LIST
+  "ref"                                return y_tab.KEYWORD_REF
+  "set"                                return y_tab.KEYWORD_SET
+  "str"                                return y_tab.KEYWORD_STR
+  (?E{
+    ast.AST.LiteralBool,
+    value = False
+  }"False")                    return y_tab.LITERAL_BOOL
+  (?E{
+    ast.AST.LiteralBool,
+    value = False
+  }"False")                    return y_tab.LITERAL_BOOL
+  (?E{
+    ast.AST.LiteralRef
+  }"None")                     return y_tab.LITERAL_REF
+  (?E{
+    ast.AST.Identifier
+  }[A-Za-z_][A-Za-z0-9_]*)     return y_tab.IDENTIFIER
+  (?E{
+    ast.AST.LiteralInt,
+    sign = yy_groups[2],
+    base = 8,
+    digits = yy_groups[3]
+  }(-?)0([0-7]+))              return y_tab.LITERAL_INT
+  (?E{
+    ast.AST.LiteralInt,
+    sign = yy_groups[2],
+    base = 16,
+    digits = yy_groups[3]
+  }(-?)0x([0-9A-Fa-f]+))       return y_tab.LITERAL_INT
+  (?E{
+    ast.AST.LiteralInt,
+    sign = yy_groups[2],
+    base = 10,
+    digits = yy_groups[3]
+  }(-?)([0-9]+))               return y_tab.LITERAL_INT
+  "'" {
+    BEGIN(SINGLE_QUOTED)
+    return ord('\'')
+  }
+  ^[[:blank:]]*"%%"[[:blank:]]*\n? {
+    BEGIN(SECTION3)
+    return y_tab.SECTION3_START
+  }
+  "/*"                         BEGIN(COMMENT)
+  [[:blank:]]+
+  [[:space:]]
+}
+
+<COMMENT>{
+  "*/"                         BEGIN(SECTION2)
+  .|\n
+}
+
+<SINGLE_QUOTED>{
+  (?E{ast.AST.Text.Escape, value = ord('\'')}"\\'")
+  (?E{ast.AST.Text.Escape, value = ord('\\')}"\\\\")
+  (?E{ast.AST.Text.Escape, value = ord('\n')}"\\n")
+  "'" {
+    BEGIN(SECTION2)
+    return ord('\'')
+  }
+  .|\n
+}
+
+<SECTION3>{
+  .*\n?
+}
diff --git a/pitree.y b/pitree.y
new file mode 100644 (file)
index 0000000..bbd55d7
--- /dev/null
+++ b/pitree.y
@@ -0,0 +1,84 @@
+%{
+  import ast
+%}
+
+%token CODEBLOCK_START CODEBLOCK_END SECTION2_START SECTION3_START
+%token KEYWORD_BOOL KEYWORD_CLASS KEYWORD_INT KEYWORD_LIST KEYWORD_REF
+%token KEYWORD_SET KEYWORD_STR IDENTIFIER LITERAL_BOOL LITERAL_INT LITERAL_REF
+
+%start pitree
+
+%%
+
+pitree
+  : %space (?E{ast.AST.Section1}section1) SECTION2_START %space (?E{ast.AST.Section2}section2) section3_opt
+
+section1
+  :
+  | section1 (?E{ast.AST.Section1.CodeBlock}CODEBLOCK_START (?E{ast.AST.Text}%space) CODEBLOCK_END)
+  ;
+
+section2
+  :
+  | section2 %space (?E{ast.AST.Section2.ClassDef}KEYWORD_CLASS IDENTIFIER base_class_opt class_body_opt ';')
+  | section2 %space (?E{ast.AST.Section2.FieldDef}type IDENTIFIER default_value_opt ';')
+  ;
+
+base_class_opt
+  : %space (?E{ast.AST.BaseClass})
+  | %space (?E{ast.AST.BaseClass}':' %space (?E{ast.AST.ClassName}class_name) )
+  ;
+
+class_name
+  : IDENTIFIER
+  | class_name '.' IDENTIFIER
+  ;
+
+class_body_opt
+  : %space (?E{ast.AST.ClassBody})
+  | %space (?E{ast.AST.ClassBody}'{' section2 '}')
+  ;
+
+type
+  : %space (?E{ast.AST.TypeBool}KEYWORD_BOOL)
+  | %space (?E{ast.AST.TypeInt}KEYWORD_INT)
+  | %space (?E{ast.AST.TypeList}KEYWORD_LIST '(' type ')')
+  | %space (?E{ast.AST.TypeRef}KEYWORD_REF)
+  | %space (?E{ast.AST.TypeSet}KEYWORD_SET '(' type ')')
+  | %space (?E{ast.AST.TypeInt}KEYWORD_STR)
+  ;
+default_value_opt
+  : (?E{ast.AST.DefaultValue})
+  | (?E{ast.AST.DefaultValue}'=' expression)
+  ;
+
+expression
+  : LITERAL_BOOL
+  | LITERAL_INT
+  | %space (?E{ast.AST.LiteralList}'[' expression_list_opt ']')
+  | LITERAL_REF
+  | %space (?E{ast.AST.LiteralSet}KEYWORD_SET '(' set_initializer_opt ')')
+  | %space (?E{ast.AST.LiteralStr}'\'' (?E{ast.AST.Text}%space) '\'')
+  ;
+
+expression_list_opt
+  :
+  | expression_list
+  | expression_list ','
+  ;
+
+expression_list
+  : expression
+  | expression_list ',' expression
+  ;
+
+set_initializer_opt
+  :
+  | '[' expression_list_opt ']'
+  ;
+
+section3_opt
+  :
+  | SECTION3_START (?E{ast.AST.Text}%space)
+  ;