From 568995e92b78c7f3e2f45472bca5e0d22df6335b Mon Sep 17 00:00:00 2001 From: Nick Downing Date: Tue, 29 Jan 2019 13:00:31 +1100 Subject: [PATCH] Update to pitree.git commit 5cba525 --- .gitignore | 27 +++---- Makefile | 7 +- element.py | 179 ---------------------------------------------- l_to_python.py | 83 +++++++++++---------- scan-code_to_l.py | 20 +++++- scan-gram_to_l.py | 36 ++++++++-- scan_to_l.py | 36 ++++++++-- xml_to_y.py | 20 +++++- y_to_python.py | 95 ++++++++++++++---------- 9 files changed, 220 insertions(+), 283 deletions(-) delete mode 100644 element.py diff --git a/.gitignore b/.gitignore index b525ad6..9c58774 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,15 @@ __pycache__ -a.c -a.i -a.xml -b.xml -lex_yy.py -t_def.py -tests/*.l.xml -tests/*.l.new.xml -tests/*.l.new -tests/*.y.xml -tests/*.y.new.xml -tests/*.y.new -y_tab.py +/a.c +/a.i +/a.xml +/b.xml +/element.py +/lex_yy.py +/t_def.py +/tests/*.l.xml +/tests/*.l.new.xml +/tests/*.l.new +/tests/*.y.xml +/tests/*.y.new.xml +/tests/*.y.new +/y_tab.py diff --git a/Makefile b/Makefile index fe4da19..e35fcf9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ -all: lex_yy.py t_def.py y_tab.py +all: element.py lex_yy.py t_def.py y_tab.py + +element.py: ../pitree.git/skel/element.py + cat $< >$@ lex_yy.py: ansi_c.l ../pilex.git/pilex.py --element --python $< @@ -10,4 +13,4 @@ y_tab.py: ansi_c.y ../piyacc.git/piyacc.py --element --python $< clean: - rm -f lex_yy.py t_def.py y_tab.py + rm -f element.py lex_yy.py t_def.py y_tab.py diff --git a/element.py b/element.py deleted file mode 100644 index 2d02217..0000000 --- a/element.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (C) 2018 Nick Downing -# SPDX-License-Identifier: GPL-2.0-only -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -# details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 51 -# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -import xml.etree.ElementTree - -class Element(xml.etree.ElementTree._Element_Py): - def __init__(self, tag = 'Element', attrib = {}, text = '', children = []): - xml.etree.ElementTree._Element_Py.__init__(self, tag, attrib) - self.ref = -1 - self.seen = False - set_text(self, 0, text) - self[:] = children - def serialize(self, ref_list): - for i in self: - # parented, enforce that child can only be parented at most once - # (although there can be unlimited numbers of numeric refs to it) - assert not i.seen - i.seen = True - if i.ref == -1: - i.serialize(ref_list) - def deserialize(self, ref_list): - for i in self: - i.deserialize(ref_list) - def copy(self, factory = None): - result = (Element if factory is None else factory)(self.tag, self.attrib) - result.text = self.text - result.tail = self.tail - result[:] = [i.copy() for i in self] - return result - def repr_serialize(self, params): - if len(self): - params.append( - 'children = [{0:s}]'.format( - ', '.join([repr(i) for i in self]) - ) - ) - def __repr__(self): - params = [] - self.repr_serialize(params) - return 'element.Element({0:s})'.format(', '.join(params)) - -bool_to_str = ['false', 'true'] -def serialize_bool(value): - return bool_to_str[int(value)] - -str_to_bool = {'false': False, 'true': True} -def deserialize_bool(text): - return str_to_bool[text] - -def serialize_int(value): - return str(value) - -def deserialize_int(text): - return int(text) - -def serialize_ref(value, ref_list): - if value is None: - ref = -1 - else: - ref = value.ref - if ref == -1: - ref = len(ref_list) - ref_list.append(value) - value.ref = ref - value.set('ref', str(ref)) - # this doesn't set the seen flag, so it will be parented by the - # root, unless it is already parented or gets parented later on - if not value.seen: - value.serialize(ref_list) - return str(ref) - -def deserialize_ref(text, ref_list): - ref = int(text) - return None if ref < 0 else ref_list[ref] - -def serialize_str(value): - return value - -def deserialize_str(text): - return text - -def serialize(value, fout, encoding = 'unicode'): - ref_list = [] - serialize_ref(value, ref_list) - parents = [i for i in ref_list if not i.seen] - root = Element('root', children = parents) - for i in range(len(root)): - set_text(root, i, '\n ') - set_text(root, len(root), '\n') - root.tail = '\n' - xml.etree.ElementTree.ElementTree(root).write(fout, encoding) - for i in root: - i.tail = None - for i in ref_list: - i.ref = -1 - del i.attrib['ref'] - i = 0 - while i < len(parents): - for j in parents[i]: - j.seen = False - parents.append(j) - i += 1 - -def deserialize(fin, factory = Element, encoding = 'unicode'): - root = xml.etree.ElementTree.parse( - fin, - xml.etree.ElementTree.XMLParser( - target = xml.etree.ElementTree.TreeBuilder(factory), - encoding = encoding - ) - ).getroot() - assert root.tag == 'root' - for i in root: - i.tail = None - i = 0 - parents = root[:] - ref_list = [] - while i < len(parents): - j = parents[i] - if 'ref' in j.attrib: - ref = int(j.attrib['ref']) - del j.attrib['ref'] - if len(ref_list) < ref + 1: - ref_list.extend([None] * (ref + 1 - len(ref_list))) - ref_list[ref] = j - parents.extend(j[:]) - i += 1 - for i in root: - i.deserialize(ref_list) - return ref_list[0] - -# compatibility scheme to access arbitrary xml.etree.ElementTree.Element-like -# objects (not just Element defined above) using a more consistent interface: -def get_text(root, i): - if i < 0: - i += len(root) + 1 - text = root.text if i == 0 else root[i - 1].tail - return '' if text is None else text - -def set_text(root, i, text): - if i < 0: - i += len(root) + 1 - if len(text) == 0: - text = None - if i == 0: - root.text = text - else: - root[i - 1].tail = text - -def to_text(root): - return ''.join( - [ - j - for i in range(len(root)) - for j in [get_text(root, i), to_text(root[i])] - ] + - [get_text(root, len(root))] - ) - -def concatenate(children, factory = Element, *args, **kwargs): - root = factory(*args, **kwargs) - for child in children: - i = len(root) - set_text(root, i, get_text(root, i) + get_text(child, 0)) - root[i:] = child[:] - return root diff --git a/l_to_python.py b/l_to_python.py index 7d3528b..90fd701 100755 --- a/l_to_python.py +++ b/l_to_python.py @@ -1,13 +1,28 @@ #!/usr/bin/env python3 import t_def -import element import lex_yy import os import sys import xml.etree.ElementTree import y_tab +def get_text(root, i): + if i < 0: + i += len(root) + 1 + text = root.text if i == 0 else root[i - 1].tail + return '' if text is None else text + +def set_text(root, i, text): + if i < 0: + i += len(root) + 1 + if len(text) == 0: + text = None + if i == 0: + root.text = text + else: + root[i - 1].tail = text + def my_rstrip(text, indent): i = len(text) while i > 0 and text[i - 1] == '}': @@ -29,13 +44,7 @@ def c_to_python(context, text): root.translate_translation_unit(context) return ''.join(context.lines) -root = xml.etree.ElementTree.parse( - sys.stdin, - xml.etree.ElementTree.XMLParser( - target = xml.etree.ElementTree.TreeBuilder(element.Element), - encoding = 'unicode' - ) -).getroot() +root = xml.etree.ElementTree.parse(sys.stdin).getroot() context = t_def.Context() @@ -54,10 +63,10 @@ with open('a.c', 'w') as fout: if len(i) == 0: # continued actions assert parent.tag == 'AST_Section2_Rule' assert len(parent) == 3 - element.set_text( + set_text( parent, 2, - element.get_text(parent, 2).rstrip('\t ') + ' /*COLUMN32*/ ' + get_text(parent, 2).rstrip('\t ') + ' /*COLUMN32*/ ' ) return node = i[0] @@ -69,24 +78,24 @@ with open('a.c', 'w') as fout: else: child_indent = indent if i.tag == 'AST': - element.set_text( + set_text( i, 0, - element.get_text(i, 0).lstrip() + get_text(i, 0).lstrip() ) for j in range(1, len(i)): - element.set_text( + set_text( i, j, '{0:s}\n{1:s}'.format( - element.get_text(i, j).strip(), + get_text(i, j).strip(), '\n' if j == 2 else '' ) ) - element.set_text( + set_text( i, len(i), - element.get_text(i, len(i)).lstrip() + get_text(i, len(i)).lstrip() ) elif i.tag == 'AST_Section1' or i.tag == 'AST_Section2': # kludge, concatenate single line codeblocks to see overall meaning, @@ -95,20 +104,20 @@ with open('a.c', 'w') as fout: while j < len(i): if i[j].tag == 'AST_Section1Or2_CodeBlock': k = j + 1 - pre_delimiter = element.get_text(i[j], 0) - post_delimiter = element.get_text(i[j], 1) + pre_delimiter = get_text(i[j], 0) + post_delimiter = get_text(i[j], 1) while ( k < len(i) and - len(element.get_text(i, k).strip()) == 0 and + len(get_text(i, k).strip()) == 0 and i[k].tag == 'AST_Section1Or2_CodeBlock' and - element.get_text(i[k], 0) == pre_delimiter and - element.get_text(i[k], 1) == post_delimiter + get_text(i[k], 0) == pre_delimiter and + get_text(i[k], 1) == post_delimiter ): k += 1 - element.set_text( + set_text( i[k - 1][0], 0, - ''.join([element.get_text(i[l][0], 0) for l in range(j, k)]) + ''.join([get_text(i[l][0], 0) for l in range(j, k)]) ) del i[j:k - 1] j += 1 @@ -116,31 +125,31 @@ with open('a.c', 'w') as fout: i.tag == 'AST_Section2_Rule' or i.tag == 'AST_Section2_Rule_FLexRule' ): - element.set_text(i, 0, element.get_text(i, 0).lstrip('\t ')) + set_text(i, 0, get_text(i, 0).lstrip('\t ')) elif i.tag == 'AST_Section2_CompoundRule': child_indent += ' ' - element.set_text( + set_text( i, 0, - indent + element.get_text(i, 0).lstrip('\t ') + indent + get_text(i, 0).lstrip('\t ') ) for j in range(1, len(i)): - element.set_text( + set_text( i, j, - #element.get_text(i, j).rstrip('\t ') + child_indent - my_rstrip(element.get_text(i, j), child_indent) + #get_text(i, j).rstrip('\t ') + child_indent + my_rstrip(get_text(i, j), child_indent) ) - element.set_text( + set_text( i, len(i), - indent + element.get_text(i, len(i)).lstrip('\t ') + indent + get_text(i, len(i)).lstrip('\t ') ) for j in i: extract(j, i, child_indent) return assert len(node) == 0 - text = element.get_text(node, 0) + text = get_text(node, 0) j = 0 while j < len(text): @@ -216,21 +225,21 @@ with open('a.i') as fin: pass elif text.index('\n') == len(text) - 1: if parent.tag == 'AST_Section2_Rule': - element.set_text( + set_text( parent, 2, - element.get_text(parent, 2).rstrip('\t ') + ' /*COLUMN32*/ ' + get_text(parent, 2).rstrip('\t ') + ' /*COLUMN32*/ ' ) text = text.lstrip('\t ') else: if parent.tag == 'AST_Section2_Rule': - element.set_text( + set_text( parent, 2, - element.get_text(parent, 2).rstrip('\t ') + ' ' + get_text(parent, 2).rstrip('\t ') + ' ' ) text = '{{\n{0:s}{1:s}}}\n'.format(text, indent) - element.set_text(node, 0, text) + set_text(node, 0, text) xml.etree.ElementTree.ElementTree(root).write( sys.stdout, diff --git a/scan-code_to_l.py b/scan-code_to_l.py index 82de2d5..5b72b56 100755 --- a/scan-code_to_l.py +++ b/scan-code_to_l.py @@ -1,9 +1,25 @@ #!/usr/bin/env python3 -import element import sys +import xml.etree.ElementTree -text = element.to_text(element.deserialize(sys.stdin)) +def get_text(root, i): + if i < 0: + i += len(root) + 1 + text = root.text if i == 0 else root[i - 1].tail + return '' if text is None else text + +def to_text(root): + return ''.join( + [ + j + for i in range(len(root)) + for j in [get_text(root, i), to_text(root[i])] + ] + + [get_text(root, len(root))] + ) + +text = to_text(xml.etree.ElementTree.parse(sys.stdin).getroot()[0]) # see tests/scan-code.l text = text.replace(r'{tag}([^\0\n>]|->)+', '{tag}') diff --git a/scan-gram_to_l.py b/scan-gram_to_l.py index 5bf4951..e082074 100755 --- a/scan-gram_to_l.py +++ b/scan-gram_to_l.py @@ -1,14 +1,40 @@ #!/usr/bin/env python3 -import element import sys +import xml.etree.ElementTree + +def get_text(root, i): + if i < 0: + i += len(root) + 1 + text = root.text if i == 0 else root[i - 1].tail + return '' if text is None else text + +def set_text(root, i, text): + if i < 0: + i += len(root) + 1 + if len(text) == 0: + text = None + if i == 0: + root.text = text + else: + root[i - 1].tail = text + +def to_text(root): + return ''.join( + [ + j + for i in range(len(root)) + for j in [get_text(root, i), to_text(root[i])] + ] + + [get_text(root, len(root))] + ) -root = element.deserialize(sys.stdin) +root = xml.etree.ElementTree.parse(sys.stdin).getroot()[0] def replace_in_action(i): if i.tag == 'AST_Section2_Rule_Action' and len(i): assert i[0].tag == 'AST_Text' - text = element.get_text(i[0], 0) + text = get_text(i[0], 0) # see tests/parse-gram.y text = text.replace('GRAM_EOF', 'y_tab.GRAM_EOF') @@ -79,13 +105,13 @@ def replace_in_action(i): text = text.replace('PERCENT_EMPTY', 'y_tab.PERCENT_EMPTY') text = text.replace('PERCENT_SPACE', 'y_tab.PERCENT_SPACE') - element.set_text(i[0], 0, text) + set_text(i[0], 0, text) else: for j in i: replace_in_action(j) replace_in_action(root) -text = element.to_text(root) +text = to_text(root) # see tests/scan-gram.l text = text.replace(r'{letter}[.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]', '{letter}') diff --git a/scan_to_l.py b/scan_to_l.py index 4486277..8f16404 100755 --- a/scan_to_l.py +++ b/scan_to_l.py @@ -1,14 +1,40 @@ #!/usr/bin/env python3 -import element import sys +import xml.etree.ElementTree + +def get_text(root, i): + if i < 0: + i += len(root) + 1 + text = root.text if i == 0 else root[i - 1].tail + return '' if text is None else text + +def set_text(root, i, text): + if i < 0: + i += len(root) + 1 + if len(text) == 0: + text = None + if i == 0: + root.text = text + else: + root[i - 1].tail = text + +def to_text(root): + return ''.join( + [ + j + for i in range(len(root)) + for j in [get_text(root, i), to_text(root[i])] + ] + + [get_text(root, len(root))] + ) -root = element.deserialize(sys.stdin) +root = xml.etree.ElementTree.parse(sys.stdin).getroot()[0] def replace_in_action(i): if i.tag == 'AST_Section2_Rule_Action' and len(i): assert i[0].tag == 'AST_Text' - text = element.get_text(i[0], 0) + text = get_text(i[0], 0) # see tests/parse.y text = text.replace('CHAR', 'y_tab.CHAR') @@ -61,13 +87,13 @@ def replace_in_action(i): text = text.replace('BEGIN_REPEAT_FLEX', 'y_tab.BEGIN_REPEAT_FLEX') text = text.replace('END_REPEAT_FLEX', 'y_tab.END_REPEAT_FLEX') - element.set_text(i[0], 0, text) + set_text(i[0], 0, text) else: for j in i: replace_in_action(j) replace_in_action(root) -text = element.to_text(root) +text = to_text(root) # see tests/scan.l text = text.replace(r'{WS}[[:blank:]]+', '{WS}') diff --git a/xml_to_y.py b/xml_to_y.py index cff1c55..d056189 100755 --- a/xml_to_y.py +++ b/xml_to_y.py @@ -1,8 +1,24 @@ #!/usr/bin/env python3 -import element import sys +import xml.etree.ElementTree + +def get_text(root, i): + if i < 0: + i += len(root) + 1 + text = root.text if i == 0 else root[i - 1].tail + return '' if text is None else text + +def to_text(root): + return ''.join( + [ + j + for i in range(len(root)) + for j in [get_text(root, i), to_text(root[i])] + ] + + [get_text(root, len(root))] + ) sys.stdout.write( - element.to_text(element.deserialize(sys.stdin)) + to_text(xml.etree.ElementTree.parse(sys.stdin).getroot()[0]) ) diff --git a/y_to_python.py b/y_to_python.py index 4e0baee..74326e8 100755 --- a/y_to_python.py +++ b/y_to_python.py @@ -1,13 +1,38 @@ #!/usr/bin/env python3 import t_def -import element import lex_yy import os import sys import xml.etree.ElementTree import y_tab +def get_text(root, i): + if i < 0: + i += len(root) + 1 + text = root.text if i == 0 else root[i - 1].tail + return '' if text is None else text + +def set_text(root, i, text): + if i < 0: + i += len(root) + 1 + if len(text) == 0: + text = None + if i == 0: + root.text = text + else: + root[i - 1].tail = text + +def to_text(root): + return ''.join( + [ + j + for i in range(len(root)) + for j in [get_text(root, i), to_text(root[i])] + ] + + [get_text(root, len(root))] + ) + def c_to_python(context, text): lex_yy.yyin = None lex_yy.yy_buffer_stack = [lex_yy.YYBufferState()] @@ -18,13 +43,7 @@ def c_to_python(context, text): root.translate_translation_unit(context) return ''.join(context.lines) -root = xml.etree.ElementTree.parse( - sys.stdin, - xml.etree.ElementTree.XMLParser( - target = xml.etree.ElementTree.TreeBuilder(element.Element), - encoding = 'unicode' - ) -).getroot() +root = xml.etree.ElementTree.parse(sys.stdin).getroot() context = t_def.Context() @@ -37,10 +56,10 @@ with open('a.c', 'w') as fout: indent = ' ' initial = True # we won't put code on same line as %{, though it's legal as input - element.set_text( + set_text( i, 0, - '{0:s}\n'.format(element.get_text(i, 0).rstrip()) + '{0:s}\n'.format(get_text(i, 0).rstrip()) ) elif ( i.tag == 'AST_Section1_InitialAction' or @@ -61,88 +80,88 @@ with open('a.c', 'w') as fout: initial = True else: if i.tag == 'AST': - element.set_text( + set_text( i, 0, - element.get_text(i, 0).lstrip() + get_text(i, 0).lstrip() ) for j in range(1, len(i)): - element.set_text( + set_text( i, j, - '\n\n{0:s}\n\n'.format(element.get_text(i, j).strip()) + '\n\n{0:s}\n\n'.format(get_text(i, j).strip()) ) - element.set_text( + set_text( i, len(i), - element.get_text(i, len(i)).lstrip() + get_text(i, len(i)).lstrip() ) elif i.tag == 'AST_Section2': - element.set_text( + set_text( i, 0, - element.get_text(i, 0).lstrip() + get_text(i, 0).lstrip() ) for j in range(1, len(i)): - element.set_text( + set_text( i, j, - '\n\n{0:s}'.format(element.get_text(i, j).lstrip()) + '\n\n{0:s}'.format(get_text(i, j).lstrip()) ) - element.set_text( + set_text( i, len(i), - element.get_text(i, len(i)).lstrip() + get_text(i, len(i)).lstrip() ) elif i.tag == 'AST_Section2_Rules': - element.set_text( + set_text( i, 0, - element.get_text(i, 0).lstrip() + get_text(i, 0).lstrip() ) # deal with ... : assert i[0].tag == 'AST_SymbolRef' - element.set_text( + set_text( i[0], 1, - '\n ' + element.get_text(i[0], 1).strip() + '\n ' + get_text(i[0], 1).strip() ) - element.set_text( + set_text( i, 1, - element.get_text(i, 1).strip() + get_text(i, 1).strip() ) for j in range(2, len(i) + 1): - element.set_text( + set_text( i, j, - '\n {0:s}'.format(element.get_text(i, j).strip()) + '\n {0:s}'.format(get_text(i, j).strip()) ) elif i.tag == 'AST_Production': for j in range(len(i)): - element.set_text( + set_text( i, j, ( - '\n {0:s}'.format(element.get_text(i, j).lstrip()) + '\n {0:s}'.format(get_text(i, j).lstrip()) if ( i[j].tag == 'AST_Production_Action' or (j > 0 and i[j - 1].tag == 'AST_Production_Action') ) else - ' {0:s}'.format(element.get_text(i, j).lstrip()) + ' {0:s}'.format(get_text(i, j).lstrip()) ) ) - element.set_text( + set_text( i, len(i), - element.get_text(i, len(i)).lstrip() + get_text(i, len(i)).lstrip() ) for j in i: extract(j) return #assert len(node) == 0 - #text = element.get_text(node, 0) - text = element.to_text(node) + #text = get_text(node, 0) + text = to_text(node) lines = [i.rstrip() for i in text.split('\n')] while len(lines) and len(lines[-1]) == 0: @@ -214,7 +233,7 @@ with open('a.i') as fin: assert text[:len(indent) + 9] == '{0:s}def a():\n'.format(indent) text = '{{\n{0:s}{1:s}}}'.format(text[len(indent) + 9:], indent) del node[:] - element.set_text(node, 0, text) + set_text(node, 0, text) xml.etree.ElementTree.ElementTree(root).write( sys.stdout, -- 2.34.1