Convert generate_flex.py to new element class, reinstate the C based tests, fix a...

diff --git a/.gitignore b/.gitignore
index c94ec9c..c05688f 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -17,7 +17,6 @@ __pycache__
 /t_def.py
 /tests/*.c
 /tests/*.o
-/tests/*.xml
 /tests/cal
 /tests/flex0
 /tests/flex1

diff --git a/generate_flex.py b/generate_flex.py
index b73f057..c99bcf3 100644 (file)
--- a/generate_flex.py
+++ b/generate_flex.py
@@ -34,11 +34,11 @@ def generate_flex(_ast, _element, home_dir, skel_file, out_file):
   # end of buffer expression (do here because only necessary for flex)
   eob_regex = regex.RegexGroup(children = [regex.RegexEmpty()])
   eob_groups = []
-  eob_regex.post_process(eob_groups, caseless = _ast[0].caseless)
+  eob_regex.post_process(eob_groups, caseless = _ast.children[0].caseless)
   assert len(eob_groups) == 1
   eob_regex.add_to_nfa(
     _nfa,
-    [(len(_ast.flex_rules) * 2, len(_ast.flex_rules) * 2 + 1)]
+    [(len(_ast.actions_text) * 2, len(_ast.actions_text) * 2 + 1)]
   )
 
   _flex_dfa = _nfa.to_dfa().to_flex_dfa()
@@ -47,9 +47,9 @@ def generate_flex(_ast, _element, home_dir, skel_file, out_file):
     skel_file = os.path.join(home_dir, 'skel/skel_flex.c')
   if out_file is None:
     out_file = (
-      _ast[0].outfile
-    if len(_ast[0].outfile) else
-      'lex.{0:s}.c'.format(_ast[0].prefix)
+      _ast.children[0].outfile
+    if len(_ast.children[0].outfile) else
+      'lex.{0:s}.c'.format(_ast.children[0].prefix)
     )
   with open(skel_file, 'r') as fin:
     with open(out_file, 'w+') as fout:
@@ -61,12 +61,12 @@ def generate_flex(_ast, _element, home_dir, skel_file, out_file):
 {0:s}/* GENERATE END */
 '''.format(
               ''
-            if _ast[0].prefix == 'yy' else
+            if _ast.children[0].prefix == 'yy' else
               ''.join(
                 [
                   '#define yy{0:s} {1:s}{2:s}\n'.format(
                     i,
-                    _ast[0].prefix,
+                    _ast.children[0].prefix,
                     i
                   )
                   for i in [
@@ -144,11 +144,11 @@ def generate_flex(_ast, _element, home_dir, skel_file, out_file):
 {0:s}/* GENERATE END */
 '''.format(
               ''
-            if _ast[0].yywrap else
+            if _ast.children[0].yywrap else
               '''#define {0:s}wrap() (/*CONSTCOND*/1)
 #define YY_SKIP_YYWRAP
 '''.format(
-                _ast[0].prefix
+                _ast.children[0].prefix
               )
             )
           )
@@ -256,7 +256,7 @@ static const flex_uint16_t yy_chk[] = {{{6:s}
             '''/* GENERATE SECTION1 BEGIN */
 {0:s}/* GENERATE END */
 '''.format(
-              ''.join([i.get_text() for i in _ast[0].code_blocks_text])
+              ''.join([i.get_text() for i in _ast.children[0].code_blocks_text])
             )
           )
         elif line == '/* GENERATE STARTCONDDECL */\n':
@@ -280,7 +280,7 @@ static const flex_uint16_t yy_chk[] = {{{6:s}
             '''/* GENERATE SECTION2INITIAL BEGIN */
 {0:s}/* GENERATE END */
 '''.format(
-              ''.join([i.get_text() for i in _ast[1].code_blocks_text])
+              ''.join([i.get_text() for i in _ast.children[1].code_blocks_text])
             )
           )
         elif line == '/* GENERATE SECTION2 */\n':
@@ -333,11 +333,11 @@ YY_RULE_SETUP
             '''/* GENERATE SECTION3 BEGIN */
 {0:s}/* GENERATE END */
 '''.format(
-              '' if len(_ast) < 3 else _ast[2].get_text()
+              '' if len(_ast.children) < 3 else _ast.children[2].get_text()
             )
           )
         else:
-          if _ast[0].prefix != 'yy':
-            line = line.replace('yywrap', '{0:s}wrap'.format(_ast[0].prefix))
+          if _ast.children[0].prefix != 'yy':
+            line = line.replace('yywrap', '{0:s}wrap'.format(_ast.children[0].prefix))
           fout.write(line)
         line = fin.readline()

diff --git a/pilex.t b/pilex.t
index d2cf631..320da89 100644 (file)
--- a/pilex.t
+++ b/pilex.t
@@ -903,7 +903,7 @@ def post_process(self):
   ]
   self.actions_text = []
   self.eof_actions_text = [
-    AST.Text(['\t\t\t\tyyterminate();\n'])
+    AST.Text(text = ['\t\t\t\tyyterminate();\n'])
   ]
   self.flex_rules = []
   self.n_groups = 0
@@ -927,7 +927,7 @@ def post_process(self):
   self.default_action = len(self.actions_text)
   self.actions_text.append(
     AST.Text(
-      [
+      text = [
         'ECHO;\n'
       if self.children[0].default else
         'YY_FATAL_ERROR( "flex scanner jammed" );\n'

diff --git a/tests/Makefile b/tests/Makefile
index 4928fc0..b60828d 100644 (file)
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -4,8 +4,7 @@ all: lex_yy.py cal flex0 flex1
 
 # Python scanner test
 lex_yy.py: cal_py.l
-       ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
-       ../pilex.py --python $<.xml
+       ../pilex.py --python $<
 
 # cal program
 cal: y.tab.o
@@ -14,11 +13,10 @@ cal: y.tab.o
 y.tab.o: y.tab.c lex.yy.c
 
 y.tab.c: cal.y
-       ../../bootstrap_bison.git/src/bison -y $< 2>$<.xml
+       bison -y $<
 
 lex.yy.c: cal.l
-       ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
-       ../pilex.py $<.xml
+       ../pilex.py $<
        # add the patch for state machine diagnostic
        #cp $@ $@.orig
        #patch $@ <$@.patch
@@ -30,8 +28,7 @@ flex0: flex0.o
 flex0.o: flex0.c
 
 flex0.c: flex0.l
-       ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
-       ../pilex.py -o $@ $<.xml
+       ../pilex.py -o $@ $<
 
 # flex1 program
 flex1: flex1.o
@@ -40,9 +37,8 @@ flex1: flex1.o
 flex1.o: flex1.c
 
 flex1.c: flex1.l
-       ../../bootstrap_flex.git/src/flex -o /dev/null $< 2>$<.xml
-       ../pilex.py -o $@ $<.xml
+       ../pilex.py -o $@ $<
 
 # other
 clean:
-       rm -f *.c *.o *.xml lex_yy.py cal flex0 flex1
+       rm -f *.c *.o lex_yy.py cal flex0 flex1

diff --git a/tests/cal.py b/tests/cal.py
new file mode 100755 (executable)
index 0000000..80a1107
--- /dev/null
+++ b/tests/cal.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2019 Nick Downing <nick@ndcode.org>
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+import lex_yy
+
+token = lex_yy.yylex()
+while token != lex_yy.YY_NULL:
+  if token == lex_yy.NUM:
+    print('NUM', lex_yy.yylval)
+  else:
+    print('{0:02x}'.format(token))
+  token = lex_yy.yylex()

diff --git a/tests/cal.y b/tests/cal.y
index e73a2b6..7637aff 100644 (file)
--- a/tests/cal.y
+++ b/tests/cal.y
@@ -31,7 +31,7 @@ E : E '+' E { $$ = $1 + $3; }
 
 #include "lex.yy.c"
 
-int main()
+int main(void)
 {
   printf("Enter the expression: ");
   yyparse();

diff --git a/tests/cal_py.l b/tests/cal_py.l
index a4fadeb..d0879aa 100644 (file)
--- a/tests/cal_py.l
+++ b/tests/cal_py.l
@@ -1,3 +1,21 @@
+/*
+ * Copyright (C) 2019 Nick Downing <nick@ndcode.org>
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+ */
+
 %{
 NUM = 0x100 
 yylval = None
@@ -25,26 +43,15 @@ DIGIT (?'mantissa'[0-9]+)\.?|(?'mantissa'[0-9]*)\.(?'fraction'[0-9]+)
   #else:
   #  mantissa = yy_groups[2]
   #  fraction = yy_groups[3]
-  #yylval = int(mantissa + fraction) * 10 ** -len(fraction)
+  #yylval = int(mantissa + fraction) * 10. ** -len(fraction)
 
   # by group names:
   mantissa = yy_groups_by_name['mantissa']
   fraction = yy_groups_by_name.get('fraction', '')
-  yylval = int(mantissa + fraction) * 10 ** -len(fraction)
+  yylval = int(mantissa + fraction) * 10. ** -len(fraction)
 
   return NUM
 }
 \n|.   {
   return ord(yytext[0])
 }
-
-%%
-
-if __name__ == '__main__':
-  token = yylex()
-  while token != 0:
-    if token == NUM:
-      print('NUM', yylval)
-    else:
-      print('{0:02x}'.format(token))
-    token = yylex()

diff --git a/tests_ast/cal.py b/tests_ast/cal.py
new file mode 100755 (executable)
index 0000000..e39465b
--- /dev/null
+++ b/tests_ast/cal.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2019 Nick Downing <nick@ndcode.org>
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+import element
+import lex_yy
+import sys
+
+token = lex_yy.yylex()
+while token != lex_yy.YY_NULL:
+  print('lex_yy.yy_element_space')
+  element.serialize(lex_yy.yy_element_space, sys.stdout)
+  print('lex_yy.yy_element_token')
+  element.serialize(lex_yy.yy_element_token, sys.stdout)
+  if token == lex_yy.NUM:
+    print('NUM', lex_yy.yylval)
+  else:
+    print('{0:02x}'.format(token))
+  token = lex_yy.yylex()

diff --git a/tests_ast/cal_py.l b/tests_ast/cal_py.l
index 0fcc85f..2b56a9e 100644 (file)
--- a/tests_ast/cal_py.l
+++ b/tests_ast/cal_py.l
@@ -20,17 +20,3 @@ DIGIT (?E{t_def.AST.Num.Mantissa}[0-9]+)\.?|(?E{t_def.AST.Num.Mantissa}[0-9]*)\.
   return ord(yytext[0])
 }
 
-%%
-
-if __name__ == '__main__':
-  token = yylex()
-  while token != 0:
-    print('yy_element_space')
-    element.serialize(yy_element_space, sys.stdout)
-    print('yy_element_token')
-    element.serialize(yy_element_token, sys.stdout)
-    if token == NUM:
-      print('NUM', yylval)
-    else:
-      print('{0:02x}'.format(token))
-    token = yylex()