import bisect
import sys
+YY_TRAILING_MASK = 0x2000
+YY_TRAILING_HEAD_MASK = 0x4000
+
# this can be redefined in SECTION1
def YY_AT_BOL():
return yy_buffer_stack[-1].at_bol
class YYTerminate(Exception):
pass
+# each input file is represented as a linked list in which the
+# head is a YYBufferState and the following items (if any) are
+# YYBufferBlock -- the YYBufferList parent class is defined so
+# that we can guarantee the "previous" item to any particular
+# YYBufferBlock is at least a YYBufferList and hence deletion
+# (or other operation that needs the "previous" item) is easy
class YYBufferList:
def __init__(self, next = None):
self.next = next
yystart = INITIAL
yystart_stack = []
-yy_threads0 = [None]
-yy_threads1 = [None]
-yy_prefix_slop = 1
-
-yy_group_text = None
-yy_group_stack = None
-yy_groups = None
-yy_groups_by_name = None
-yy_action = None
+
yytext = ''
yytext_len = 0
+yylineno = 0
+
+yy_state_buf = []
+
YY_NULL = 0
def REJECT():
+ # can't have already committed (can't call unput then REJECT)
+ assert yytext_len == len(yytext) # cannot have already committed
raise YYReject()
def yyterminate():
def yyless(i):
global yytext, yytext_len
+ # can't have already committed (can't call unput then yyless)
+ assert yytext_len == len(yytext)
assert yytext_len >= i
yytext = yytext[:i]
yytext_len = i
-def unput(text):
- global yyin, yytext_len
+def yycommit():
+ # now user code cannot call REJECT() / yyless() anymore,
+ # thus we know the match was of length yytext_len and we
+ # can skip that many chars, popping buffer stack for real
+ # we leave yytext == 0 afterwards, so double commit is OK
+ # (double commit occurs if user calls the unput function)
+ global yytext_len
+ print('commit', yytext_len)
while yytext_len:
block = yy_buffer_stack[-1].next
while block is None or block.pos >= len(block.text):
i = min(yytext_len, len(block.text) - block.pos)
block.pos += i
yytext_len -= i
+
+def unput(text):
+ yycommit() # user cannot call REJECT() / yyless() anymore
yy_buffer_stack[-1].next = YYBufferBlock(yy_buffer_stack[-1].next, 0, text)
def ECHO():
yyout.write(yytext)
-def yy_rule_start():
- global yytext, yytext_len
- yytext = yy_group_text[:yy_group_stack[-1]]
- yytext_len = yy_group_stack[-1]
- del yy_group_stack[-2:]
- YY_RULE_START()
-
-def yy_group_end():
- pass
-
def BEGIN(start):
global yystart
yystart = start
# GENERATE SECTION2
def yylex():
- global \
- yyin, \
- yy_threads0, \
- yy_threads1, \
- yy_prefix_slop, \
- yy_group_text, \
- yy_group_stack, \
- yy_action, \
- yytext, \
- yytext_len
+ global yyin, yytext, yytext_len, yylineno
# GENERATE SECTION2INITIAL
- while True:
- while yytext_len:
- block = yy_buffer_stack[-1].next
- while block is None or block.pos >= len(block.text):
- if block is None:
- yy_buffer_stack.pop()
- block = yy_buffer_stack[-1].next
- yyin = yy_buffer_stack[-1].file_in
- else:
- block = block.next
- yy_buffer_stack[-1].next = block
- i = min(yytext_len, len(block.text) - block.pos)
- block.pos += i
- yytext_len -= i
+ # user may modify yyin, but only while outside yylex()
+ yy_buffer_stack[-1].file_in = yyin
+ # start:
+ while True:
+ # this will collect the remainder of each block as the block is
+ # exhausted, any partial block will be added after DFA is stuck
match = ''
- match_len = 0
- del yy_threads0[yy_prefix_slop:]
- yy_threads0.append(None)
+ # this is used to keep track of newlines, firstly during the scan
+ # and secondly when user calls REJECT() or yyless(), basically we
+ # will count newlines added/removed when it is extended/truncated
+ yytext = ''
+ # as we progress through the text and exhaust each input file
+ # on the buffer stack, we will "virtually" pop the stack by
+ # decrementing buffer_ptr, but actually leave the stack alone
+ # since it will be popped for real after executing the action
buffer_ptr = len(yy_buffer_stack) - 1
+
+ # block variable keeps track of where we are in current file's
+ # linked list, if None it means we have run out of blocks from
+ # the current file and in that case we have to append another
+ # block -- we maintain the invariant block == block_prev.next
+ # so we can append the block by pointing block_prev.next at it
block_prev = yy_buffer_stack[buffer_ptr]
block = block_prev.next
if block is not None:
block_pos = block.pos
- action = yy_dfa_start_action[
- yystart * 2 + int(yy_buffer_stack[-1].at_bol)
- ]
- while action != -1:
- state, transition = yy_dfa_actions[action]
- #print('i', i, 'action', action, 'state', state, 'transition', transition)
-
- i = yy_prefix_slop
- assert len(yy_threads1) == yy_prefix_slop
- for trans in transition:
- if trans[0] == 0: #DFA.TRANSITION_POP:
- i += trans[1]
- elif trans[0] == 1: #DFA.TRANSITION_DUP:
- while i < trans[1]:
- yy_threads0[:0] = [None] * yy_prefix_slop
- yy_threads1[:0] = [None] * yy_prefix_slop
- i += yy_prefix_slop
- yy_prefix_slop *= 2
- yy_threads0[i - trans[1]:i] = yy_threads0[i:i + trans[1]]
- i -= trans[1]
- elif trans[0] == 2: #DFA.TRANSITION_MARK:
- yy_threads0[i:i + trans[1]] = [
- (match_len, trans[2], thread)
- for thread in yy_threads0[i:i + trans[1]]
- ]
- elif trans[0] == 3: #DFA.TRANSITION_MOVE:
- yy_threads1.extend(yy_threads0[i:i + trans[1]])
- i += trans[1]
- #elif trans[0] == DFA.TRANSITION_DEL:
- # del yy_threads1[-trans[1]:]
- else:
- assert False
- assert i == len(yy_threads0)
- yy_threads0, yy_threads1 = yy_threads1, yy_threads0
- del yy_threads1[yy_prefix_slop:]
-
- if state == 0:
- # there is only one match, which is complete
- assert len(yy_threads0) == yy_prefix_slop + 1
- assert yy_dfa_states[state][2] == [0]
- break
-
- yy_buffer_stack[-1].file_in = yyin
+ # yy_current_state variable keeps track of where we are in the
+ # DFA, and yy_state_buf keeps track of the historical values of
+ # yy_current_state before each character -- when DFA is stuck
+ # we will look backwards in yy_state_buf for an accepting state
+ yy_current_state = yystart * 2 + 1 + YY_AT_BOL()
+ del yy_state_buf[:]
+
+ # yy_match:
+ print('yy_match')
+ while True:
+ # try to get a character
while block is None or block_pos >= len(block.text):
if block is None:
+ # out of blocks, read and append one from current file
file_in = yy_buffer_stack[buffer_ptr].file_in
text = '' if file_in is None else file_in.readline()
if len(text):
block_pos = 0
block_prev.next = block
else:
- # do not re-attempt read once EOF is reached
+ # EOF for current file
+
+ # mark file as being exhausted, so we do not re-attempt read
yy_buffer_stack[buffer_ptr].file_in = None
- yyin = yy_buffer_stack[-1].file_in
+
+ # "virtually" pop the buffer stack, maintaining invariant
buffer_ptr -= 1
if buffer_ptr < 0:
- break # EOF
+ # EOF and no more input stacked (should call yywrap here?)
+ if len(yy_state_buf):
+ # had some characters already, so go and process them first
+ break # goto yy_find_action
+ # EOF on attempt to get first character, means we are done
+ try:
+ token = yy_eof_actions[yystart]()
+ yyin = yy_buffer_stack[-1].file_in = yyin # user may modify
+ return token
+ except YYTerminate:
+ yyin = yy_buffer_stack[-1].file_in = yyin # user may modify
+ return 0
block_prev = yy_buffer_stack[buffer_ptr]
block = block_prev.next
if block is not None:
block_pos = block.pos
else:
- i = match_len - len(match)
+ # out of data for current block
+
+ # take most recent data from block for yytext
+ # at this point len(yy_state_buf) == number of chars scanned
+ i = len(yy_state_buf) - len(match)
if i:
match += block.text[block_pos - i:]
+
+ # advance past exhausted block, maintaining invariant
block_prev = block
block = block_prev.next
if block is not None:
block_pos = block.pos
- else:
- #print('block_pos', block_pos, 'block.text', block.text)
- action = yy_dfa_states[state][1][
- bisect.bisect_right(
- yy_dfa_states[state][0],
- ord(block.text[block_pos])
- )
- ]
- block_pos += 1
- match_len += 1
- continue
- # EOF
- if i == 0:
- try:
- return yy_eof_actions[yystart]()
- except YYTerminate:
- return 0
- break
-
- i = match_len - len(match)
+ else:
+ # character available
+ yy_state_buf.append(yy_current_state)
+ yy_c = ord(block.text[block_pos])
+ print('yy_c', yy_c)
+ while yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state:
+ yy_current_state = yy_def[yy_current_state];
+ yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
+ if yy_base[yy_current_state] != 0:
+ block_pos += 1
+ continue # goto yy_match
+
+ # we get here when either there is EOF and had some characters, or
+ # we had a character which makes DFA stuck -- do best action we can
+ break # goto yy_find_action
+
+ # if the following assertion fails there is a hole in state machine,
+ # this is not supposed to happen due to the automatic catch-all rule
+ yy_state_ptr = len(yy_state_buf) - 1
+ assert yy_state_ptr >= 0
+ yy_current_state = yy_state_buf[yy_state_ptr]
+ yy_lp = yy_accept[yy_current_state]
+
+ # take most recent data from block for yytext
+ # at this point yy_state_ptr == number of chars scanned
+ i = yy_state_ptr - len(match)
if i:
- assert block is not None
match += block.text[block_pos - i:]
- for i in yy_dfa_states[state][2]:
- yy_group_text = match
- yy_group_stack = []
- yy_groups = None
- yy_groups_by_name = None
- yy_action = None
- yytext = None
- yytext_len = None
-
- thread = yy_threads0[yy_prefix_slop + i]
- #print('thread', thread)
- while thread is not None:
- pos, ref_data, thread = thread
- yy_group_stack.append(pos)
- ref_data()
-
- try:
- return yy_action()
- except YYReject:
- pass
- except YYContinue:
- break
- except YYTerminate:
- return 0
+ # yy_find_action:
+ # the following variables are used to handle trailing context
+ # suppose there is a rule like: foo/bar
+ # then state after 3 characters will be given YY_TRAILING_MASK
+ # and state after 6 characters will be given YY_TRAILING_HEAD_MASK
+ # so we look backwards for the YY_TRAILING_MASK, and when we find
+ # it, store the state after 6 characters into yy_full_XXX, then we
+ # keep looking for YY_TRAILING_HEAD_MASK, and when we find it, we
+ # execute the action -- and if it rejects then restart the search,
+ # but from the 6 characters point (yy_full_XXX) not 3 characters
+ yy_looking_for_trail_begin = 0
+ yy_full_state_ptr = None
+ yy_full_lp = None
+
+ # find_rule:
+ # we branch to this label when backing up (i.e. reject)
+ while True: # until we find what rule matched
+ if yy_lp < yy_accept[yy_current_state + 1]:
+ yy_act = yy_acclist[yy_lp]
+ if (yy_act & YY_TRAILING_HEAD_MASK) or yy_looking_for_trail_begin:
+ if yy_act == yy_looking_for_trail_begin:
+ yy_looking_for_trail_begin = 0
+ yy_act &= ~YY_TRAILING_HEAD_MASK
+ break
+ elif yy_act & YY_TRAILING_MASK:
+ yy_looking_for_trail_begin = (
+ (yy_act & ~YY_TRAILING_MASK) | YY_TRAILING_HEAD_MASK
+ )
+ yy_full_state_ptr = yy_state_ptr
+ yy_full_lp = yy_lp
+ else:
+ yy_full_state_ptr = yy_state_ptr
+ yy_full_lp = yy_lp
+ break
+ yy_lp += 1
+ else:
+ # if the following assertion fails there is a hole in state machine,
+ # this is not supposed to happen due to the automatic catch-all rule
+ yy_state_ptr -= 1
+ assert yy_state_ptr >= 0
+ yy_current_state = yy_state_buf[yy_state_ptr]
+ yy_lp = yy_accept[yy_current_state]
+
+ # at this point yy_state_ptr == number of chars needed
+ # truncate or extend yytext from match, counting added/removed newlines
+ if len(yytext) < yy_state_ptr:
+ i = match.find('\n', len(yytext), yy_state_ptr)
+ while i >= 0:
+ yylineno += 1
+ i = match.find('\n', i + 1, yy_state_ptr)
else:
- raise Exception('scanner jammed')
+ i = yytext.find('\n', yy_state_ptr)
+ while i >= 0:
+ yylineno -= 1
+ i = yytext.find('\n', i + 1)
+ yytext = match[:yy_state_ptr]
+
+ try:
+ yytext_len = len(yytext) # this is used by yycommit()
+ token = yy_actions[yy_act]()
+ yycommit() # user cannot call REJECT() / yyless() anymore
+ yyin = yy_buffer_stack[-1].file_in = yyin # user may modify
+ return token
+ except YYReject:
+ yy_state_ptr = yy_full_state_ptr
+ yy_current_state = yy_state_buf[yy_state_ptr]
+ yy_lp = yy_full_lp + 1
+ # goto find_rule
+ except YYContinue:
+ yycommit() # user cannot call REJECT() / yyless() anymore
+ break # goto start
+ except YYTerminate:
+ yycommit() # user cannot call REJECT() / yyless() anymore
+ yyin = yy_buffer_stack[-1].file_in = yyin # user may modify
+ return 0
+
+ # goto find_rule
+
+ # goto start
# GENERATE SECTION3
--- /dev/null
+# Copyright (C) 2019 Nick Downing <nick@ndcode.org>
+# SPDX-License-Identifier: GPL-2.0-with-bison-exception
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+#
+# As a special exception, you may create a larger work that contains part or
+# all of the pilex lexical scanner skeleton and distribute that work under
+# terms of your choice, so long as that work isn't itself a lexical scanner
+# generator using the skeleton or a modified version thereof as a lexical
+# scanner skeleton. Alternatively, if you modify or redistribute the lexical
+# scanner skeleton itself, you may (at your option) remove this special
+# exception, which will cause the skeleton and the resulting pilex output
+# files to be licensed under the GNU General Public License without this
+# special exception.
+
+import bisect
+import sys
+
+# this can be redefined in SECTION1
+def YY_AT_BOL():
+ return yy_buffer_stack[-1].at_bol
+
+def yy_set_bol(at_bol):
+ yy_buffer_stack[-1].at_bol = at_bol
+
+def YY_USER_ACTION():
+ pass
+
+def YY_RULE_START():
+ # note that this should also be done after yyless() and REJECT(),
+ # and state should be saved in case they result in a null string,
+ # however, it doesn't seem to be in flex, maintain compatibility:
+ if len(yytext):
+ yy_set_bol(yytext[-1] == '\n')
+ YY_USER_ACTION()
+
+# GENERATE SECTION1
+
+# GENERATE STARTCONDDECL
+
+class YYReject(Exception):
+ pass
+
+class YYContinue(Exception):
+ pass
+
+class YYTerminate(Exception):
+ pass
+
+class YYBufferList:
+ def __init__(self, next = None):
+ self.next = next
+
+class YYBufferBlock(YYBufferList):
+ def __init__(self, next = None, pos = 0, text = ''):
+ YYBufferList.__init__(self, next)
+ self.pos = pos
+ self.text = text
+
+class YYBufferState(YYBufferList):
+ def __init__(self, next = None, file_in = None, at_bol = True):
+ YYBufferList.__init__(self, next)
+ self.file_in = file_in
+ self.at_bol = at_bol
+
+yyin = sys.stdin
+yyout = sys.stdout
+yy_buffer_stack = [YYBufferState()]
+
+yystart = INITIAL
+yystart_stack = []
+yy_threads0 = [None]
+yy_threads1 = [None]
+yy_prefix_slop = 1
+
+yy_group_text = None
+yy_group_stack = None
+yy_groups = None
+yy_groups_by_name = None
+yy_action = None
+yytext = ''
+yytext_len = 0
+
+YY_NULL = 0
+
+def REJECT():
+ raise YYReject()
+
+def yyterminate():
+ raise YYTerminate()
+
+def yyless(i):
+ global yytext, yytext_len
+ assert yytext_len >= i
+ yytext = yytext[:i]
+ yytext_len = i
+
+def unput(text):
+ global yyin, yytext_len
+ while yytext_len:
+ block = yy_buffer_stack[-1].next
+ while block is None or block.pos >= len(block.text):
+ if block is None:
+ yy_buffer_stack.pop()
+ block = yy_buffer_stack[-1].next
+ yyin = yy_buffer_stack[-1].file_in
+ else:
+ block = block.next
+ yy_buffer_stack[-1].next = block
+ i = min(yytext_len, len(block.text) - block.pos)
+ block.pos += i
+ yytext_len -= i
+ yy_buffer_stack[-1].next = YYBufferBlock(yy_buffer_stack[-1].next, 0, text)
+
+def ECHO():
+ yyout.write(yytext)
+
+def yy_rule_start():
+ global yytext, yytext_len
+ yytext = yy_group_text[:yy_group_stack[-1]]
+ yytext_len = yy_group_stack[-1]
+ del yy_group_stack[-2:]
+ YY_RULE_START()
+
+def yy_group_end():
+ pass
+
+def BEGIN(start):
+ global yystart
+ yystart = start
+
+def YY_START():
+ return yystart
+
+def yy_push_state(start):
+ global yystart
+ yystart_stack.append(yystart)
+ yystart = start
+
+def yy_pop_state():
+ global yystart
+ yystart = yystart_stack.pop()
+
+# GENERATE SECTION2
+
+def yylex():
+ global \
+ yyin, \
+ yy_threads0, \
+ yy_threads1, \
+ yy_prefix_slop, \
+ yy_group_text, \
+ yy_group_stack, \
+ yy_action, \
+ yytext, \
+ yytext_len
+
+ # GENERATE SECTION2INITIAL
+
+ while True:
+ while yytext_len:
+ block = yy_buffer_stack[-1].next
+ while block is None or block.pos >= len(block.text):
+ if block is None:
+ yy_buffer_stack.pop()
+ block = yy_buffer_stack[-1].next
+ yyin = yy_buffer_stack[-1].file_in
+ else:
+ block = block.next
+ yy_buffer_stack[-1].next = block
+ i = min(yytext_len, len(block.text) - block.pos)
+ block.pos += i
+ yytext_len -= i
+
+ match = ''
+ match_len = 0
+
+ del yy_threads0[yy_prefix_slop:]
+ yy_threads0.append(None)
+
+ buffer_ptr = len(yy_buffer_stack) - 1
+ block_prev = yy_buffer_stack[buffer_ptr]
+ block = block_prev.next
+ if block is not None:
+ block_pos = block.pos
+
+ action = yy_dfa_start_action[
+ yystart * 2 + int(yy_buffer_stack[-1].at_bol)
+ ]
+ while action != -1:
+ state, transition = yy_dfa_actions[action]
+ #print('i', i, 'action', action, 'state', state, 'transition', transition)
+
+ i = yy_prefix_slop
+ assert len(yy_threads1) == yy_prefix_slop
+ for trans in transition:
+ if trans[0] == 0: #DFA.TRANSITION_POP:
+ i += trans[1]
+ elif trans[0] == 1: #DFA.TRANSITION_DUP:
+ while i < trans[1]:
+ yy_threads0[:0] = [None] * yy_prefix_slop
+ yy_threads1[:0] = [None] * yy_prefix_slop
+ i += yy_prefix_slop
+ yy_prefix_slop *= 2
+ yy_threads0[i - trans[1]:i] = yy_threads0[i:i + trans[1]]
+ i -= trans[1]
+ elif trans[0] == 2: #DFA.TRANSITION_MARK:
+ yy_threads0[i:i + trans[1]] = [
+ (match_len, trans[2], thread)
+ for thread in yy_threads0[i:i + trans[1]]
+ ]
+ elif trans[0] == 3: #DFA.TRANSITION_MOVE:
+ yy_threads1.extend(yy_threads0[i:i + trans[1]])
+ i += trans[1]
+ #elif trans[0] == DFA.TRANSITION_DEL:
+ # del yy_threads1[-trans[1]:]
+ else:
+ assert False
+ assert i == len(yy_threads0)
+ yy_threads0, yy_threads1 = yy_threads1, yy_threads0
+ del yy_threads1[yy_prefix_slop:]
+
+ if state == 0:
+ # there is only one match, which is complete
+ assert len(yy_threads0) == yy_prefix_slop + 1
+ assert yy_dfa_states[state][2] == [0]
+ break
+
+ yy_buffer_stack[-1].file_in = yyin
+ while block is None or block_pos >= len(block.text):
+ if block is None:
+ file_in = yy_buffer_stack[buffer_ptr].file_in
+ text = '' if file_in is None else file_in.readline()
+ if len(text):
+ block = YYBufferBlock(None, 0, text)
+ block_pos = 0
+ block_prev.next = block
+ else:
+ # do not re-attempt read once EOF is reached
+ yy_buffer_stack[buffer_ptr].file_in = None
+ yyin = yy_buffer_stack[-1].file_in
+ buffer_ptr -= 1
+ if buffer_ptr < 0:
+ break # EOF
+ block_prev = yy_buffer_stack[buffer_ptr]
+ block = block_prev.next
+ if block is not None:
+ block_pos = block.pos
+ else:
+ i = match_len - len(match)
+ if i:
+ match += block.text[block_pos - i:]
+ block_prev = block
+ block = block_prev.next
+ if block is not None:
+ block_pos = block.pos
+ else:
+ #print('block_pos', block_pos, 'block.text', block.text)
+ action = yy_dfa_states[state][1][
+ bisect.bisect_right(
+ yy_dfa_states[state][0],
+ ord(block.text[block_pos])
+ )
+ ]
+ block_pos += 1
+ match_len += 1
+ continue
+ # EOF
+ if i == 0:
+ try:
+ return yy_eof_actions[yystart]()
+ except YYTerminate:
+ return 0
+ break
+
+ i = match_len - len(match)
+ if i:
+ assert block is not None
+ match += block.text[block_pos - i:]
+
+ for i in yy_dfa_states[state][2]:
+ yy_group_text = match
+ yy_group_stack = []
+ yy_groups = None
+ yy_groups_by_name = None
+ yy_action = None
+ yytext = None
+ yytext_len = None
+
+ thread = yy_threads0[yy_prefix_slop + i]
+ #print('thread', thread)
+ while thread is not None:
+ pos, ref_data, thread = thread
+ yy_group_stack.append(pos)
+ ref_data()
+
+ try:
+ return yy_action()
+ except YYReject:
+ pass
+ except YYContinue:
+ break
+ except YYTerminate:
+ return 0
+ else:
+ raise Exception('scanner jammed')
+
+# GENERATE SECTION3
--- /dev/null
+# Copyright (C) 2019 Nick Downing <nick@ndcode.org>
+# SPDX-License-Identifier: GPL-2.0-with-bison-exception
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+#
+# As a special exception, you may create a larger work that contains part or
+# all of the pilex lexical scanner skeleton and distribute that work under
+# terms of your choice, so long as that work isn't itself a lexical scanner
+# generator using the skeleton or a modified version thereof as a lexical
+# scanner skeleton. Alternatively, if you modify or redistribute the lexical
+# scanner skeleton itself, you may (at your option) remove this special
+# exception, which will cause the skeleton and the resulting pilex output
+# files to be licensed under the GNU General Public License without this
+# special exception.
+
+import bisect
+import sys
+
+# this can be redefined in SECTION1
+def YY_AT_BOL():
+ return yy_buffer_stack[-1].at_bol
+
+def yy_set_bol(at_bol):
+ yy_buffer_stack[-1].at_bol = at_bol
+
+def YY_USER_ACTION():
+ pass
+
+def YY_RULE_START():
+ # note that this should also be done after yyless() and REJECT(),
+ # and state should be saved in case they result in a null string,
+ # however, it doesn't seem to be in flex, maintain compatibility:
+ if len(yytext):
+ yy_set_bol(yytext[-1] == '\n')
+ YY_USER_ACTION()
+
+# GENERATE SECTION1 BEGIN
+NUM = 0x100
+yylval = None
+# GENERATE END
+
+# GENERATE STARTCONDDECL BEGIN
+INITIAL = 0
+# GENERATE END
+
+class YYReject(Exception):
+ pass
+
+class YYContinue(Exception):
+ pass
+
+class YYTerminate(Exception):
+ pass
+
+class YYBufferList:
+ def __init__(self, next = None):
+ self.next = next
+
+class YYBufferBlock(YYBufferList):
+ def __init__(self, next = None, pos = 0, text = ''):
+ YYBufferList.__init__(self, next)
+ self.pos = pos
+ self.text = text
+
+class YYBufferState(YYBufferList):
+ def __init__(self, next = None, file_in = None, at_bol = True):
+ YYBufferList.__init__(self, next)
+ self.file_in = file_in
+ self.at_bol = at_bol
+
+yyin = sys.stdin
+yyout = sys.stdout
+yy_buffer_stack = [YYBufferState()]
+
+yystart = INITIAL
+yystart_stack = []
+yy_threads0 = [None]
+yy_threads1 = [None]
+yy_prefix_slop = 1
+
+yy_group_text = None
+yy_group_stack = None
+yy_groups = None
+yy_groups_by_name = None
+yy_action = None
+yytext = ''
+yytext_len = 0
+
+YY_NULL = 0
+
+def REJECT():
+ raise YYReject()
+
+def yyterminate():
+ raise YYTerminate()
+
+def yyless(i):
+ global yytext, yytext_len
+ assert yytext_len >= i
+ yytext = yytext[:i]
+ yytext_len = i
+
+def unput(text):
+ global yyin, yytext_len
+ while yytext_len:
+ block = yy_buffer_stack[-1].next
+ while block is None or block.pos >= len(block.text):
+ if block is None:
+ yy_buffer_stack.pop()
+ block = yy_buffer_stack[-1].next
+ yyin = yy_buffer_stack[-1].file_in
+ else:
+ block = block.next
+ yy_buffer_stack[-1].next = block
+ i = min(yytext_len, len(block.text) - block.pos)
+ block.pos += i
+ yytext_len -= i
+ yy_buffer_stack[-1].next = YYBufferBlock(yy_buffer_stack[-1].next, 0, text)
+
+def ECHO():
+ yyout.write(yytext)
+
+def yy_rule_start():
+ global yytext, yytext_len
+ yytext = yy_group_text[:yy_group_stack[-1]]
+ yytext_len = yy_group_stack[-1]
+ del yy_group_stack[-2:]
+ YY_RULE_START()
+
+def yy_group_end():
+ pass
+
+def BEGIN(start):
+ global yystart
+ yystart = start
+
+def YY_START():
+ return yystart
+
+def yy_push_state(start):
+ global yystart
+ yystart_stack.append(yystart)
+ yystart = start
+
+def yy_pop_state():
+ global yystart
+ yystart = yystart_stack.pop()
+
+# GENERATE SECTION2 BEGIN
+def yy_action0():
+ raise YYContinue()
+def yy_action1():
+ global yylval
+ print('yy_groups', yy_groups)
+ print('yy_groups_by_name', yy_groups_by_name)
+
+ # by Python parser:
+ #yylval = float(yytext)
+
+ # by group numbers:
+ #if yy_groups[1] is not None:
+ # mantissa = yy_groups[1]
+ # fraction = ''
+ #else:
+ # mantissa = yy_groups[2]
+ # fraction = yy_groups[3]
+ #yylval = int(mantissa + fraction) * 10. ** -len(fraction)
+
+ # by group names:
+ mantissa = yy_groups_by_name['mantissa']
+ fraction = yy_groups_by_name.get('fraction', '')
+ yylval = int(mantissa + fraction) * 10. ** -len(fraction)
+
+ return NUM
+ raise YYContinue()
+def yy_action2():
+ return ord(yytext[0])
+ raise YYContinue()
+def yy_action3():
+ ECHO()
+ raise YYContinue()
+def yy_rule0():
+ global yy_groups, yy_groups_by_name, yy_action
+ yy_groups = [yy_group_text[:yy_group_stack[-1]]]
+ yy_groups_by_name = {}
+ yy_action = yy_action0
+def yy_rule1():
+ global yy_groups, yy_groups_by_name, yy_action
+ yy_groups = [yy_group_text[:yy_group_stack[-1]], None, None, None]
+ yy_groups_by_name = {}
+ yy_action = yy_action1
+def yy_rule2():
+ global yy_groups, yy_groups_by_name, yy_action
+ yy_groups = [yy_group_text[:yy_group_stack[-1]]]
+ yy_groups_by_name = {}
+ yy_action = yy_action2
+def yy_rule3():
+ global yy_groups, yy_groups_by_name, yy_action
+ yy_groups = [yy_group_text[:yy_group_stack[-1]]]
+ yy_groups_by_name = {}
+ yy_action = yy_action3
+def yy_group0():
+ yy_temp = yy_group_text[yy_group_stack[-1]:yy_group_stack[-2]]
+ yy_groups[1] = yy_temp
+ yy_groups_by_name['mantissa'] = yy_temp
+ del yy_group_stack[-2:]
+def yy_group1():
+ yy_temp = yy_group_text[yy_group_stack[-1]:yy_group_stack[-2]]
+ yy_groups[2] = yy_temp
+ yy_groups_by_name['mantissa'] = yy_temp
+ del yy_group_stack[-2:]
+def yy_group2():
+ yy_temp = yy_group_text[yy_group_stack[-1]:yy_group_stack[-2]]
+ yy_groups[3] = yy_temp
+ yy_groups_by_name['fraction'] = yy_temp
+ del yy_group_stack[-2:]
+yy_dfa_states = [([256], [0], [0]), ([10, 11, 32, 33, 46, 47, 48, 58, 256], [3,
+4, 3, 5, 3, 6, 3, 7, 3], []), ([10, 11, 32, 33, 46, 47, 48, 58, 256], [3, 4, 3,
+5, 3, 8, 3, 9, 3], []), ([48, 58, 256], [10, 11, 10], [1]), ([46, 47, 48, 58,
+256], [12, 13, 12, 14, 12], [4]), ([48, 58, 256], [10, 15, 10], [1]), ([46, 47,
+48, 58, 256], [12, 16, 12, 17, 12], [4]), ([48, 58, 256], [10, 11, 10], [1]), (
+[48, 58, 256], [10, 15, 10], [1])]
+yy_dfa_actions = [(0, []), (1, [(1, 1), (1, 1), (1, 1), (1, 1), (3, 1), (1, 1),
+(2, 1, yy_group0), (3, 1), (2, 1, yy_group1), (1, 1), (3, 1), (2, 1,
+yy_group_end), (3, 1), (1, 1), (3, 1), (3, 1), (3, 1), (1, 1), (1, 1), (1, 1),
+(0, 1), (1, 1), (2, 1, yy_group0), (0, 1), (2, 1, yy_group1), (1, 1), (0, 1), (
+2, 1, yy_group_end), (0, 1), (1, 1), (0, 1), (0, 1), (0, 1)]), (2, [(1, 1), (1,
+1), (1, 1), (1, 1), (3, 1), (1, 1), (2, 1, yy_group0), (3, 1), (2, 1, yy_group1
+), (1, 1), (3, 1), (2, 1, yy_group_end), (3, 1), (1, 1), (3, 1), (3, 1), (3, 1
+), (1, 1), (1, 1), (1, 1), (0, 1), (1, 1), (2, 1, yy_group0), (0, 1), (2, 1,
+yy_group1), (1, 1), (0, 1), (2, 1, yy_group_end), (0, 1), (1, 1), (0, 1), (0, 1
+), (0, 1)]), (0, [(1, 7), (0, 5), (2, 1, yy_rule_start), (2, 1, yy_rule2), (0,
+1), (2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 6), (2, 1, yy_rule_start), (2,
+1, yy_rule2), (3, 1), (2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 1)]), (0, [(
+1, 7), (0, 4), (2, 1, yy_rule_start), (2, 1, yy_rule2), (0, 2), (2, 1,
+yy_rule_start), (2, 1, yy_rule3), (0, 5), (2, 1, yy_rule_start), (2, 1,
+yy_rule2), (3, 1), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 1)]), (
+0, [(1, 7), (2, 1, yy_rule_start), (2, 1, yy_rule0), (0, 5), (2, 1,
+yy_rule_start), (2, 1, yy_rule2), (0, 1), (2, 1, yy_rule_start), (2, 1,
+yy_rule3), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule0), (3, 1), (0, 4), (2,
+1, yy_rule_start), (2, 1, yy_rule2), (0, 1), (2, 1, yy_rule_start), (2, 1,
+yy_rule3), (0, 1)]), (3, [(1, 7), (0, 3), (2, 1, yy_group2), (3, 1), (0, 1), (
+2, 1, yy_rule_start), (2, 1, yy_rule2), (0, 1), (2, 1, yy_rule_start), (2, 1,
+yy_rule3), (0, 4), (2, 1, yy_group2), (0, 2), (2, 1, yy_rule_start), (2, 1,
+yy_rule2), (3, 1), (2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 1)]), (4, [(1,
+7), (0, 1), (1, 1), (3, 1), (2, 1, yy_group_end), (1, 1), (3, 1), (2, 1,
+yy_rule_start), (2, 1, yy_rule1), (0, 1), (1, 1), (3, 1), (2, 1, yy_group_end),
+(3, 1), (0, 2), (2, 1, yy_rule_start), (2, 1, yy_rule2), (0, 1), (2, 1,
+yy_rule_start), (2, 1, yy_rule3), (0, 2), (1, 1), (0, 1), (2, 1, yy_group_end),
+(1, 1), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1), (3, 1), (1, 1), (0, 1
+), (2, 1, yy_group_end), (0, 3), (2, 1, yy_rule_start), (2, 1, yy_rule2), (0, 1
+), (2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 1)]), (5, [(1, 7), (0, 3), (2,
+1, yy_group2), (3, 1), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule2), (0, 1),
+(2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 4), (2, 1, yy_group2), (0, 2), (2,
+1, yy_rule_start), (2, 1, yy_rule2), (3, 1), (2, 1, yy_rule_start), (2, 1,
+yy_rule3), (0, 1)]), (6, [(1, 7), (0, 1), (1, 1), (3, 1), (2, 1, yy_group_end),
+(1, 1), (3, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1), (0, 1), (1, 1), (3, 1
+), (2, 1, yy_group_end), (3, 1), (0, 2), (2, 1, yy_rule_start), (2, 1, yy_rule2
+), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 2), (1, 1), (0, 1), (2,
+1, yy_group_end), (1, 1), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1), (3,
+1), (1, 1), (0, 1), (2, 1, yy_group_end), (0, 3), (2, 1, yy_rule_start), (2, 1,
+yy_rule2), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule3), (0, 1)]), (0, [(1,
+1), (0, 2), (3, 1)]), (7, [(1, 1), (1, 1), (3, 1), (2, 1, yy_group_end), (2, 1,
+yy_rule_start), (2, 1, yy_rule1), (0, 1), (1, 1), (0, 1), (2, 1, yy_group_end),
+(2, 1, yy_rule_start), (2, 1, yy_rule1), (3, 1), (0, 1)]), (0, [(1, 4), (0, 8),
+(3, 1)]), (3, [(1, 4), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1), (0, 2),
+(2, 1, yy_group2), (3, 1), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1), (3,
+1), (0, 1), (2, 1, yy_group2), (0, 2)]), (4, [(1, 4), (1, 1), (3, 1), (2, 1,
+yy_group_end), (1, 1), (3, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1), (0, 2),
+(1, 1), (3, 1), (2, 1, yy_group_end), (3, 1), (0, 1), (1, 1), (0, 1), (2, 1,
+yy_group_end), (1, 1), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1), (3, 1),
+(0, 1), (1, 1), (0, 1), (2, 1, yy_group_end), (0, 3)]), (8, [(1, 1), (1, 1), (
+3, 1), (2, 1, yy_group_end), (2, 1, yy_rule_start), (2, 1, yy_rule1), (0, 1), (
+1, 1), (0, 1), (2, 1, yy_group_end), (2, 1, yy_rule_start), (2, 1, yy_rule1), (
+3, 1), (0, 1)]), (5, [(1, 4), (0, 1), (2, 1, yy_rule_start), (2, 1, yy_rule1),
+(0, 2), (2, 1, yy_group2), (3, 1), (0, 1), (2, 1, yy_rule_start), (2, 1,
+yy_rule1), (3, 1), (0, 1), (2, 1, yy_group2), (0, 2)]), (6, [(1, 4), (1, 1), (
+3, 1), (2, 1, yy_group_end), (1, 1), (3, 1), (2, 1, yy_rule_start), (2, 1,
+yy_rule1), (0, 2), (1, 1), (3, 1), (2, 1, yy_group_end), (3, 1), (0, 1), (1, 1
+), (0, 1), (2, 1, yy_group_end), (1, 1), (0, 1), (2, 1, yy_rule_start), (2, 1,
+yy_rule1), (3, 1), (0, 1), (1, 1), (0, 1), (2, 1, yy_group_end), (0, 3)])]
+yy_dfa_start_action = [1, 2]
+def yy_eof_action0():
+ yyterminate()
+ return 0
+yy_eof_actions = [
+ yy_eof_action0
+]
+# GENERATE END
+
+def yylex():
+ global \
+ yyin, \
+ yy_threads0, \
+ yy_threads1, \
+ yy_prefix_slop, \
+ yy_group_text, \
+ yy_group_stack, \
+ yy_action, \
+ yytext, \
+ yytext_len
+
+ # GENERATE SECTION2INITIAL BEGIN
+ # GENERATE END
+
+ while True:
+ while yytext_len:
+ block = yy_buffer_stack[-1].next
+ while block is None or block.pos >= len(block.text):
+ if block is None:
+ yy_buffer_stack.pop()
+ block = yy_buffer_stack[-1].next
+ yyin = yy_buffer_stack[-1].file_in
+ else:
+ block = block.next
+ yy_buffer_stack[-1].next = block
+ i = min(yytext_len, len(block.text) - block.pos)
+ block.pos += i
+ yytext_len -= i
+
+ match = ''
+ match_len = 0
+
+ del yy_threads0[yy_prefix_slop:]
+ yy_threads0.append(None)
+
+ buffer_ptr = len(yy_buffer_stack) - 1
+ block_prev = yy_buffer_stack[buffer_ptr]
+ block = block_prev.next
+ if block is not None:
+ block_pos = block.pos
+
+ action = yy_dfa_start_action[
+ yystart * 2 + int(yy_buffer_stack[-1].at_bol)
+ ]
+ while action != -1:
+ state, transition = yy_dfa_actions[action]
+ #print('i', i, 'action', action, 'state', state, 'transition', transition)
+
+ i = yy_prefix_slop
+ assert len(yy_threads1) == yy_prefix_slop
+ for trans in transition:
+ if trans[0] == 0: #DFA.TRANSITION_POP:
+ i += trans[1]
+ elif trans[0] == 1: #DFA.TRANSITION_DUP:
+ while i < trans[1]:
+ yy_threads0[:0] = [None] * yy_prefix_slop
+ yy_threads1[:0] = [None] * yy_prefix_slop
+ i += yy_prefix_slop
+ yy_prefix_slop *= 2
+ yy_threads0[i - trans[1]:i] = yy_threads0[i:i + trans[1]]
+ i -= trans[1]
+ elif trans[0] == 2: #DFA.TRANSITION_MARK:
+ yy_threads0[i:i + trans[1]] = [
+ (match_len, trans[2], thread)
+ for thread in yy_threads0[i:i + trans[1]]
+ ]
+ elif trans[0] == 3: #DFA.TRANSITION_MOVE:
+ yy_threads1.extend(yy_threads0[i:i + trans[1]])
+ i += trans[1]
+ #elif trans[0] == DFA.TRANSITION_DEL:
+ # del yy_threads1[-trans[1]:]
+ else:
+ assert False
+ assert i == len(yy_threads0)
+ yy_threads0, yy_threads1 = yy_threads1, yy_threads0
+ del yy_threads1[yy_prefix_slop:]
+
+ if state == 0:
+ # there is only one match, which is complete
+ assert len(yy_threads0) == yy_prefix_slop + 1
+ assert yy_dfa_states[state][2] == [0]
+ break
+
+ yy_buffer_stack[-1].file_in = yyin
+ while block is None or block_pos >= len(block.text):
+ if block is None:
+ file_in = yy_buffer_stack[buffer_ptr].file_in
+ text = '' if file_in is None else file_in.readline()
+ if len(text):
+ block = YYBufferBlock(None, 0, text)
+ block_pos = 0
+ block_prev.next = block
+ else:
+ # do not re-attempt read once EOF is reached
+ yy_buffer_stack[buffer_ptr].file_in = None
+ yyin = yy_buffer_stack[-1].file_in
+ buffer_ptr -= 1
+ if buffer_ptr < 0:
+ break # EOF
+ block_prev = yy_buffer_stack[buffer_ptr]
+ block = block_prev.next
+ if block is not None:
+ block_pos = block.pos
+ else:
+ i = match_len - len(match)
+ if i:
+ match += block.text[block_pos - i:]
+ block_prev = block
+ block = block_prev.next
+ if block is not None:
+ block_pos = block.pos
+ else:
+ #print('block_pos', block_pos, 'block.text', block.text)
+ action = yy_dfa_states[state][1][
+ bisect.bisect_right(
+ yy_dfa_states[state][0],
+ ord(block.text[block_pos])
+ )
+ ]
+ block_pos += 1
+ match_len += 1
+ continue
+ # EOF
+ if i == 0:
+ try:
+ return yy_eof_actions[yystart]()
+ except YYTerminate:
+ return 0
+ break
+
+ i = match_len - len(match)
+ if i:
+ assert block is not None
+ match += block.text[block_pos - i:]
+
+ for i in yy_dfa_states[state][2]:
+ yy_group_text = match
+ yy_group_stack = []
+ yy_groups = None
+ yy_groups_by_name = None
+ yy_action = None
+ yytext = None
+ yytext_len = None
+
+ thread = yy_threads0[yy_prefix_slop + i]
+ #print('thread', thread)
+ while thread is not None:
+ pos, ref_data, thread = thread
+ yy_group_stack.append(pos)
+ ref_data()
+
+ try:
+ return yy_action()
+ except YYReject:
+ pass
+ except YYContinue:
+ break
+ except YYTerminate:
+ return 0
+ else:
+ raise Exception('scanner jammed')
+
+# GENERATE SECTION3 BEGIN
+# GENERATE END
public / pilex.git / commitdiff