from ndcode.piyacc import t_def
from ndcode.piyacc import y_tab
- scanner_cursor = state.boundary()
+ scanner_cursor = state.Boundary()
def YY_USER_ACTION():
state.location_compute(y_tab.yylloc, scanner_cursor, yytext)
gram_last_string = ''
bracketed_id_str = None
- bracketed_id_loc = 0
- bracketed_id_start = 0
+ bracketed_id_loc = None
+ bracketed_id_start = None
bracketed_id_context_state = -1
obstack_for_string = []
# these should be yylex()-local, but moved to here, see further down:
nesting = 0
context_state = -1
- id_loc = state.location()
+ id_loc = None
code_start = None
token_start = None
- scanner_cursor = state.boundary('<stdin>', 0, 0)
+ scanner_cursor = state.Boundary('<stdin>', 1, 1)
%}
%x SC_YACC_COMMENT
# from an action since the action functions are not nested to yylex():
#nesting = 0
#context_state = 0
- #id_loc = state.location()
+ #id_loc = None
#code_start = scanner_cursor.copy()
#token_start = scanner_cursor.copy()
#first = True
"//".* #continue
"/*" {
global token_start, context_state
- token_start = y_tab.yylloc.start
+ token_start = y_tab.yylloc.start.copy()
context_state = YY_START()
BEGIN(SC_YACC_COMMENT)
}
(?E{t_def.AST.ID}{id}) {
global id_loc, bracketed_id_str
y_tab.yylval = yytext
- id_loc = y_tab.yylloc
+ id_loc = y_tab.yylloc.copy()
bracketed_id_str = None
markup_push(element.Element)
BEGIN(SC_AFTER_IDENTIFIER)
/* Characters. */
"'" {
global token_start
- token_start = y_tab.yylloc.start
+ token_start = y_tab.yylloc.start.copy()
BEGIN(SC_ESCAPED_CHARACTER)
markup_push(element.Element)
markup_push(t_def.AST.Char)
/* Strings. */
"\"" {
global token_start
- token_start = y_tab.yylloc.start
+ token_start = y_tab.yylloc.start.copy()
BEGIN(SC_ESCAPED_STRING)
markup_push(element.Element)
markup_push(t_def.AST.String)
/* Prologue. */
"%{" {
global code_start
- code_start = y_tab.yylloc.start
+ code_start = y_tab.yylloc.start.copy()
BEGIN(SC_PROLOGUE)
markup_push(element.Element)
markup_push(t_def.AST.Section1.Prologue)
global nesting, code_start
obstack_for_string.append(yytext)
nesting = 0
- code_start = y_tab.yylloc.start
+ code_start = y_tab.yylloc.start.copy()
BEGIN(SC_BRACED_CODE)
markup_push(element.Element)
# new way, includes braces, wrapped by <AST_Production_Action> later
"%?"[ \f\n\t\v]*"{" {
global nesting, code_start
nesting = 0
- code_start = y_tab.yylloc.start
+ code_start = y_tab.yylloc.start.copy()
BEGIN(SC_PREDICATE)
markup_push(element.Element)
markup_push(t_def.AST.BracedPredicate)
global nesting, code_start
obstack_for_string.append(yytext)
nesting = 0
- code_start = y_tab.yylloc.start
+ code_start = y_tab.yylloc.start.copy()
BEGIN(SC_ELEMENT_GROUP)
markup_push(element.Element)
markup_flush(len(yytext))
"<" {
global nesting, token_start
nesting = 0
- token_start = y_tab.yylloc.start
+ token_start = y_tab.yylloc.start.copy()
BEGIN(SC_TAG)
markup_push(element.Element)
markup_push(t_def.AST.TagRef)
"[" {
global bracketed_id_str, bracketed_id_start, bracketed_id_context_state
bracketed_id_str = None
- bracketed_id_start = y_tab.yylloc.start
+ bracketed_id_start = y_tab.yylloc.start.copy()
bracketed_id_context_state = YY_START()
BEGIN(SC_BRACKETED_ID)
}
{
"[" {
global bracketed_id_start, bracketed_id_context_state
- if bracketed_id_str is not None:
- scanner_cursor.column -= len(yytext)
- markup_yyless(0)
- markup_pop_token() # element.Element
- BEGIN(SC_RETURN_BRACKETED_ID)
- y_tab.yylloc = id_loc
- return y_tab.ID
- else:
- markup_pop_token() # element.Element
- bracketed_id_start = y_tab.yylloc.start
- bracketed_id_context_state = YY_START()
- BEGIN(SC_BRACKETED_ID)
+ # at this point bracketed_id_str has to be None
+ #if bracketed_id_str is not None:
+ # scanner_cursor.column -= len(yytext)
+ # markup_yyless(0)
+ # markup_pop_token() # element.Element
+ # BEGIN(SC_RETURN_BRACKETED_ID)
+ # y_tab.yylloc = id_loc.copy()
+ # return y_tab.ID
+ #else:
+ # markup_pop_token() # element.Element
+ # bracketed_id_start = y_tab.yylloc.start.copy()
+ # bracketed_id_context_state = YY_START()
+ # BEGIN(SC_BRACKETED_ID)
+ assert bracketed_id_str is None
+ markup_pop_token() # element.Element
+ bracketed_id_start = y_tab.yylloc.start.copy()
+ bracketed_id_context_state = YY_START()
+ BEGIN(SC_BRACKETED_ID)
}
":" {
markup_pop_token() # element.Element
- BEGIN(SC_RETURN_BRACKETED_ID if bracketed_id_str else INITIAL)
- y_tab.yylloc = id_loc
+ # at this point bracketed_id_str has to be None
+ #BEGIN(SC_RETURN_BRACKETED_ID if bracketed_id_str is not None else INITIAL)
+ assert bracketed_id_str is None
+ BEGIN(INITIAL)
+ y_tab.yylloc = id_loc.copy()
markup_flush(len(yytext))
return y_tab.ID_COLON
}
. {
+ global scanner_cursor
+
scanner_cursor.column -= len(yytext)
markup_yyless(0)
- # total kludge: put back all whitespace/comments after the ID, and rescan
- # (this will mess up the position tracking, need to revisit and fix later)
+ # put back all whitespace/comments after the ID, it will be rescanned
assert len(yy_element_space.text) == len(yy_element_space.children) + 1
unput(yy_element_space.text[-1])
yy_element_space.text[-1] = ''
+ scanner_cursor = id_loc.end.copy()
markup_pop_token() # element.Element
- BEGIN(SC_RETURN_BRACKETED_ID if bracketed_id_str else INITIAL)
- y_tab.yylloc = id_loc
+ # at this point bracketed_id_str has to be None
+ #BEGIN(SC_RETURN_BRACKETED_ID if bracketed_id_str is not None else INITIAL)
+ assert bracketed_id_str is None
+ BEGIN(INITIAL)
+ y_tab.yylloc = id_loc.copy()
return y_tab.ID
}
<<EOF>> {
+ global scanner_cursor
+
+ # put back all whitespace/comments after the ID, it will be rescanned
+ assert len(yy_element_space.text) == len(yy_element_space.children) + 1
+ unput(yy_element_space.text[-1])
+ yy_element_space.text[-1] = ''
+ scanner_cursor = id_loc.end.copy()
+
markup_pop_token() # element.Element
- BEGIN(SC_RETURN_BRACKETED_ID if bracketed_id_str else INITIAL)
- y_tab.yylloc = id_loc
+ # at this point bracketed_id_str has to be None
+ #BEGIN(SC_RETURN_BRACKETED_ID if bracketed_id_str is not None else INITIAL)
+ assert bracketed_id_str is None
+ BEGIN(INITIAL)
+ y_tab.yylloc = id_loc.copy()
return y_tab.ID
}
}
state.complain(y_tab.yylloc, state.complaint, 'unexpected identifier in bracketed name: {0:s}'.format(state.quote(yytext)))
else:
bracketed_id_str = yytext
- bracketed_id_loc = y_tab.yylloc
+ bracketed_id_loc = y_tab.yylloc.copy()
}
"]" {
global bracketed_id_str
if INITIAL == bracketed_id_context_state:
y_tab.yylval = bracketed_id_str
bracketed_id_str = None
- y_tab.yylloc = bracketed_id_loc
+ y_tab.yylloc = bracketed_id_loc.copy()
return y_tab.BRACKETED_ID
else:
state.complain(y_tab.yylloc, state.complaint, 'an identifier expected')
markup_yyless(0)
y_tab.yylval = bracketed_id_str
bracketed_id_str = None
- y_tab.yylloc = bracketed_id_loc
+ y_tab.yylloc = bracketed_id_loc.copy()
BEGIN(INITIAL)
return y_tab.BRACKETED_ID
}
global gram_last_string
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = token_start
+ y_tab.yylloc.start = token_start.copy()
y_tab.yylval = gram_last_string
BEGIN(INITIAL)
markup_pop() # t_def.AST.Text
global gram_last_string
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = token_start
+ y_tab.yylloc.start = token_start.copy()
if len(gram_last_string) == 0:
state.complain(y_tab.yylloc, state.Wother, 'empty character literal')
y_tab.yylval = ord('\'')
if nesting < 0:
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = token_start
+ y_tab.yylloc.start = token_start.copy()
y_tab.yylval = gram_last_string
#del obstack_for_string[:]
BEGIN(INITIAL)
global context_state, token_start
obstack_for_string.append(yytext)
context_state = YY_START()
- token_start = y_tab.yylloc.start
+ token_start = y_tab.yylloc.start.copy()
BEGIN(SC_CHARACTER)
}
"\"" {
global context_state, token_start
obstack_for_string.append(yytext)
context_state = YY_START()
- token_start = y_tab.yylloc.start
+ token_start = y_tab.yylloc.start.copy()
BEGIN(SC_STRING)
}
"/"{splice}"*" {
global context_state, token_start
obstack_for_string.append(yytext)
context_state = YY_START()
- token_start = y_tab.yylloc.start
+ token_start = y_tab.yylloc.start.copy()
BEGIN(SC_COMMENT)
}
"/"{splice}"/" {
if nesting < 0:
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = code_start
+ y_tab.yylloc.start = code_start.copy()
y_tab.yylval = gram_last_string
BEGIN(INITIAL)
# new way, includes braces, wrapped by <AST_Production_Action> later
if nesting < 0:
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = code_start
+ y_tab.yylloc.start = code_start.copy()
y_tab.yylval = gram_last_string
BEGIN(INITIAL)
markup_pop() # t_def.AST.Text
if nesting < 0:
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = code_start
+ y_tab.yylloc.start = code_start.copy()
y_tab.yylval = gram_last_string
#del obstack_for_string[:]
BEGIN(INITIAL)
global gram_last_string
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = code_start
+ y_tab.yylloc.start = code_start.copy()
y_tab.yylval = gram_last_string
BEGIN(INITIAL)
markup_pop() # t_def.AST.Text
global gram_last_string
gram_last_string = ''.join(obstack_for_string)
del obstack_for_string[:] # not strictly correct
- y_tab.yylloc.start = code_start
+ y_tab.yylloc.start = code_start.copy()
y_tab.yylval = gram_last_string
BEGIN(INITIAL)
return y_tab.EPILOGUE
# boundary_set(&scanner_cursor, current_file, lineno, 1)
def unexpected_end(start, msg, token_end):
- loc = state.location(start.copy(), scanner_cursor.copy())
+ loc = state.Location(start.copy(), scanner_cursor.copy())
scanner_cursor.column -= len(token_end)
unput(token_end)
token_end = state.quote(token_end)
# recognized by several iterations of yylex(), it would be better to
# try to use more complex regular expressions to match all in one go:
+# longer string to return to user is built up in yy_element_space, by
+# moving each piece matched from yy_element_token to yy_element_space
+
+# move first n characters matched in yy_element_token to yy_element_space
def markup_flush(n):
assert len(yy_element_space.text) == len(yy_element_space.children) + 1
assert len(yy_element_token.text) == len(yy_element_token.children) + 1
yy_element_space.text[-1] += yy_element_token.text[0][:n]
yy_element_token.text[0] = yy_element_token.text[0][n:]
+# put back last n characters matched in yy_element_token onto the input
def markup_yyless(n):
assert len(yy_element_space.text) == len(yy_element_space.children) + 1
assert len(yy_element_token.text) == len(yy_element_token.children) + 1
yyless(n)
yy_element_token.text[0] = yy_element_token.text[0][:n]
+# mark the current position in yy_element_space for insertion of markup
def markup_push(factory, *args, **kwargs):
global yy_element_space
assert len(yy_element_space.text) == len(yy_element_space.children) + 1
markup_stack.append(yy_element_space)
yy_element_space = factory(*args, **kwargs)
+# insert markup in yy_element_space between marked and current position
def markup_pop():
global yy_element_space
assert len(yy_element_space.text) == len(yy_element_space.children) + 1
yy_element_space.children.append(_element)
yy_element_space.text.append('')
+# similar to markup_flush() followed by markup_pop(), but leaves the
+# marked-up item in yy_element_token to be returned to yylex() caller
+# note: requires previous push call to be markup_push(element.Element)
def markup_pop_token():
global yy_element_space, yy_element_token
public / piyacc.git / commitdiff