1 /* dfa - DFA construction routines */
4 * Copyright (c) 1990 The Regents of the University of California.
7 * This code is derived from software contributed to Berkeley by
10 * The United States Government has rights in this work pursuant
11 * to contract no. DE-AC03-76SF00098 between the United States
12 * Department of Energy and the University of California.
14 * Redistribution and use in source and binary forms are permitted provided
15 * that: (1) source distributions retain this entire copyright notice and
16 * comment, and (2) distributions including binaries display the following
17 * acknowledgement: ``This product includes software developed by the
18 * University of California, Berkeley and its contributors'' in the
19 * documentation or other materials provided with the distribution and in
20 * all advertising materials mentioning features or use of this software.
21 * Neither the name of the University nor the names of its contributors may
22 * be used to endorse or promote products derived from this software without
23 * specific prior written permission.
24 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
25 * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
26 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
31 "@(#) $Id: dfa.c,v 1.2 1994/06/24 10:56:41 ceriel Exp $ (LBL)";
37 /* declare functions that have forward references */
39 void dump_associated_rules PROTO((FILE*, int));
40 void dump_transitions PROTO((FILE*, int[]));
41 void sympartition PROTO((int[], int, int[], int[]));
42 int symfollowset PROTO((int[], int, int, int[]));
45 /* check_for_backtracking - check a DFA state for backtracking
48 * int ds, state[numecs];
49 * check_for_backtracking( ds, state );
51 * ds is the number of the state to check and state[] is its out-transitions,
52 * indexed by equivalence class, and state_rules[] is the set of rules
53 * associated with this state
56 void check_for_backtracking( ds, state )
61 if ( (reject && ! dfaacc[ds].dfaacc_set) || ! dfaacc[ds].dfaacc_state )
62 { /* state is non-accepting */
65 if ( backtrack_report )
67 fprintf( backtrack_file, "State #%d is non-accepting -\n", ds );
69 /* identify the state */
70 dump_associated_rules( backtrack_file, ds );
72 /* now identify it further using the out- and jam-transitions */
73 dump_transitions( backtrack_file, state );
75 putc( '\n', backtrack_file );
81 /* check_trailing_context - check to see if NFA state set constitutes
82 * "dangerous" trailing context
85 * int nfa_states[num_states+1], num_states;
86 * int accset[nacc+1], nacc;
87 * check_trailing_context( nfa_states, num_states, accset, nacc );
90 * Trailing context is "dangerous" if both the head and the trailing
91 * part are of variable size \and/ there's a DFA state which contains
92 * both an accepting state for the head part of the rule and NFA states
93 * which occur after the beginning of the trailing context.
94 * When such a rule is matched, it's impossible to tell if having been
95 * in the DFA state indicates the beginning of the trailing context
96 * or further-along scanning of the pattern. In these cases, a warning
99 * nfa_states[1 .. num_states] is the list of NFA states in the DFA.
100 * accset[1 .. nacc] is the list of accepting numbers for the DFA state.
103 void check_trailing_context( nfa_states, num_states, accset, nacc )
104 int *nfa_states, num_states;
111 for ( i = 1; i <= num_states; ++i )
113 int ns = nfa_states[i];
114 register int type = state_type[ns];
115 register int ar = assoc_rule[ns];
117 if ( type == STATE_NORMAL || rule_type[ar] != RULE_VARIABLE )
121 else if ( type == STATE_TRAILING_CONTEXT )
123 /* potential trouble. Scan set of accepting numbers for
124 * the one marking the end of the "head". We assume that
125 * this looping will be fairly cheap since it's rare that
126 * an accepting number set is large.
128 for ( j = 1; j <= nacc; ++j )
129 if ( accset[j] & YY_TRAILING_HEAD_MASK )
132 "%s: Dangerous trailing context in rule at line %d\n",
133 program_name, rule_linenum[ar] );
141 /* dump_associated_rules - list the rules associated with a DFA state
146 * dump_associated_rules( file, ds );
148 * goes through the set of NFA states associated with the DFA and
149 * extracts the first MAX_ASSOC_RULES unique rules, sorts them,
150 * and writes a report to the given file
153 void dump_associated_rules( file, ds )
159 register int num_associated_rules = 0;
160 int rule_set[MAX_ASSOC_RULES + 1];
162 int size = dfasiz[ds];
164 for ( i = 1; i <= size; ++i )
166 register rule_num = rule_linenum[assoc_rule[dset[i]]];
168 for ( j = 1; j <= num_associated_rules; ++j )
169 if ( rule_num == rule_set[j] )
172 if ( j > num_associated_rules )
174 if ( num_associated_rules < MAX_ASSOC_RULES )
175 rule_set[++num_associated_rules] = rule_num;
179 bubble( rule_set, num_associated_rules );
181 fprintf( file, " associated rule line numbers:" );
183 for ( i = 1; i <= num_associated_rules; ++i )
188 fprintf( file, "\t%d", rule_set[i] );
195 /* dump_transitions - list the transitions associated with a DFA state
200 * dump_transitions( file, state );
202 * goes through the set of out-transitions and lists them in human-readable
203 * form (i.e., not as equivalence classes); also lists jam transitions
204 * (i.e., all those which are not out-transitions, plus EOF). The dump
205 * is done to the given file.
208 void dump_transitions( file, state )
214 int out_char_set[CSIZE];
216 for ( i = 0; i < csize; ++i )
218 ec = abs( ecgroup[i] );
219 out_char_set[i] = state[ec];
222 fprintf( file, " out-transitions: " );
224 list_character_set( file, out_char_set );
226 /* now invert the members of the set to get the jam transitions */
227 for ( i = 0; i < csize; ++i )
228 out_char_set[i] = ! out_char_set[i];
230 fprintf( file, "\n jam-transitions: EOF " );
232 list_character_set( file, out_char_set );
238 /* epsclosure - construct the epsilon closure of a set of ndfa states
241 * int t[current_max_dfa_size], numstates, accset[num_rules + 1], nacc;
244 * t = epsclosure( t, &numstates, accset, &nacc, &hashval );
247 * the epsilon closure is the set of all states reachable by an arbitrary
248 * number of epsilon transitions which themselves do not have epsilon
249 * transitions going out, unioned with the set of states which have non-null
250 * accepting numbers. t is an array of size numstates of nfa state numbers.
251 * Upon return, t holds the epsilon closure and numstates is updated. accset
252 * holds a list of the accepting numbers, and the size of accset is given
253 * by nacc. t may be subjected to reallocation if it is not large enough
254 * to hold the epsilon closure.
256 * hashval is the hash value for the dfa corresponding to the state set
259 int *epsclosure( t, ns_addr, accset, nacc_addr, hv_addr )
260 int *t, *ns_addr, accset[], *nacc_addr, *hv_addr;
263 register int stkpos, ns, tsp;
264 int numstates = *ns_addr, nacc, hashval, transsym, nfaccnum;
266 static int did_stk_init = false, *stk;
268 #define MARK_STATE(state) \
269 trans1[state] = trans1[state] - MARKER_DIFFERENCE;
271 #define IS_MARKED(state) (trans1[state] < 0)
273 #define UNMARK_STATE(state) \
274 trans1[state] = trans1[state] + MARKER_DIFFERENCE;
276 #define CHECK_ACCEPT(state) \
278 nfaccnum = accptnum[state]; \
279 if ( nfaccnum != NIL ) \
280 accset[++nacc] = nfaccnum; \
283 #define DO_REALLOCATION \
285 current_max_dfa_size += MAX_DFA_SIZE_INCREMENT; \
287 t = reallocate_integer_array( t, current_max_dfa_size ); \
288 stk = reallocate_integer_array( stk, current_max_dfa_size ); \
291 #define PUT_ON_STACK(state) \
293 if ( ++stkend >= current_max_dfa_size ) \
295 stk[stkend] = state; \
299 #define ADD_STATE(state) \
301 if ( ++numstates >= current_max_dfa_size ) \
303 t[numstates] = state; \
304 hashval = hashval + state; \
307 #define STACK_STATE(state) \
309 PUT_ON_STACK(state) \
310 CHECK_ACCEPT(state) \
311 if ( nfaccnum != NIL || transchar[state] != SYM_EPSILON ) \
315 if ( ! did_stk_init )
317 stk = allocate_integer_array( current_max_dfa_size );
321 nacc = stkend = hashval = 0;
323 for ( nstate = 1; nstate <= numstates; ++nstate )
327 /* the state could be marked if we've already pushed it onto
330 if ( ! IS_MARKED(ns) )
334 hashval = hashval + ns;
337 for ( stkpos = 1; stkpos <= stkend; ++stkpos )
340 transsym = transchar[ns];
342 if ( transsym == SYM_EPSILON )
344 tsp = trans1[ns] + MARKER_DIFFERENCE;
346 if ( tsp != NO_TRANSITION )
348 if ( ! IS_MARKED(tsp) )
353 if ( tsp != NO_TRANSITION )
354 if ( ! IS_MARKED(tsp) )
360 /* clear out "visit" markers */
362 for ( stkpos = 1; stkpos <= stkend; ++stkpos )
364 if ( IS_MARKED(stk[stkpos]) )
366 UNMARK_STATE(stk[stkpos])
369 flexfatal( "consistency check failed in epsclosure()" );
372 *ns_addr = numstates;
380 /* increase_max_dfas - increase the maximum number of DFAs */
382 void increase_max_dfas()
385 current_max_dfas += MAX_DFAS_INCREMENT;
389 base = reallocate_integer_array( base, current_max_dfas );
390 def = reallocate_integer_array( def, current_max_dfas );
391 dfasiz = reallocate_integer_array( dfasiz, current_max_dfas );
392 accsiz = reallocate_integer_array( accsiz, current_max_dfas );
393 dhash = reallocate_integer_array( dhash, current_max_dfas );
394 dss = reallocate_int_ptr_array( dss, current_max_dfas );
395 dfaacc = reallocate_dfaacc_union( dfaacc, current_max_dfas );
398 nultrans = reallocate_integer_array( nultrans, current_max_dfas );
402 /* ntod - convert an ndfa to a dfa
407 * creates the dfa corresponding to the ndfa we've constructed. the
408 * dfa starts out in state #1.
414 int *accset, ds, nacc, newds;
415 int sym, hashval, numstates, dsize;
416 int num_full_table_rows; /* used only for -f */
418 int targptr, totaltrans, i, comstate, comfreq, targ;
419 int *epsclosure(), snstods(), symlist[CSIZE + 1];
420 int num_start_states;
421 int todo_head, todo_next;
423 /* note that the following are indexed by *equivalence classes*
424 * and not by characters. Since equivalence classes are indexed
425 * beginning with 1, even if the scanner accepts NUL's, this
426 * means that (since every character is potentially in its own
427 * equivalence class) these arrays must have room for indices
428 * from 1 to CSIZE, so their size must be CSIZE + 1.
430 int duplist[CSIZE + 1], state[CSIZE + 1];
431 int targfreq[CSIZE + 1], targstate[CSIZE + 1];
433 /* this is so find_table_space(...) will know where to start looking in
434 * chk/nxt for unused records for space to put in the state
439 accset = allocate_integer_array( num_rules + 1 );
440 nset = allocate_integer_array( current_max_dfa_size );
442 /* the "todo" queue is represented by the head, which is the DFA
443 * state currently being processed, and the "next", which is the
444 * next DFA state number available (not in use). We depend on the
445 * fact that snstods() returns DFA's \in increasing order/, and thus
446 * need only know the bounds of the dfas to be processed.
448 todo_head = todo_next = 0;
450 for ( i = 0; i <= csize; ++i )
456 for ( i = 0; i <= num_rules; ++i )
462 fputs( "\n\nDFA Dump:\n\n", stderr );
467 /* check to see whether we should build a separate table for transitions
468 * on NUL characters. We don't do this for full-speed (-F) scanners,
469 * since for them we don't have a simple state number lying around with
470 * which to index the table. We also don't bother doing it for scanners
471 * unless (1) NUL is in its own equivalence class (indicated by a
472 * positive value of ecgroup[NUL]), (2) NUL's equilvalence class is
473 * the last equivalence class, and (3) the number of equivalence classes
474 * is the same as the number of characters. This latter case comes about
475 * when useecs is false or when its true but every character still
476 * manages to land in its own class (unlikely, but it's cheap to check
477 * for). If all these things are true then the character code needed
478 * to represent NUL's equivalence class for indexing the tables is
479 * going to take one more bit than the number of characters, and therefore
480 * we won't be assured of being able to fit it into a YY_CHAR variable.
481 * This rules out storing the transitions in a compressed table, since
482 * the code for interpreting them uses a YY_CHAR variable (perhaps it
483 * should just use an integer, though; this is worth pondering ... ###).
485 * Finally, for full tables, we want the number of entries in the
486 * table to be a power of two so the array references go fast (it
487 * will just take a shift to compute the major index). If encoding
488 * NUL's transitions in the table will spoil this, we give it its
489 * own table (note that this will be the case if we're not using
490 * equivalence classes).
493 /* note that the test for ecgroup[0] == numecs below accomplishes
494 * both (1) and (2) above
496 if ( ! fullspd && ecgroup[0] == numecs )
497 { /* NUL is alone in its equivalence class, which is the last one */
498 int use_NUL_table = (numecs == csize);
500 if ( fulltbl && ! use_NUL_table )
501 { /* we still may want to use the table if numecs is a power of 2 */
504 for ( power_of_two = 1; power_of_two <= csize; power_of_two *= 2 )
505 if ( numecs == power_of_two )
507 use_NUL_table = true;
513 nultrans = allocate_integer_array( current_max_dfas );
514 /* from now on, nultrans != nil indicates that we're
515 * saving null transitions for later, separate encoding
522 for ( i = 0; i <= numecs; ++i )
524 place_state( state, 0, 0 );
530 /* we won't be including NUL's transitions in the table,
531 * so build it for entries from 0 .. numecs - 1
533 num_full_table_rows = numecs;
536 /* take into account the fact that we'll be including
537 * the NUL entries in the transition table. Build it
540 num_full_table_rows = numecs + 1;
542 /* declare it "short" because it's a real long-shot that that
543 * won't be large enough.
545 printf( "static short int yy_nxt[][%d] =\n {\n",
546 /* '}' so vi doesn't get too confused */
547 num_full_table_rows );
549 /* generate 0 entries for state #0 */
550 for ( i = 0; i < num_full_table_rows; ++i )
553 /* force ',' and dataflush() next call to mk2data */
554 datapos = NUMDATAITEMS;
556 /* force extra blank line next dataflush() */
557 dataline = NUMDATALINES;
560 /* create the first states */
562 num_start_states = lastsc * 2;
564 for ( i = 1; i <= num_start_states; ++i )
568 /* for each start condition, make one state for the case when
569 * we're at the beginning of the line (the '%' operator) and
570 * one for the case when we're not
573 nset[numstates] = scset[(i / 2) + 1];
575 nset[numstates] = mkbranch( scbol[i / 2], scset[i / 2] );
577 nset = epsclosure( nset, &numstates, accset, &nacc, &hashval );
579 if ( snstods( nset, numstates, accset, nacc, hashval, &ds ) )
585 if ( variable_trailing_context_rules && nacc > 0 )
586 check_trailing_context( nset, numstates, accset, nacc );
592 if ( ! snstods( nset, 0, accset, 0, 0, &end_of_buffer_state ) )
593 flexfatal( "could not create unique end-of-buffer state" );
600 while ( todo_head < todo_next )
605 for ( i = 1; i <= numecs; ++i )
614 fprintf( stderr, "state # %d:\n", ds );
616 sympartition( dset, dsize, symlist, duplist );
618 for ( sym = 1; sym <= numecs; ++sym )
624 if ( duplist[sym] == NIL )
625 { /* symbol has unique out-transitions */
626 numstates = symfollowset( dset, dsize, sym, nset );
627 nset = epsclosure( nset, &numstates, accset,
630 if ( snstods( nset, numstates, accset,
631 nacc, hashval, &newds ) )
633 totnst = totnst + numstates;
637 if ( variable_trailing_context_rules && nacc > 0 )
638 check_trailing_context( nset, numstates,
645 fprintf( stderr, "\t%d\t%d\n", sym, newds );
647 targfreq[++targptr] = 1;
648 targstate[targptr] = newds;
654 /* sym's equivalence class has the same transitions
655 * as duplist(sym)'s equivalence class
657 targ = state[duplist[sym]];
661 fprintf( stderr, "\t%d\t%d\n", sym, targ );
663 /* update frequency count for destination state */
666 while ( targstate[++i] != targ )
678 numsnpairs = numsnpairs + totaltrans;
680 if ( caseins && ! useecs )
684 for ( i = 'A', j = 'a'; i <= 'Z'; ++i, ++j )
688 if ( ds > num_start_states )
689 check_for_backtracking( ds, state );
693 nultrans[ds] = state[NUL_ec];
694 state[NUL_ec] = 0; /* remove transition */
699 /* supply array's 0-element */
700 if ( ds == end_of_buffer_state )
701 mk2data( -end_of_buffer_state );
703 mk2data( end_of_buffer_state );
705 for ( i = 1; i < num_full_table_rows; ++i )
706 /* jams are marked by negative of state number */
707 mk2data( state[i] ? state[i] : -ds );
709 /* force ',' and dataflush() next call to mk2data */
710 datapos = NUMDATAITEMS;
712 /* force extra blank line next dataflush() */
713 dataline = NUMDATALINES;
717 place_state( state, ds, totaltrans );
719 else if ( ds == end_of_buffer_state )
720 /* special case this state to make sure it does what it's
721 * supposed to, i.e., jam on end-of-buffer
723 stack1( ds, 0, 0, JAMSTATE );
725 else /* normal, compressed state */
727 /* determine which destination state is the most common, and
728 * how many transitions to it there are
734 for ( i = 1; i <= targptr; ++i )
735 if ( targfreq[i] > comfreq )
737 comfreq = targfreq[i];
738 comstate = targstate[i];
741 bldtbl( state, ds, totaltrans, comstate, comfreq );
748 else if ( ! fullspd )
750 cmptmps(); /* create compressed template entries */
752 /* create tables for all the states with only one out-transition */
755 mk1tbl( onestate[onesp], onesym[onesp], onenext[onesp],
765 /* snstods - converts a set of ndfa states into a dfa state
768 * int sns[numstates], numstates, newds, accset[num_rules + 1], nacc, hashval;
770 * is_new_state = snstods( sns, numstates, accset, nacc, hashval, &newds );
772 * on return, the dfa state number is in newds.
775 int snstods( sns, numstates, accset, nacc, hashval, newds_addr )
776 int sns[], numstates, accset[], nacc, hashval, *newds_addr;
783 for ( i = 1; i <= lastdfa; ++i )
784 if ( hashval == dhash[i] )
786 if ( numstates == dfasiz[i] )
792 /* we sort the states in sns so we can compare it to
793 * oldsns quickly. we use bubble because there probably
794 * aren't very many states
796 bubble( sns, numstates );
800 for ( j = 1; j <= numstates; ++j )
801 if ( sns[j] != oldsns[j] )
820 if ( ++lastdfa >= current_max_dfas )
825 dss[newds] = (int *) malloc( (unsigned) ((numstates + 1) * sizeof( int )) );
828 flexfatal( "dynamic memory failure in snstods()" );
830 /* if we haven't already sorted the states in sns, we do so now, so that
831 * future comparisons with it can be made quickly
835 bubble( sns, numstates );
837 for ( i = 1; i <= numstates; ++i )
838 dss[newds][i] = sns[i];
840 dfasiz[newds] = numstates;
841 dhash[newds] = hashval;
846 dfaacc[newds].dfaacc_set = (int *) 0;
848 dfaacc[newds].dfaacc_state = 0;
855 /* we sort the accepting set in increasing order so the disambiguating
856 * rule that the first rule listed is considered match in the event of
857 * ties will work. We use a bubble sort since the list is probably
861 bubble( accset, nacc );
863 dfaacc[newds].dfaacc_set =
864 (int *) malloc( (unsigned) ((nacc + 1) * sizeof( int )) );
866 if ( ! dfaacc[newds].dfaacc_set )
867 flexfatal( "dynamic memory failure in snstods()" );
869 /* save the accepting set for later */
870 for ( i = 1; i <= nacc; ++i )
871 dfaacc[newds].dfaacc_set[i] = accset[i];
873 accsiz[newds] = nacc;
877 { /* find lowest numbered rule so the disambiguating rule will work */
880 for ( i = 1; i <= nacc; ++i )
884 dfaacc[newds].dfaacc_state = j;
893 /* symfollowset - follow the symbol transitions one step
896 * int ds[current_max_dfa_size], dsize, transsym;
897 * int nset[current_max_dfa_size], numstates;
898 * numstates = symfollowset( ds, dsize, transsym, nset );
901 int symfollowset( ds, dsize, transsym, nset )
902 int ds[], dsize, transsym, nset[];
905 int ns, tsp, sym, i, j, lenccl, ch, numstates;
910 for ( i = 1; i <= dsize; ++i )
911 { /* for each nfa state ns in the state set of ds */
917 { /* it's a character class */
919 ccllist = cclmap[sym];
920 lenccl = ccllen[sym];
924 for ( j = 0; j < lenccl; ++j )
925 { /* loop through negated character class */
926 ch = ccltbl[ccllist + j];
932 break; /* transsym isn't in negated ccl */
934 else if ( ch == transsym )
935 /* next 2 */ goto bottom;
938 /* didn't find transsym in ccl */
939 nset[++numstates] = tsp;
943 for ( j = 0; j < lenccl; ++j )
945 ch = ccltbl[ccllist + j];
953 else if ( ch == transsym )
955 nset[++numstates] = tsp;
961 else if ( sym >= 'A' && sym <= 'Z' && caseins )
962 flexfatal( "consistency check failed in symfollowset" );
964 else if ( sym == SYM_EPSILON )
968 else if ( abs( ecgroup[sym] ) == transsym )
969 nset[++numstates] = tsp;
975 return ( numstates );
979 /* sympartition - partition characters with same out-transitions
982 * integer ds[current_max_dfa_size], numstates, duplist[numecs];
984 * sympartition( ds, numstates, symlist, duplist );
987 void sympartition( ds, numstates, symlist, duplist )
988 int ds[], numstates, duplist[];
992 int tch, i, j, k, ns, dupfwd[CSIZE + 1], lenccl, cclp, ich;
994 /* partitioning is done by creating equivalence classes for those
995 * characters which have out-transitions from the given state. Thus
996 * we are really creating equivalence classes of equivalence classes.
999 for ( i = 1; i <= numecs; ++i )
1000 { /* initialize equivalence class list */
1006 dupfwd[numecs] = NIL;
1008 for ( i = 1; i <= numstates; ++i )
1011 tch = transchar[ns];
1013 if ( tch != SYM_EPSILON )
1015 if ( tch < -lastccl || tch > csize )
1017 if ( tch > csize && tch <= CSIZE )
1018 flexerror( "scanner requires -8 flag" );
1022 "bad transition character detected in sympartition()" );
1026 { /* character transition */
1027 /* abs() needed for fake %t ec's */
1028 int ec = abs( ecgroup[tch] );
1030 mkechar( ec, dupfwd, duplist );
1035 { /* character class */
1038 lenccl = ccllen[tch];
1040 mkeccl( ccltbl + cclp, lenccl, dupfwd, duplist, numecs,
1047 for ( k = 0; k < lenccl; ++k )
1049 ich = ccltbl[cclp + k];
1054 for ( ++j; j < ich; ++j )
1058 for ( ++j; j <= numecs; ++j )
1063 for ( k = 0; k < lenccl; ++k )
1065 ich = ccltbl[cclp + k];