def next_token
return [false, false] if @src.eos?
if ret = @src.scan(EM_OPEN_RE)
@pre << ret
[:EM_OPEN, ret]
elsif ret = @src.scan(EM_CLOSE_RE)
@pre << ret
[:EM_CLOSE, ret]
elsif ret = @src.scan(CODE_OPEN_RE)
@pre << ret
[:CODE_OPEN, ret]
elsif ret = @src.scan(CODE_CLOSE_RE)
@pre << ret
[:CODE_CLOSE, ret]
elsif ret = @src.scan(VAR_OPEN_RE)
@pre << ret
[:VAR_OPEN, ret]
elsif ret = @src.scan(VAR_CLOSE_RE)
@pre << ret
[:VAR_CLOSE, ret]
elsif ret = @src.scan(KBD_OPEN_RE)
@pre << ret
[:KBD_OPEN, ret]
elsif ret = @src.scan(KBD_CLOSE_RE)
@pre << ret
[:KBD_CLOSE, ret]
elsif ret = @src.scan(INDEX_OPEN_RE)
@pre << ret
[:INDEX_OPEN, ret]
elsif ret = @src.scan(INDEX_CLOSE_RE)
@pre << ret
[:INDEX_CLOSE, ret]
elsif ret = @src.scan(REF_OPEN_RE)
@pre << ret
[:REF_OPEN, ret]
elsif ret = @src.scan(REF_CLOSE_RE)
@pre << ret
[:REF_CLOSE, ret]
elsif ret = @src.scan(FOOTNOTE_OPEN_RE)
@pre << ret
[:FOOTNOTE_OPEN, ret]
elsif ret = @src.scan(FOOTNOTE_CLOSE_RE)
@pre << ret
[:FOOTNOTE_CLOSE, ret]
elsif ret = @src.scan(VERB_OPEN_RE)
@pre << ret
[:VERB_OPEN, ret]
elsif ret = @src.scan(VERB_CLOSE_RE)
@pre << ret
[:VERB_CLOSE, ret]
elsif ret = @src.scan(BAR_RE)
@pre << ret
[:BAR, ret]
elsif ret = @src.scan(QUOTE_RE)
@pre << ret
[:QUOTE, ret]
elsif ret = @src.scan(SLASH_RE)
@pre << ret
[:SLASH, ret]
elsif ret = @src.scan(BACK_SLASH_RE)
@pre << ret
[:BACK_SLASH, ret]
elsif ret = @src.scan(URL_RE)
@pre << ret
[:URL, ret]
elsif ret = @src.scan(OTHER_RE)
@pre << ret
[:OTHER, ret]
else
ret = @src.rest
@pre << ret
@src.terminate
[:OTHER, ret]
end
end