Constants
EXPR_BEG = :EXPR_BEG
EXPR_DATA = :EXPR_DATA
EXPR_TEXT = :EXPR_TEXT
EXPR_RTEXT = :EXPR_RTEXT
EXPR_CTEXT = :EXPR_CTEXT
T_SPACE = :SPACE
T_NIL = :NIL
T_NUMBER = :NUMBER
T_ATOM = :ATOM
T_QUOTED = :QUOTED
T_LPAR = :LPAR
T_RPAR = :RPAR
T_BSLASH = :BSLASH
T_STAR = :STAR
T_LBRA = :LBRA
T_RBRA = :RBRA
T_LITERAL = :LITERAL
T_PLUS = :PLUS
T_PERCENT = :PERCENT
T_CRLF = :CRLF
T_EOF = :EOF
T_TEXT = :TEXT
BEG_REGEXP = /\G(?:\ (?# 1: SPACE )( +)|\ (?# 2: NIL )(NIL)(?=[\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+])|\ (?# 3: NUMBER )(\d+)(?=[\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+])|\ (?# 4: ATOM )([^\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+]+)|\ (?# 5: QUOTED )"((?:[^\x00\r\n"\\]|\\["\\])*)"|\ (?# 6: LPAR )(\()|\ (?# 7: RPAR )(\))|\ (?# 8: BSLASH )(\\)|\ (?# 9: STAR )(\*)|\ (?# 10: LBRA )(\[)|\ (?# 11: RBRA )(\])|\ (?# 12: LITERAL )\{(\d+)\}\r\n|\ (?# 13: PLUS )(\+)|\ (?# 14: PERCENT )(%)|\ (?# 15: CRLF )(\r\n)|\ (?# 16: EOF )(\z))/ni
DATA_REGEXP = /\G(?:\ (?# 1: SPACE )( )|\ (?# 2: NIL )(NIL)|\ (?# 3: NUMBER )(\d+)|\ (?# 4: QUOTED )"((?:[^\x00\r\n"\\]|\\["\\])*)"|\ (?# 5: LITERAL )\{(\d+)\}\r\n|\ (?# 6: LPAR )(\()|\ (?# 7: RPAR )(\)))/ni
TEXT_REGEXP = /\G(?:\ (?# 1: TEXT )([^\x00\r\n]*))/ni
RTEXT_REGEXP = /\G(?:\ (?# 1: LBRA )(\[)|\ (?# 2: TEXT )([^\x00\r\n]*))/ni
CTEXT_REGEXP = /\G(?:\ (?# 1: TEXT )([^\x00\r\n\]]*))/ni
Token = Struct.new(:symbol, :value)
ADDRESS_REGEXP = /\G\ (?# 1: NAME )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)") \ (?# 2: ROUTE )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)") \ (?# 3: MAILBOX )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)") \ (?# 4: HOST )(?:NIL|"((?:[^\x80-\xff\x00\r\n"\\]|\\["\\])*)")\ \)/ni
FLAG_REGEXP = /\ (?# FLAG )\\([^\x80-\xff(){ \x00-\x1f\x7f%"\\]+)|\ (?# ATOM )([^\x80-\xff(){ \x00-\x1f\x7f%*"\\]+)/n
STRING_TOKENS = [T_QUOTED, T_LITERAL, T_NIL]
ATOM_TOKENS = [ T_ATOM, T_NUMBER, T_NIL, T_LBRA, T_RBRA, T_PLUS