lex

Tokenizes the Ruby program and returns an array of an array, which is formatted like [[lineno, column], type, token, state].

require 'ripper'
require 'pp'

pp Ripper.lex("def m(a) nil end")
#=> [[[1,  0], :on_kw,     "def", Ripper::EXPR_FNAME                   ],
     [[1,  3], :on_sp,     " ",   Ripper::EXPR_FNAME                   ],
     [[1,  4], :on_ident,  "m",   Ripper::EXPR_ENDFN                   ],
     [[1,  5], :on_lparen, "(",   Ripper::EXPR_LABEL | Ripper::EXPR_BEG],
     [[1,  6], :on_ident,  "a",   Ripper::EXPR_ARG                     ],
     [[1,  7], :on_rparen, ")",   Ripper::EXPR_ENDFN                   ],
     [[1,  8], :on_sp,     " ",   Ripper::EXPR_BEG                     ],
     [[1,  9], :on_kw,     "nil", Ripper::EXPR_END                     ],
     [[1, 12], :on_sp,     " ",   Ripper::EXPR_END                     ],
     [[1, 13], :on_kw,     "end", Ripper::EXPR_END                     ]]