tokenize

# File lib/rdoc/tom_doc.rb, line 224
  def tokenize text
    text = text.sub(/\A(Public|Internal|Deprecated):\s+/, '')

    setup_scanner text

    until @s.eos? do
      pos = @s.pos

      # leading spaces will be reflected by the column of the next token
      # the only thing we loose are trailing spaces at the end of the file
      next if @s.scan(/ +/)

      @tokens << case
                 when @s.scan(/\r?\n/) then
                   token = [:NEWLINE, @s.matched, *token_pos(pos)]
                   @line_pos = char_pos @s.pos
                   @line += 1
                   token
                 when @s.scan(/(Examples|Signature)$/) then
                   @tokens << [:HEADER, 3, *token_pos(pos)]

                   [:TEXT, @s[1], *token_pos(pos)]
                 when @s.scan(/([:\w][\w\[\]]*)[ ]+- /) then
                   [:NOTE, @s[1], *token_pos(pos)]
                 else
                   @s.scan(/.*/)
                   [:TEXT, @s.matched.sub(/\r$/, ''), *token_pos(pos)]
                 end
    end

    self
  end