tokenize

# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 59
  def tokenize(input)
    require 'strscan'
    s = StringScanner.new input

    until s.eos? do
      pos = s.pos

      pos = s.pos if leading_whitespace = s.scan(/ +/)

      if s.scan(/[<|=>]{7}/)
        message = "your #{@filename} contains merge conflict markers"
        column, line = token_pos pos

        raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename
      end

      @tokens <<
        case
        when s.scan(/\r?\n/) then
          token = Token.new(:newline, nil, *token_pos(pos))
          @line_pos = s.pos
          @line += 1
          token
        when s.scan(/[A-Z]+/) then
          if leading_whitespace
            text = s.matched
            text += s.scan(/[^\s)]*/).to_s # in case of no match
            Token.new(:text, text, *token_pos(pos))
          else
            Token.new(:section, s.matched, *token_pos(pos))
          end
        when s.scan(/([a-z]+):\s/) then
          s.pos -= 1 # rewind for possible newline
          Token.new(:entry, s[1], *token_pos(pos))
        when s.scan(/\(/) then
          Token.new(:l_paren, nil, *token_pos(pos))
        when s.scan(/\)/) then
          Token.new(:r_paren, nil, *token_pos(pos))
        when s.scan(/<=|>=|=|~>|<|>|!=/) then
          Token.new(:requirement, s.matched, *token_pos(pos))
        when s.scan(/,/) then
          Token.new(:comma, nil, *token_pos(pos))
        when s.scan(/!/) then
          Token.new(:bang, nil, *token_pos(pos))
        when s.scan(/[^\s),!]*/) then
          Token.new(:text, s.matched, *token_pos(pos))
        else
          raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
        end
    end

    @tokens
  end