tokenize

# File lib/rubygems/request_set/lockfile.rb, line 566
  def tokenize # :nodoc:
    @line     = 0
    @line_pos = 0

    @platforms     = []
    @tokens        = []
    @current_token = nil

    lock_file = "#{@gem_deps_file}.lock"

    @input = File.read lock_file
    s      = StringScanner.new @input

    until s.eos? do
      pos = s.pos

      pos = s.pos if leading_whitespace = s.scan(/ +/)

      if s.scan(/[<|=>]{7}/) then
        message = "your #{lock_file} contains merge conflict markers"
        column, line = token_pos pos

        raise ParseError.new message, column, line, lock_file
      end

      @tokens <<
        case
        when s.scan(/\r?\n/) then
          token = [:newline, nil, *token_pos(pos)]
          @line_pos = s.pos
          @line += 1
          token
        when s.scan(/[A-Z]+/) then
          if leading_whitespace then
            text = s.matched
            text += s.scan(/[^\s)]*/).to_s # in case of no match
            [:text, text, *token_pos(pos)]
          else
            [:section, s.matched, *token_pos(pos)]
          end
        when s.scan(/([a-z]+):\s/) then
          s.pos -= 1 # rewind for possible newline
          [:entry, s[1], *token_pos(pos)]
        when s.scan(/\(/) then
          [:l_paren, nil, *token_pos(pos)]
        when s.scan(/\)/) then
          [:r_paren, nil, *token_pos(pos)]
        when s.scan(/<=|>=|=|~>|<|>|!=/) then
          [:requirement, s.matched, *token_pos(pos)]
        when s.scan(/,/) then
          [:comma, nil, *token_pos(pos)]
        when s.scan(/!/) then
          [:bang, nil, *token_pos(pos)]
        when s.scan(/[^\s),!]*/) then
          [:text, s.matched, *token_pos(pos)]
        else
          raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
        end
    end

    @tokens
  rescue Errno::ENOENT
    @tokens
  end