tokenize

# File activesupport/lib/active_support/core_ext/erb/util.rb, line 161
    def self.tokenize(source) # :nodoc:
      require "strscan"
      source = StringScanner.new(source.chomp)
      tokens = []

      start_re = /<%(?:={1,2}|-|\#|%)?/
      finish_re = /(?:[-=])?%>/

      while !source.eos?
        pos = source.pos
        source.scan_until(/(?:#{start_re}|#{finish_re})/)
        raise NotImplementedError if source.matched.nil?
        len = source.pos - source.matched.bytesize - pos

        case source.matched
        when start_re
          tokens << [:TEXT, source.string[pos, len]] if len > 0
          tokens << [:OPEN, source.matched]
          if source.scan(/(.*?)(?=#{finish_re}|\z)/)
            tokens << [:CODE, source.matched] unless source.matched.empty?
            tokens << [:CLOSE, source.scan(finish_re)] unless source.eos?
          else
            raise NotImplementedError
          end
        when finish_re
          tokens << [:CODE, source.string[pos, len]] if len > 0
          tokens << [:CLOSE, source.matched]
        else
          raise NotImplementedError, source.matched
        end

        unless source.eos? || source.exist?(start_re) || source.exist?(finish_re)
          tokens << [:TEXT, source.rest]
          source.terminate
        end
      end

      tokens
    end