tokenize(source)
public
Tokenizes a line of ERB. This is really just for
error reporting and nobody should use it.
# File activesupport/lib/active_support/core_ext/erb/util.rb, line 161
def self.tokenize(source) # :nodoc:
require "strscan"
source = StringScanner.new(source.chomp)
tokens = []
start_re = /<%(?:={1,2}|-|\#|%)?/
finish_re = /(?:[-=])?%>/
while !source.eos?
pos = source.pos
source.scan_until(/(?:#{start_re}|#{finish_re})/)
return [[:PLAIN, source.string]] unless source.matched?
len = source.pos - source.matched.bytesize - pos
case source.matched
when start_re
tokens << [:TEXT, source.string.byteslice(pos, len)] if len > 0
tokens << [:OPEN, source.matched]
if source.scan(/(.*?)(?=#{finish_re}|\z)/)
tokens << [:CODE, source.matched] unless source.matched.empty?
tokens << [:CLOSE, source.scan(finish_re)] unless source.eos?
else
raise NotImplementedError
end
when finish_re
tokens << [:CODE, source.string.byteslice(pos, len)] if len > 0
tokens << [:CLOSE, source.matched]
else
raise NotImplementedError, source.matched
end
unless source.eos? || source.exist?(start_re) || source.exist?(finish_re)
tokens << [:TEXT, source.rest]
source.terminate
end
end
tokens
end
