def process_token(command_state)
token << src.matched if token =~ /^\w/ && src.scan(/[\!\?](?!=)/)
result = nil
last_state = lex_state
case token
when /^\$/ then
self.lex_state, result = :expr_end, :tGVAR
when /^@@/ then
self.lex_state, result = :expr_end, :tCVAR
when /^@/ then
self.lex_state, result = :expr_end, :tIVAR
else
if token =~ /[!?]$/ then
result = :tFID
else
if lex_state == :expr_fname then
if src.scan(/=(?:(?![~>=])|(?==>))/) then
result = :tIDENTIFIER
token << src.matched
end
end
result ||= if token =~ /^[A-Z]/ then
:tCONSTANT
else
:tIDENTIFIER
end
end
unless lex_state == :expr_dot then
keyword = Keyword.keyword token
if keyword then
state = lex_state
self.lex_state = keyword.state
self.yacc_value = token
if keyword.id0 == :kDO then
self.command_start = true
return :kDO_COND if cond.is_in_state
return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
return :kDO_BLOCK if state == :expr_endarg
return :kDO
end
return keyword.id0 if state == :expr_beg
self.lex_state = :expr_beg if keyword.id0 != keyword.id1
return keyword.id1
end
end
if (lex_state == :expr_beg || lex_state == :expr_mid ||
lex_state == :expr_dot || lex_state == :expr_arg ||
lex_state == :expr_cmdarg) then
if command_state then
self.lex_state = :expr_cmdarg
else
self.lex_state = :expr_arg
end
else
self.lex_state = :expr_end
end
end
self.yacc_value = token
self.lex_state = :expr_end if
last_state != :expr_dot && self.parser.env[token.to_sym] == :lvar
return result
end