def yylex
c = ''
space_seen = false
command_state = false
src = self.src
self.token = nil
self.yacc_value = nil
return yylex_string if lex_strterm
command_state = self.command_start
self.command_start = false
last_state = lex_state
loop do
if src.scan(/\ |\t|\r|\f|\13/) then
space_seen = true
next
elsif src.check(/[^a-zA-Z]/) then
if src.scan(/\n|#/) then
self.lineno = nil
c = src.matched
if c == '#' then
src.unread c
while src.scan(/\s*#.*(\n+|\z)/) do
@comments << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
end
if src.eos? then
return RubyLexer::EOF
end
end
src.scan(/\n+/)
if [:expr_beg, :expr_fname,
:expr_dot, :expr_class].include? lex_state then
next
end
self.command_start = true
self.lex_state = :expr_beg
return :tNL
elsif src.scan(/[\]\)\}]/) then
cond.lexpop
cmdarg.lexpop
self.lex_state = :expr_end
self.yacc_value = src.matched
result = {
")" => :tRPAREN,
"]" => :tRBRACK,
"}" => :tRCURLY
}[src.matched]
return result
elsif src.check(/\./) then
if src.scan(/\.\.\./) then
self.lex_state = :expr_beg
self.yacc_value = "..."
return :tDOT3
elsif src.scan(/\.\./) then
self.lex_state = :expr_beg
self.yacc_value = ".."
return :tDOT2
elsif src.scan(/\.\d/) then
rb_compile_error "no .<digit> floating literal anymore put 0 before dot"
elsif src.scan(/\./) then
self.lex_state = :expr_dot
self.yacc_value = "."
return :tDOT
end
elsif src.scan(/\,/) then
self.lex_state = :expr_beg
self.yacc_value = ","
return :tCOMMA
elsif src.scan(/\(/) then
result = :tLPAREN2
self.command_start = true
if lex_state == :expr_beg || lex_state == :expr_mid then
result = :tLPAREN
elsif space_seen then
if lex_state == :expr_cmdarg then
result = :tLPAREN_ARG
elsif lex_state == :expr_arg then
warning("don't put space before argument parentheses")
result = :tLPAREN2
end
end
self.expr_beg_push "("
return result
elsif src.check(/\=/) then
if src.scan(/\=\=\=/) then
self.fix_arg_lex_state
self.yacc_value = "==="
return :tEQQ
elsif src.scan(/\=\=/) then
self.fix_arg_lex_state
self.yacc_value = "=="
return :tEQ
elsif src.scan(/\=~/) then
self.fix_arg_lex_state
self.yacc_value = "=~"
return :tMATCH
elsif src.scan(/\=>/) then
self.fix_arg_lex_state
self.yacc_value = "=>"
return :tASSOC
elsif src.scan(/\=/) then
if src.was_begin_of_line and src.scan(/begin(?=\s)/) then
@comments << '=' << src.matched
unless src.scan(/.*?\n=end\s*(\n|\z)/m) then
@comments.clear
rb_compile_error("embedded document meets end of file")
end
@comments << src.matched
next
else
self.fix_arg_lex_state
self.yacc_value = '='
return :tEQL
end
end
elsif src.scan(/\"(#{ESC_RE}|#(#{ESC_RE}|[^\{\#\@\$\"\\])|[^\"\\\#])*\"/o) then
self.yacc_value = src.matched[1..-2].gsub(ESC_RE) { unescape $1 }
self.lex_state = :expr_end
return :tSTRING
elsif src.scan(/\"/) then
self.lex_strterm = [:strterm, STR_DQUOTE, '"', "\0"]
self.yacc_value = "\""
return :tSTRING_BEG
elsif src.scan(/\@\@?\w*/) then
self.token = src.matched
rb_compile_error "`#{token}` is not allowed as a variable name" if
token =~ /\@\d/
return process_token(command_state)
elsif src.scan(/\:\:/) then
if (lex_state == :expr_beg ||
lex_state == :expr_mid ||
lex_state == :expr_class ||
(lex_state.is_argument && space_seen)) then
self.lex_state = :expr_beg
self.yacc_value = "::"
return :tCOLON3
end
self.lex_state = :expr_dot
self.yacc_value = "::"
return :tCOLON2
elsif lex_state != :expr_end && lex_state != :expr_endarg && src.scan(/:([a-zA-Z_]\w*(?:[?!]|=(?!>))?)/) then
self.yacc_value = src[1]
self.lex_state = :expr_end
return :tSYMBOL
elsif src.scan(/\:/) then
if (lex_state == :expr_end || lex_state == :expr_endarg||
src.check(/\s/)) then
self.lex_state = :expr_beg
self.yacc_value = ":"
return :tCOLON
end
case
when src.scan(/\'/) then
self.lex_strterm = [:strterm, STR_SSYM, src.matched, "\0"]
when src.scan(/\"/) then
self.lex_strterm = [:strterm, STR_DSYM, src.matched, "\0"]
end
self.lex_state = :expr_fname
self.yacc_value = ":"
return :tSYMBEG
elsif src.check(/[0-9]/) then
return parse_number
elsif src.scan(/\[/) then
result = src.matched
if lex_state == :expr_fname || lex_state == :expr_dot then
self.lex_state = :expr_arg
case
when src.scan(/\]\=/) then
self.yacc_value = "[]="
return :tASET
when src.scan(/\]/) then
self.yacc_value = "[]"
return :tAREF
else
rb_compile_error "unexpected '['"
end
elsif lex_state == :expr_beg || lex_state == :expr_mid then
result = :tLBRACK
elsif lex_state.is_argument && space_seen then
result = :tLBRACK
end
self.expr_beg_push "["
return result
elsif src.scan(/\'(\\.|[^\'])*\'/) then
self.yacc_value = src.matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'")
self.lex_state = :expr_end
return :tSTRING
elsif src.check(/\|/) then
if src.scan(/\|\|\=/) then
self.lex_state = :expr_beg
self.yacc_value = "||"
return :tOP_ASGN
elsif src.scan(/\|\|/) then
self.lex_state = :expr_beg
self.yacc_value = "||"
return :tOROP
elsif src.scan(/\|\=/) then
self.lex_state = :expr_beg
self.yacc_value = "|"
return :tOP_ASGN
elsif src.scan(/\|/) then
self.fix_arg_lex_state
self.yacc_value = "|"
return :tPIPE
end
elsif src.scan(/\{/) then
result = if lex_state.is_argument || lex_state == :expr_end then
:tLCURLY
elsif lex_state == :expr_endarg then
:tLBRACE_ARG
else
:tLBRACE
end
self.expr_beg_push "{"
return result
elsif src.scan(/[+-]/) then
sign = src.matched
utype, type = if sign == "+" then
[:tUPLUS, :tPLUS]
else
[:tUMINUS, :tMINUS]
end
if lex_state == :expr_fname || lex_state == :expr_dot then
self.lex_state = :expr_arg
if src.scan(/@/) then
self.yacc_value = "#{sign}@"
return utype
else
self.yacc_value = sign
return type
end
end
if src.scan(/\=/) then
self.lex_state = :expr_beg
self.yacc_value = sign
return :tOP_ASGN
end
if (lex_state == :expr_beg || lex_state == :expr_mid ||
(lex_state.is_argument && space_seen && !src.check(/\s/))) then
if lex_state.is_argument then
arg_ambiguous
end
self.lex_state = :expr_beg
self.yacc_value = sign
if src.check(/\d/) then
if utype == :tUPLUS then
return self.parse_number
else
return :tUMINUS_NUM
end
end
return utype
end
self.lex_state = :expr_beg
self.yacc_value = sign
return type
elsif src.check(/\*/) then
if src.scan(/\*\*=/) then
self.lex_state = :expr_beg
self.yacc_value = "**"
return :tOP_ASGN
elsif src.scan(/\*\*/) then
self.yacc_value = "**"
self.fix_arg_lex_state
return :tPOW
elsif src.scan(/\*\=/) then
self.lex_state = :expr_beg
self.yacc_value = "*"
return :tOP_ASGN
elsif src.scan(/\*/) then
result = if lex_state.is_argument && space_seen && src.check(/\S/) then
warning("`*' interpreted as argument prefix")
:tSTAR
elsif lex_state == :expr_beg || lex_state == :expr_mid then
:tSTAR
else
:tSTAR2
end
self.yacc_value = "*"
self.fix_arg_lex_state
return result
end
elsif src.check(/\!/) then
if src.scan(/\!\=/) then
self.lex_state = :expr_beg
self.yacc_value = "!="
return :tNEQ
elsif src.scan(/\!~/) then
self.lex_state = :expr_beg
self.yacc_value = "!~"
return :tNMATCH
elsif src.scan(/\!/) then
self.lex_state = :expr_beg
self.yacc_value = "!"
return :tBANG
end
elsif src.check(/\</) then
if src.scan(/\<\=\>/) then
self.fix_arg_lex_state
self.yacc_value = "<=>"
return :tCMP
elsif src.scan(/\<\=/) then
self.fix_arg_lex_state
self.yacc_value = "<="
return :tLEQ
elsif src.scan(/\<\<\=/) then
self.fix_arg_lex_state
self.lex_state = :expr_beg
self.yacc_value = "\<\<"
return :tOP_ASGN
elsif src.scan(/\<\</) then
if (! [:expr_end, :expr_dot,
:expr_endarg, :expr_class].include?(lex_state) &&
(!lex_state.is_argument || space_seen)) then
tok = self.heredoc_identifier
if tok then
return tok
end
end
self.fix_arg_lex_state
self.yacc_value = "\<\<"
return :tLSHFT
elsif src.scan(/\</) then
self.fix_arg_lex_state
self.yacc_value = "<"
return :tLT
end
elsif src.check(/\>/) then
if src.scan(/\>\=/) then
self.fix_arg_lex_state
self.yacc_value = ">="
return :tGEQ
elsif src.scan(/\>\>=/) then
self.fix_arg_lex_state
self.lex_state = :expr_beg
self.yacc_value = ">>"
return :tOP_ASGN
elsif src.scan(/\>\>/) then
self.fix_arg_lex_state
self.yacc_value = ">>"
return :tRSHFT
elsif src.scan(/\>/) then
self.fix_arg_lex_state
self.yacc_value = ">"
return :tGT
end
elsif src.scan(/\`/) then
self.yacc_value = "`"
case lex_state
when :expr_fname then
self.lex_state = :expr_end
return :tBACK_REF2
when :expr_dot then
self.lex_state = if command_state then
:expr_cmdarg
else
:expr_arg
end
return :tBACK_REF2
end
self.lex_strterm = [:strterm, STR_XQUOTE, '`', "\0"]
return :tXSTRING_BEG
elsif src.scan(/\?/) then
if lex_state == :expr_end || lex_state == :expr_endarg then
self.lex_state = :expr_beg
self.yacc_value = "?"
return :tEH
end
if src.eos? then
rb_compile_error "incomplete character syntax"
end
if src.check(/\s|\v/) then
unless lex_state.is_argument then
c2 = { " " => 's',
"\n" => 'n',
"\t" => 't',
"\v" => 'v',
"\r" => 'r',
"\f" => 'f' }[src.matched]
if c2 then
warning("invalid character syntax; use ?\\" + c2)
end
end
self.lex_state = :expr_beg
self.yacc_value = "?"
return :tEH
elsif src.check(/\w(?=\w)/) then
self.lex_state = :expr_beg
self.yacc_value = "?"
return :tEH
end
c = if src.scan(/\\/) then
self.read_escape
else
src.getch
end
self.lex_state = :expr_end
self.yacc_value = c[0].ord & 0xff
return :tINTEGER
elsif src.check(/\&/) then
if src.scan(/\&\&\=/) then
self.yacc_value = "&&"
self.lex_state = :expr_beg
return :tOP_ASGN
elsif src.scan(/\&\&/) then
self.lex_state = :expr_beg
self.yacc_value = "&&"
return :tANDOP
elsif src.scan(/\&\=/) then
self.yacc_value = "&"
self.lex_state = :expr_beg
return :tOP_ASGN
elsif src.scan(/&/) then
result = if lex_state.is_argument && space_seen &&
!src.check(/\s/) then
warning("`&' interpreted as argument prefix")
:tAMPER
elsif lex_state == :expr_beg || lex_state == :expr_mid then
:tAMPER
else
:tAMPER2
end
self.fix_arg_lex_state
self.yacc_value = "&"
return result
end
elsif src.scan(/\//) then
if lex_state == :expr_beg || lex_state == :expr_mid then
self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
self.yacc_value = "/"
return :tREGEXP_BEG
end
if src.scan(/\=/) then
self.yacc_value = "/"
self.lex_state = :expr_beg
return :tOP_ASGN
end
if lex_state.is_argument && space_seen then
unless src.scan(/\s/) then
arg_ambiguous
self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
self.yacc_value = "/"
return :tREGEXP_BEG
end
end
self.fix_arg_lex_state
self.yacc_value = "/"
return :tDIVIDE
elsif src.scan(/\^=/) then
self.lex_state = :expr_beg
self.yacc_value = "^"
return :tOP_ASGN
elsif src.scan(/\^/) then
self.fix_arg_lex_state
self.yacc_value = "^"
return :tCARET
elsif src.scan(/\;/) then
self.command_start = true
self.lex_state = :expr_beg
self.yacc_value = ";"
return :tSEMI
elsif src.scan(/\~/) then
if lex_state == :expr_fname || lex_state == :expr_dot then
src.scan(/@/)
end
self.fix_arg_lex_state
self.yacc_value = "~"
return :tTILDE
elsif src.scan(/\\/) then
if src.scan(/\n/) then
self.lineno = nil
space_seen = true
next
end
rb_compile_error "bare backslash only allowed before newline"
elsif src.scan(/\%/) then
if lex_state == :expr_beg || lex_state == :expr_mid then
return parse_quote
end
if src.scan(/\=/) then
self.lex_state = :expr_beg
self.yacc_value = "%"
return :tOP_ASGN
end
if lex_state.is_argument && space_seen && ! src.check(/\s/) then
return parse_quote
end
self.fix_arg_lex_state
self.yacc_value = "%"
return :tPERCENT
elsif src.check(/\$/) then
if src.scan(/(\$_)(\w+)/) then
self.lex_state = :expr_end
self.token = src.matched
return process_token(command_state)
elsif src.scan(/\$_/) then
self.lex_state = :expr_end
self.token = src.matched
self.yacc_value = src.matched
return :tGVAR
elsif src.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
self.lex_state = :expr_end
self.yacc_value = src.matched
return :tGVAR
elsif src.scan(/\$([\&\`\'\+])/) then
self.lex_state = :expr_end
if last_state == :expr_fname then
self.yacc_value = src.matched
return :tGVAR
else
self.yacc_value = src[1].to_sym
return :tBACK_REF
end
elsif src.scan(/\$([1-9]\d*)/) then
self.lex_state = :expr_end
if last_state == :expr_fname then
self.yacc_value = src.matched
return :tGVAR
else
self.yacc_value = src[1].to_i
return :tNTH_REF
end
elsif src.scan(/\$0/) then
self.lex_state = :expr_end
self.token = src.matched
return process_token(command_state)
elsif src.scan(/\$\W|\$\z/) then
self.lex_state = :expr_end
self.yacc_value = "$"
return "$"
elsif src.scan(/\$\w+/)
self.lex_state = :expr_end
self.token = src.matched
return process_token(command_state)
end
elsif src.check(/\_/) then
if src.beginning_of_line? && src.scan(/\__END__(\n|\Z)/) then
self.lineno = nil
return RubyLexer::EOF
elsif src.scan(/\_\w*/) then
self.token = src.matched
return process_token(command_state)
end
end
end
if src.scan(/\004|\032|\000/) || src.eos? then
return RubyLexer::EOF
else
if src.scan(/\W/) then
rb_compile_error "Invalid char #{src.matched.inspect} in expression"
end
end
self.token = src.matched if self.src.scan(/\w+/)
return process_token(command_state)
end
end