# File lib/ruby_lexer.rb, line 618
  def yylex # 826 lines

    c = ''
    space_seen = false
    command_state = false
    src = self.src

    self.token = nil
    self.yacc_value = nil

    return yylex_string if lex_strterm

    command_state = self.command_start
    self.command_start = false

    last_state = lex_state

    loop do # START OF CASE
      if src.scan(/\ |\t|\r|\f|\13/) then # white spaces, 13 = '\v
        space_seen = true
        next
      elsif src.check(/[^a-zA-Z]/) then
        if src.scan(/\n|#/) then
          self.lineno = nil
          c = src.matched
          if c == '#' then
            src.unread c # ok

            while src.scan(/\s*#.*(\n+|\z)/) do
              @comments << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
            end

            if src.eos? then
              return RubyLexer::EOF
            end
          end

          # Replace a string of newlines with a single one
          src.scan(/\n+/)

          if [:expr_beg, :expr_fname,
              :expr_dot, :expr_class].include? lex_state then
            next
          end

          self.command_start = true
          self.lex_state = :expr_beg
          return :tNL
        elsif src.scan(/[\]\)\}]/) then
          cond.lexpop
          cmdarg.lexpop
          self.lex_state = :expr_end
          self.yacc_value = src.matched
          result = {
            ")" => :tRPAREN,
            "]" => :tRBRACK,
            "}" => :tRCURLY
          }[src.matched]
          return result
        elsif src.check(/\./) then
          if src.scan(/\.\.\./) then
            self.lex_state = :expr_beg
            self.yacc_value = "..."
            return :tDOT3
          elsif src.scan(/\.\./) then
            self.lex_state = :expr_beg
            self.yacc_value = ".."
            return :tDOT2
          elsif src.scan(/\.\d/) then
            rb_compile_error "no .<digit> floating literal anymore put 0 before dot"
          elsif src.scan(/\./) then
            self.lex_state = :expr_dot
            self.yacc_value = "."
            return :tDOT
          end
        elsif src.scan(/\,/) then
          self.lex_state = :expr_beg
          self.yacc_value = ","
          return :tCOMMA
        elsif src.scan(/\(/) then
          result = :tLPAREN2
          self.command_start = true
          if lex_state == :expr_beg || lex_state == :expr_mid then
            result = :tLPAREN
          elsif space_seen then
            if lex_state == :expr_cmdarg then
              result = :tLPAREN_ARG
            elsif lex_state == :expr_arg then
              warning("don't put space before argument parentheses")
              result = :tLPAREN2
            end
          end

          self.expr_beg_push "("

          return result
        elsif src.check(/\=/) then
          if src.scan(/\=\=\=/) then
            self.fix_arg_lex_state
            self.yacc_value = "==="
            return :tEQQ
          elsif src.scan(/\=\=/) then
            self.fix_arg_lex_state
            self.yacc_value = "=="
            return :tEQ
          elsif src.scan(/\=~/) then
            self.fix_arg_lex_state
            self.yacc_value = "=~"
            return :tMATCH
          elsif src.scan(/\=>/) then
            self.fix_arg_lex_state
            self.yacc_value = "=>"
            return :tASSOC
          elsif src.scan(/\=/) then
            if src.was_begin_of_line and src.scan(/begin(?=\s)/) then
              @comments << '=' << src.matched

              unless src.scan(/.*?\n=end\s*(\n|\z)/m) then
                @comments.clear
                rb_compile_error("embedded document meets end of file")
              end

              @comments << src.matched

              next
            else
              self.fix_arg_lex_state
              self.yacc_value = '='
              return :tEQL
            end
          end
        elsif src.scan(/\"(#{ESC_RE}|#(#{ESC_RE}|[^\{\#\@\$\"\\])|[^\"\\\#])*\"/o) then
          self.yacc_value = src.matched[1..-2].gsub(ESC_RE) { unescape $1 }
          self.lex_state = :expr_end
          return :tSTRING
        elsif src.scan(/\"/) then # FALLBACK
          self.lex_strterm = [:strterm, STR_DQUOTE, '"', "\0"] # TODO: question this
          self.yacc_value = "\""
          return :tSTRING_BEG
        elsif src.scan(/\@\@?\w*/) then
          self.token = src.matched

          rb_compile_error "`#{token}` is not allowed as a variable name" if
            token =~ /\@\d/

          return process_token(command_state)
        elsif src.scan(/\:\:/) then
          if (lex_state == :expr_beg ||
              lex_state == :expr_mid ||
              lex_state == :expr_class ||
              (lex_state.is_argument && space_seen)) then
            self.lex_state = :expr_beg
            self.yacc_value = "::"
            return :tCOLON3
          end

          self.lex_state = :expr_dot
          self.yacc_value = "::"
          return :tCOLON2
        elsif lex_state != :expr_end && lex_state != :expr_endarg && src.scan(/:([a-zA-Z_]\w*(?:[?!]|=(?!>))?)/) then
          self.yacc_value = src[1]
          self.lex_state = :expr_end
          return :tSYMBOL
        elsif src.scan(/\:/) then
          # ?: / then / when
          if (lex_state == :expr_end || lex_state == :expr_endarg||
              src.check(/\s/)) then
            self.lex_state = :expr_beg
            self.yacc_value = ":"
            return :tCOLON
          end

          case
          when src.scan(/\'/) then
            self.lex_strterm = [:strterm, STR_SSYM, src.matched, "\0"]
          when src.scan(/\"/) then
            self.lex_strterm = [:strterm, STR_DSYM, src.matched, "\0"]
          end

          self.lex_state = :expr_fname
          self.yacc_value = ":"
          return :tSYMBEG
        elsif src.check(/[0-9]/) then
          return parse_number
        elsif src.scan(/\[/) then
          result = src.matched

          if lex_state == :expr_fname || lex_state == :expr_dot then
            self.lex_state = :expr_arg
            case
            when src.scan(/\]\=/) then
              self.yacc_value = "[]="
              return :tASET
            when src.scan(/\]/) then
              self.yacc_value = "[]"
              return :tAREF
            else
              rb_compile_error "unexpected '['"
            end
          elsif lex_state == :expr_beg || lex_state == :expr_mid then
            result = :tLBRACK
          elsif lex_state.is_argument && space_seen then
            result = :tLBRACK
          end

          self.expr_beg_push "["

          return result
        elsif src.scan(/\'(\\.|[^\'])*\'/) then
          self.yacc_value = src.matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'")
          self.lex_state = :expr_end
          return :tSTRING
        elsif src.check(/\|/) then
          if src.scan(/\|\|\=/) then
            self.lex_state = :expr_beg
            self.yacc_value = "||"
            return :tOP_ASGN
          elsif src.scan(/\|\|/) then
            self.lex_state = :expr_beg
            self.yacc_value = "||"
            return :tOROP
          elsif src.scan(/\|\=/) then
            self.lex_state = :expr_beg
            self.yacc_value = "|"
            return :tOP_ASGN
          elsif src.scan(/\|/) then
            self.fix_arg_lex_state
            self.yacc_value = "|"
            return :tPIPE
          end
        elsif src.scan(/\{/) then
          result = if lex_state.is_argument || lex_state == :expr_end then
                     :tLCURLY      #  block (primary)
                   elsif lex_state == :expr_endarg then
                     :tLBRACE_ARG  #  block (expr)
                   else
                     :tLBRACE      #  hash
                   end

          self.expr_beg_push "{"

          return result
        elsif src.scan(/[+-]/) then
          sign = src.matched
          utype, type = if sign == "+" then
                          [:tUPLUS, :tPLUS]
                        else
                          [:tUMINUS, :tMINUS]
                        end

          if lex_state == :expr_fname || lex_state == :expr_dot then
            self.lex_state = :expr_arg
            if src.scan(/@/) then
              self.yacc_value = "#{sign}@"
              return utype
            else
              self.yacc_value = sign
              return type
            end
          end

          if src.scan(/\=/) then
            self.lex_state = :expr_beg
            self.yacc_value = sign
            return :tOP_ASGN
          end

          if (lex_state == :expr_beg || lex_state == :expr_mid ||
              (lex_state.is_argument && space_seen && !src.check(/\s/))) then
            if lex_state.is_argument then
              arg_ambiguous
            end

            self.lex_state = :expr_beg
            self.yacc_value = sign

            if src.check(/\d/) then
              if utype == :tUPLUS then
                return self.parse_number
              else
                return :tUMINUS_NUM
              end
            end

            return utype
          end

          self.lex_state = :expr_beg
          self.yacc_value = sign
          return type
        elsif src.check(/\*/) then
          if src.scan(/\*\*=/) then
            self.lex_state = :expr_beg
            self.yacc_value = "**"
            return :tOP_ASGN
          elsif src.scan(/\*\*/) then
            self.yacc_value = "**"
            self.fix_arg_lex_state
            return :tPOW
          elsif src.scan(/\*\=/) then
            self.lex_state = :expr_beg
            self.yacc_value = "*"
            return :tOP_ASGN
          elsif src.scan(/\*/) then
            result = if lex_state.is_argument && space_seen && src.check(/\S/) then
                       warning("`*' interpreted as argument prefix")
                       :tSTAR
                     elsif lex_state == :expr_beg || lex_state == :expr_mid then
                       :tSTAR
                     else
                       :tSTAR2
                     end
            self.yacc_value = "*"
            self.fix_arg_lex_state

            return result
          end
        elsif src.check(/\!/) then
          if src.scan(/\!\=/) then
            self.lex_state = :expr_beg
            self.yacc_value = "!="
            return :tNEQ
          elsif src.scan(/\!~/) then
            self.lex_state = :expr_beg
            self.yacc_value = "!~"
            return :tNMATCH
          elsif src.scan(/\!/) then
            self.lex_state = :expr_beg
            self.yacc_value = "!"
            return :tBANG
          end
        elsif src.check(/\</) then
          if src.scan(/\<\=\>/) then
            self.fix_arg_lex_state
            self.yacc_value = "<=>"
            return :tCMP
          elsif src.scan(/\<\=/) then
            self.fix_arg_lex_state
            self.yacc_value = "<="
            return :tLEQ
          elsif src.scan(/\<\<\=/) then
            self.fix_arg_lex_state
            self.lex_state = :expr_beg
            self.yacc_value = "\<\<"
            return :tOP_ASGN
          elsif src.scan(/\<\</) then
            if (! [:expr_end,    :expr_dot,
                   :expr_endarg, :expr_class].include?(lex_state) &&
                (!lex_state.is_argument || space_seen)) then
              tok = self.heredoc_identifier
              if tok then
                return tok
              end
            end

            self.fix_arg_lex_state
            self.yacc_value = "\<\<"
            return :tLSHFT
          elsif src.scan(/\</) then
            self.fix_arg_lex_state
            self.yacc_value = "<"
            return :tLT
          end
        elsif src.check(/\>/) then
          if src.scan(/\>\=/) then
            self.fix_arg_lex_state
            self.yacc_value = ">="
            return :tGEQ
          elsif src.scan(/\>\>=/) then
            self.fix_arg_lex_state
            self.lex_state = :expr_beg
            self.yacc_value = ">>"
            return :tOP_ASGN
          elsif src.scan(/\>\>/) then
            self.fix_arg_lex_state
            self.yacc_value = ">>"
            return :tRSHFT
          elsif src.scan(/\>/) then
            self.fix_arg_lex_state
            self.yacc_value = ">"
            return :tGT
          end
        elsif src.scan(/\`/) then
          self.yacc_value = "`"
          case lex_state
          when :expr_fname then
            self.lex_state = :expr_end
            return :tBACK_REF2
          when :expr_dot then
            self.lex_state = if command_state then
                               :expr_cmdarg
                             else
                               :expr_arg
                             end
            return :tBACK_REF2
          end
          self.lex_strterm = [:strterm, STR_XQUOTE, '`', "\0"]
          return :tXSTRING_BEG
        elsif src.scan(/\?/) then
          if lex_state == :expr_end || lex_state == :expr_endarg then
            self.lex_state = :expr_beg
            self.yacc_value = "?"
            return :tEH
          end

          if src.eos? then
            rb_compile_error "incomplete character syntax"
          end

          if src.check(/\s|\v/) then
            unless lex_state.is_argument then
              c2 = { " " => 's',
                    "\n" => 'n',
                    "\t" => 't',
                    "\v" => 'v',
                    "\r" => 'r',
                    "\f" => 'f' }[src.matched]

              if c2 then
                warning("invalid character syntax; use ?\\" + c2)
              end
            end

            # ternary
            self.lex_state = :expr_beg
            self.yacc_value = "?"
            return :tEH
          elsif src.check(/\w(?=\w)/) then # ternary, also
            self.lex_state = :expr_beg
            self.yacc_value = "?"
            return :tEH
          end

          c = if src.scan(/\\/) then
                self.read_escape
              else
                src.getch
              end
          self.lex_state = :expr_end
          self.yacc_value = c[0].ord & 0xff
          return :tINTEGER
        elsif src.check(/\&/) then
          if src.scan(/\&\&\=/) then
            self.yacc_value = "&&"
            self.lex_state = :expr_beg
            return :tOP_ASGN
          elsif src.scan(/\&\&/) then
            self.lex_state = :expr_beg
            self.yacc_value = "&&"
            return :tANDOP
          elsif src.scan(/\&\=/) then
            self.yacc_value = "&"
            self.lex_state = :expr_beg
            return :tOP_ASGN
          elsif src.scan(/&/) then
            result = if lex_state.is_argument && space_seen &&
                         !src.check(/\s/) then
                       warning("`&' interpreted as argument prefix")
                       :tAMPER
                     elsif lex_state == :expr_beg || lex_state == :expr_mid then
                       :tAMPER
                     else
                       :tAMPER2
                     end

            self.fix_arg_lex_state
            self.yacc_value = "&"
            return result
          end
        elsif src.scan(/\//) then
          if lex_state == :expr_beg || lex_state == :expr_mid then
            self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
            self.yacc_value = "/"
            return :tREGEXP_BEG
          end

          if src.scan(/\=/) then
            self.yacc_value = "/"
            self.lex_state = :expr_beg
            return :tOP_ASGN
          end

          if lex_state.is_argument && space_seen then
            unless src.scan(/\s/) then
              arg_ambiguous
              self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
              self.yacc_value = "/"
              return :tREGEXP_BEG
            end
          end

          self.fix_arg_lex_state
          self.yacc_value = "/"

          return :tDIVIDE
        elsif src.scan(/\^=/) then
          self.lex_state = :expr_beg
          self.yacc_value = "^"
          return :tOP_ASGN
        elsif src.scan(/\^/) then
          self.fix_arg_lex_state
          self.yacc_value = "^"
          return :tCARET
        elsif src.scan(/\;/) then
          self.command_start = true
          self.lex_state = :expr_beg
          self.yacc_value = ";"
          return :tSEMI
        elsif src.scan(/\~/) then
          if lex_state == :expr_fname || lex_state == :expr_dot then
            src.scan(/@/)
          end

          self.fix_arg_lex_state
          self.yacc_value = "~"

          return :tTILDE
        elsif src.scan(/\\/) then
          if src.scan(/\n/) then
            self.lineno = nil
            space_seen = true
            next
          end
          rb_compile_error "bare backslash only allowed before newline"
        elsif src.scan(/\%/) then
          if lex_state == :expr_beg || lex_state == :expr_mid then
            return parse_quote
          end

          if src.scan(/\=/) then
            self.lex_state = :expr_beg
            self.yacc_value = "%"
            return :tOP_ASGN
          end

          if lex_state.is_argument && space_seen && ! src.check(/\s/) then
            return parse_quote
          end

          self.fix_arg_lex_state
          self.yacc_value = "%"

          return :tPERCENT
        elsif src.check(/\$/) then
          if src.scan(/(\$_)(\w+)/) then
            self.lex_state = :expr_end
            self.token = src.matched
            return process_token(command_state)
          elsif src.scan(/\$_/) then
            self.lex_state = :expr_end
            self.token = src.matched
            self.yacc_value = src.matched
            return :tGVAR
          elsif src.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
            self.lex_state = :expr_end
            self.yacc_value = src.matched
            return :tGVAR
          elsif src.scan(/\$([\&\`\'\+])/) then
            self.lex_state = :expr_end
            # Explicit reference to these vars as symbols...
            if last_state == :expr_fname then
              self.yacc_value = src.matched
              return :tGVAR
            else
              self.yacc_value = src[1].to_sym
              return :tBACK_REF
            end
          elsif src.scan(/\$([1-9]\d*)/) then
            self.lex_state = :expr_end
            if last_state == :expr_fname then
              self.yacc_value = src.matched
              return :tGVAR
            else
              self.yacc_value = src[1].to_i
              return :tNTH_REF
            end
          elsif src.scan(/\$0/) then
            self.lex_state = :expr_end
            self.token = src.matched
            return process_token(command_state)
          elsif src.scan(/\$\W|\$\z/) then # TODO: remove?
            self.lex_state = :expr_end
            self.yacc_value = "$"
            return "$"
          elsif src.scan(/\$\w+/)
            self.lex_state = :expr_end
            self.token = src.matched
            return process_token(command_state)
          end
        elsif src.check(/\_/) then
          if src.beginning_of_line? && src.scan(/\__END__(\n|\Z)/) then
            self.lineno = nil
            return RubyLexer::EOF
          elsif src.scan(/\_\w*/) then
            self.token = src.matched
            return process_token(command_state)
          end
        end
      end # END OF CASE

      if src.scan(/\004|\032|\000/) || src.eos? then # ^D, ^Z, EOF
        return RubyLexer::EOF
      else # alpha check
        if src.scan(/\W/) then
          rb_compile_error "Invalid char #{src.matched.inspect} in expression"
        end
      end

      self.token = src.matched if self.src.scan(/\w+/)

      return process_token(command_state)
    end
  end