# File lib/ruby_lexer.rb, line 1232
  def process_token(command_state)

    token << src.matched if token =~ /^\w/ && src.scan(/[\!\?](?!=)/)

    result = nil
    last_state = lex_state


    case token
    when /^\$/ then
      self.lex_state, result = :expr_end, :tGVAR
    when /^@@/ then
      self.lex_state, result = :expr_end, :tCVAR
    when /^@/ then
      self.lex_state, result = :expr_end, :tIVAR
    else
      if token =~ /[!?]$/ then
        result = :tFID
      else
        if lex_state == :expr_fname then
          # ident=, not =~ => == or followed by =>
          # TODO test lexing of a=>b vs a==>b
          if src.scan(/=(?:(?![~>=])|(?==>))/) then
            result = :tIDENTIFIER
            token << src.matched
          end
        end

        result ||= if token =~ /^[A-Z]/ then
                     :tCONSTANT
                   else
                     :tIDENTIFIER
                   end
      end

      unless lex_state == :expr_dot then
        # See if it is a reserved word.
        keyword = Keyword.keyword token

        if keyword then
          state           = lex_state
          self.lex_state  = keyword.state
          self.yacc_value = token

          if keyword.id0 == :kDO then
            self.command_start = true
            return :kDO_COND  if cond.is_in_state
            return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
            return :kDO_BLOCK if state == :expr_endarg
            return :kDO
          end

          return keyword.id0 if state == :expr_beg

          self.lex_state = :expr_beg if keyword.id0 != keyword.id1

          return keyword.id1
        end
      end

      if (lex_state == :expr_beg || lex_state == :expr_mid ||
          lex_state == :expr_dot || lex_state == :expr_arg ||
          lex_state == :expr_cmdarg) then
        if command_state then
          self.lex_state = :expr_cmdarg
        else
          self.lex_state = :expr_arg
        end
      else
        self.lex_state = :expr_end
      end
    end

    self.yacc_value = token


    self.lex_state = :expr_end if
      last_state != :expr_dot && self.parser.env[token.to_sym] == :lvar

    return result
  end