# File lib/ruby_lexer.rb, line 1315
  def process_token(command_state)

    token << src.matched if token =~ IDENT_RE && src.scan(/[\!\?](?!=)/)

    result = nil
    last_state = lex_state

    case token
    when /^\$/ then
      self.lex_state, result = :expr_end, :tGVAR
    when /^@@/ then
      self.lex_state, result = :expr_end, :tCVAR
    when /^@/ then
      self.lex_state, result = :expr_end, :tIVAR
    else
      if token =~ /[!?]$/ then
        result = :tFID
      else
        if in_lex_state? :expr_fname then
          # ident=, not =~ => == or followed by =>
          # TODO test lexing of a=>b vs a==>b
          if src.scan(/=(?:(?![~>=])|(?==>))/) then
            result = :tIDENTIFIER
            token << src.matched
          end
        end

        result ||= if token =~ /^[A-Z]/ then
                     :tCONSTANT
                   else
                     :tIDENTIFIER
                   end
      end

      unless ruby18
        if is_label_possible? command_state then
          colon = src.scan(/:/)

          if colon && src.peek(1) != ":" then
            self.lex_state = :expr_beg
            self.yacc_value = [token, src.lineno]
            return :tLABEL
          end

          src.unscan if colon
        end
      end

      unless in_lex_state? :expr_dot then
        # See if it is a reserved word.
        keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses
                    RubyParserStuff::Keyword.keyword18 token
                  else
                    RubyParserStuff::Keyword.keyword19 token
                  end

        if keyword then
          state           = lex_state
          self.lex_state  = keyword.state
          self.yacc_value = [token, src.lineno]

          if state == :expr_fname then
            self.yacc_value = keyword.name
            return keyword.id0
          end

          if keyword.id0 == :kDO then
            self.command_start = true
            return :kDO_COND  if cond.is_in_state
            return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
            return :kDO_BLOCK if state == :expr_endarg
            if defined?(@hack_expects_lambda) && @hack_expects_lambda
              @hack_expects_lambda = false
              return :kDO_LAMBDA
            end
            return :kDO
          end

          return keyword.id0 if state == :expr_beg or state == :expr_value

          self.lex_state = :expr_beg if keyword.id0 != keyword.id1

          return keyword.id1
        end
      end

      # TODO:
      # if (mb == ENC_CODERANGE_7BIT && lex_state != EXPR_DOT) {

      self.lex_state =
        if is_beg? || in_lex_state?(:expr_dot) || is_arg? then
          if command_state then
            :expr_cmdarg
          else
            :expr_arg
          end
        elsif ruby19 && in_lex_state?(:expr_fname) then
          :expr_endfn
        else
          :expr_end
        end

    end

    self.yacc_value = token


    self.lex_state = :expr_end if
      last_state != :expr_dot && self.parser.env[token.to_sym] == :lvar

    return result
  end