def yylex
c = ''
self.space_seen = false
command_state = false
src = self.src
self.token = nil
self.yacc_value = nil
return yylex_string if lex_strterm
command_state = self.command_start
self.command_start = false
last_state = lex_state
loop do
if src.scan(/[\ \t\r\f\v]/) then
self.space_seen = true
next
elsif src.check(/[^a-zA-Z]/) then
if src.scan(/\n|#/) then
self.lineno = nil
c = src.matched
if c == '#' then
src.pos -= 1
while src.scan(/\s*#.*(\n+|\z)/) do
@comments << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
end
return RubyLexer::EOF if src.eos?
end
src.scan(/\n+/)
next if in_lex_state?(:expr_beg, :expr_fname, :expr_dot, :expr_class,
:expr_value)
if src.scan(/([\ \t\r\f\v]*)\./) then
self.space_seen = true unless src[1].empty?
src.pos -= 1
next unless src.check(/\.\./)
end
self.command_start = true
self.lex_state = :expr_beg
return :tNL
elsif src.scan(/[\]\)\}]/) then
cond.lexpop
cmdarg.lexpop
tern.lexpop
self.lex_state = :expr_end
self.yacc_value = src.matched
result = {
")" => :tRPAREN,
"]" => :tRBRACK,
"}" => :tRCURLY
}[src.matched]
return result
elsif src.scan(/\.\.\.?|,|![=~]?/) then
self.lex_state = :expr_beg
tok = self.yacc_value = src.matched
return TOKENS[tok]
elsif src.check(/\./) then
if src.scan(/\.\d/) then
rb_compile_error "no .<digit> floating literal anymore put 0 before dot"
elsif src.scan(/\./) then
self.lex_state = :expr_dot
self.yacc_value = "."
return :tDOT
end
elsif src.scan(/\(/) then
result = if ruby18 then
yylex_paren18
else
yylex_paren19
end
self.expr_beg_push "("
return result
elsif src.check(/\=/) then
if src.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
self.fix_arg_lex_state
tok = self.yacc_value = src.matched
return TOKENS[tok]
elsif src.scan(/\=begin(?=\s)/) then
@comments << src.matched
unless src.scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then
@comments.clear
rb_compile_error("embedded document meets end of file")
end
@comments << src.matched
next
else
raise "you shouldn't be able to get here"
end
elsif src.scan(/\"(#{ESC_RE}|#(#{ESC_RE}|[^\{\#\@\$\"\\])|[^\"\\\#])*\"/o) then
self.yacc_value = src.matched[1..-2].gsub(ESC_RE) { unescape $1 }
self.lex_state = :expr_end
return :tSTRING
elsif src.scan(/\"/) then
self.lex_strterm = [:strterm, STR_DQUOTE, '"', "\0"]
self.yacc_value = "\""
return :tSTRING_BEG
elsif src.scan(/\@\@?\w*/) then
self.token = src.matched
rb_compile_error "`#{token}` is not allowed as a variable name" if
token =~ /\@\d/
return process_token(command_state)
elsif src.scan(/\:\:/) then
if is_beg? || in_lex_state?(:expr_class) || is_space_arg? then
self.lex_state = :expr_beg
self.yacc_value = "::"
return :tCOLON3
end
self.lex_state = :expr_dot
self.yacc_value = "::"
return :tCOLON2
elsif ! is_end? && src.scan(/:([a-zA-Z_]#{IDENT_CHAR_RE}*(?:[?!]|=(?==>)|=(?![=>]))?)/) then
self.yacc_value = src[1]
self.lex_state = :expr_end
return :tSYMBOL
elsif src.scan(/\:/) then
if is_end? || src.check(/\s/) then
self.lex_state = :expr_beg
self.yacc_value = ":"
return :tCOLON
end
case
when src.scan(/\'/) then
self.lex_strterm = [:strterm, STR_SSYM, src.matched, "\0"]
when src.scan(/\"/) then
self.lex_strterm = [:strterm, STR_DSYM, src.matched, "\0"]
end
self.lex_state = :expr_fname
self.yacc_value = ":"
return :tSYMBEG
elsif src.check(/[0-9]/) then
return parse_number
elsif src.scan(/\[/) then
result = src.matched
if in_lex_state? :expr_fname, :expr_dot then
self.lex_state = :expr_arg
case
when src.scan(/\]\=/) then
self.yacc_value = "[]="
return :tASET
when src.scan(/\]/) then
self.yacc_value = "[]"
return :tAREF
else
rb_compile_error "unexpected '['"
end
elsif is_beg? then
self.tern.push false
result = :tLBRACK
elsif is_arg? && space_seen then
self.tern.push false
result = :tLBRACK
else
result = :tLBRACK2
end
self.expr_beg_push "["
return result
elsif src.scan(/\'(\\.|[^\'])*\'/) then
self.yacc_value = src.matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'")
self.lex_state = :expr_end
return :tSTRING
elsif src.check(/\|/) then
if src.scan(/\|\|\=/) then
self.lex_state = :expr_beg
self.yacc_value = "||"
return :tOP_ASGN
elsif src.scan(/\|\|/) then
self.lex_state = :expr_beg
self.yacc_value = "||"
return :tOROP
elsif src.scan(/\|\=/) then
self.lex_state = :expr_beg
self.yacc_value = "|"
return :tOP_ASGN
elsif src.scan(/\|/) then
self.fix_arg_lex_state
self.yacc_value = "|"
return :tPIPE
end
elsif src.scan(/\{/) then
if defined?(@hack_expects_lambda) && @hack_expects_lambda
@hack_expects_lambda = false
self.lex_state = :expr_beg
return :tLAMBEG
end
result = if is_arg? || in_lex_state?(:expr_end) then
:tLCURLY
elsif in_lex_state?(:expr_endarg) then
:tLBRACE_ARG
else
self.tern.push false
:tLBRACE
end
self.expr_beg_push "{"
self.command_start = true unless result == :tLBRACE
return result
elsif src.scan(/->/) then
@hack_expects_lambda = true
self.lex_state = :expr_arg
return :tLAMBDA
elsif src.scan(/[+-]/) then
sign = src.matched
utype, type = if sign == "+" then
[:tUPLUS, :tPLUS]
else
[:tUMINUS, :tMINUS]
end
if in_lex_state? :expr_fname, :expr_dot then
self.lex_state = :expr_arg
if src.scan(/@/) then
self.yacc_value = "#{sign}@"
return utype
else
self.yacc_value = sign
return type
end
end
if src.scan(/\=/) then
self.lex_state = :expr_beg
self.yacc_value = sign
return :tOP_ASGN
end
if (is_beg? ||
(is_arg? && space_seen && !src.check(/\s/))) then
if is_arg? then
arg_ambiguous
end
self.lex_state = :expr_beg
self.yacc_value = sign
if src.check(/\d/) then
if utype == :tUPLUS then
return self.parse_number
else
return :tUMINUS_NUM
end
end
return utype
end
self.lex_state = :expr_beg
self.yacc_value = sign
return type
elsif src.check(/\*/) then
if src.scan(/\*\*=/) then
self.lex_state = :expr_beg
self.yacc_value = "**"
return :tOP_ASGN
elsif src.scan(/\*\*/) then
self.yacc_value = "**"
self.fix_arg_lex_state
return :tPOW
elsif src.scan(/\*\=/) then
self.lex_state = :expr_beg
self.yacc_value = "*"
return :tOP_ASGN
elsif src.scan(/\*/) then
result = if is_arg? && space_seen && src.check(/\S/) then
warning("`*' interpreted as argument prefix")
:tSTAR
elsif is_beg? then
:tSTAR
else
:tSTAR2
end
self.yacc_value = "*"
self.fix_arg_lex_state
return result
end
elsif src.check(/\</) then
if src.scan(/\<\=\>/) then
self.fix_arg_lex_state
self.yacc_value = "<=>"
return :tCMP
elsif src.scan(/\<\=/) then
self.fix_arg_lex_state
self.yacc_value = "<="
return :tLEQ
elsif src.scan(/\<\<\=/) then
self.fix_arg_lex_state
self.lex_state = :expr_beg
self.yacc_value = "\<\<"
return :tOP_ASGN
elsif src.scan(/\<\</) then
if (! in_lex_state?(:expr_end, :expr_dot,
:expr_endarg, :expr_class) &&
(!is_arg? || space_seen)) then
tok = self.heredoc_identifier
return tok if tok
end
self.fix_arg_lex_state
self.yacc_value = "\<\<"
return :tLSHFT
elsif src.scan(/\</) then
self.fix_arg_lex_state
self.yacc_value = "<"
return :tLT
end
elsif src.check(/\>/) then
if src.scan(/\>\=/) then
self.fix_arg_lex_state
self.yacc_value = ">="
return :tGEQ
elsif src.scan(/\>\>=/) then
self.fix_arg_lex_state
self.lex_state = :expr_beg
self.yacc_value = ">>"
return :tOP_ASGN
elsif src.scan(/\>\>/) then
self.fix_arg_lex_state
self.yacc_value = ">>"
return :tRSHFT
elsif src.scan(/\>/) then
self.fix_arg_lex_state
self.yacc_value = ">"
return :tGT
end
elsif src.scan(/\`/) then
self.yacc_value = "`"
case lex_state
when :expr_fname then
self.lex_state = :expr_end
return :tBACK_REF2
when :expr_dot then
self.lex_state = if command_state then
:expr_cmdarg
else
:expr_arg
end
return :tBACK_REF2
end
self.lex_strterm = [:strterm, STR_XQUOTE, '`', "\0"]
return :tXSTRING_BEG
elsif src.scan(/\?/) then
if is_end? then
self.lex_state = ruby18 ? :expr_beg : :expr_value
self.tern.push true
self.yacc_value = "?"
return :tEH
end
if src.eos? then
rb_compile_error "incomplete character syntax"
end
if src.check(/\s|\v/) then
unless is_arg? then
c2 = { " " => 's',
"\n" => 'n',
"\t" => 't',
"\v" => 'v',
"\r" => 'r',
"\f" => 'f' }[src.matched]
if c2 then
warning("invalid character syntax; use ?\\" + c2)
end
end
self.lex_state = ruby18 ? :expr_beg : :expr_value
self.tern.push true
self.yacc_value = "?"
return :tEH
elsif src.check(/\w(?=\w)/) then
self.lex_state = :expr_beg
self.tern.push true
self.yacc_value = "?"
return :tEH
end
c = if src.scan(/\\/) then
self.read_escape
else
src.getch
end
self.lex_state = :expr_end
if version == 18 then
self.yacc_value = c[0].ord & 0xff
return :tINTEGER
else
self.yacc_value = c
return :tSTRING
end
elsif src.check(/\&/) then
if src.scan(/\&\&\=/) then
self.yacc_value = "&&"
self.lex_state = :expr_beg
return :tOP_ASGN
elsif src.scan(/\&\&/) then
self.lex_state = :expr_beg
self.yacc_value = "&&"
return :tANDOP
elsif src.scan(/\&\=/) then
self.yacc_value = "&"
self.lex_state = :expr_beg
return :tOP_ASGN
elsif src.scan(/&/) then
result = if is_arg? && space_seen &&
!src.check(/\s/) then
warning("`&' interpreted as argument prefix")
:tAMPER
elsif in_lex_state? :expr_beg, :expr_mid then
:tAMPER
else
:tAMPER2
end
self.fix_arg_lex_state
self.yacc_value = "&"
return result
end
elsif src.scan(/\//) then
if is_beg? then
self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
self.yacc_value = "/"
return :tREGEXP_BEG
end
if src.scan(/\=/) then
self.yacc_value = "/"
self.lex_state = :expr_beg
return :tOP_ASGN
end
if is_arg? && space_seen then
unless src.scan(/\s/) then
arg_ambiguous
self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
self.yacc_value = "/"
return :tREGEXP_BEG
end
end
self.fix_arg_lex_state
self.yacc_value = "/"
return :tDIVIDE
elsif src.scan(/\^=/) then
self.lex_state = :expr_beg
self.yacc_value = "^"
return :tOP_ASGN
elsif src.scan(/\^/) then
self.fix_arg_lex_state
self.yacc_value = "^"
return :tCARET
elsif src.scan(/\;/) then
self.command_start = true
self.lex_state = :expr_beg
self.yacc_value = ";"
return :tSEMI
elsif src.scan(/\~/) then
if in_lex_state? :expr_fname, :expr_dot then
src.scan(/@/)
end
self.fix_arg_lex_state
self.yacc_value = "~"
return :tTILDE
elsif src.scan(/\\/) then
if src.scan(/\r?\n/) then
self.lineno = nil
self.space_seen = true
next
end
rb_compile_error "bare backslash only allowed before newline"
elsif src.scan(/\%/) then
if is_beg? then
return parse_quote
end
if src.scan(/\=/) then
self.lex_state = :expr_beg
self.yacc_value = "%"
return :tOP_ASGN
end
return parse_quote if is_arg? && space_seen && ! src.check(/\s/)
self.fix_arg_lex_state
self.yacc_value = "%"
return :tPERCENT
elsif src.check(/\$/) then
if src.scan(/(\$_)(\w+)/) then
self.lex_state = :expr_end
self.token = src.matched
return process_token(command_state)
elsif src.scan(/\$_/) then
self.lex_state = :expr_end
self.token = src.matched
self.yacc_value = src.matched
return :tGVAR
elsif src.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
self.lex_state = :expr_end
self.yacc_value = src.matched
return :tGVAR
elsif src.scan(/\$([\&\`\'\+])/) then
self.lex_state = :expr_end
if last_state == :expr_fname then
self.yacc_value = src.matched
return :tGVAR
else
self.yacc_value = src[1].to_sym
return :tBACK_REF
end
elsif src.scan(/\$([1-9]\d*)/) then
self.lex_state = :expr_end
if last_state == :expr_fname then
self.yacc_value = src.matched
return :tGVAR
else
self.yacc_value = src[1].to_i
return :tNTH_REF
end
elsif src.scan(/\$0/) then
self.lex_state = :expr_end
self.token = src.matched
return process_token(command_state)
elsif src.scan(/\$\W|\$\z/) then
self.lex_state = :expr_end
self.yacc_value = "$"
return "$"
elsif src.scan(/\$\w+/)
self.lex_state = :expr_end
self.token = src.matched
return process_token(command_state)
end
elsif src.check(/\_/) then
if src.beginning_of_line? && src.scan(/\__END__(\n|\Z)/) then
self.lineno = nil
return RubyLexer::EOF
elsif src.scan(/\_\w*/) then
self.token = src.matched
return process_token(command_state)
end
end
end
if src.scan(/\004|\032|\000/) || src.eos? then
return RubyLexer::EOF
else
unless src.check IDENT_RE then
rb_compile_error "Invalid char #{src.matched.inspect} in expression"
end
end
self.token = src.matched if self.src.scan IDENT_RE
return process_token(command_state)
end
end