class RubyLexer
lexical scanner definition for ruby
Constants
- ESC
- FLOAT
- FLOAT_BAD
- IDENT
- INT_BIN
- INT_DEC
- INT_DEC2
- INT_HEX
- INT_OCT
- INT_OCT_BAD
- NUM_BAD
- SIMPLE_STRING
- SSTRING
Attributes
brace_nest[RW]
cmdarg[RW]
command_start[RW]
command_state[RW]
comments[W]
cond[RW]
extra_lineno[RW]
filename[RW]
last_state[RW]
lex_state[RW]
Additional context surrounding tokens that both the lexer and grammar use.
lex_state=[RW]
Additional context surrounding tokens that both the lexer and grammar use.
lex_strterm[RW]
lineno[RW]
lpar_beg[RW]
match[RW]
old_lineno=[RW]
paren_nest[RW]
parser[RW]
space_seen[RW]
ss[RW]
state[RW]
string_buffer[RW]
string_nest[RW]
token[RW]
Last token read via next_token.
version[RW]
What version of ruby to parse. 18 and 19 are the only valid values currently supported.
Public Class Methods
new(v = 18)
click to toggle source
# File lib/ruby_lexer.rb, line 119 def initialize v = 18 self.version = v @lex_state = :expr_none self.cmdarg = RubyParserStuff::StackState.new(:cmdarg, $DEBUG) self.cond = RubyParserStuff::StackState.new(:cond, $DEBUG) reset end
Public Instance Methods
action() { || ... }
click to toggle source
# File lib/ruby_lexer.rex.rb, line 43 def action yield end
arg_ambiguous()
click to toggle source
# File lib/ruby_lexer.rb, line 129 def arg_ambiguous self.warning("Ambiguous first argument. make sure.") end
arg_state()
click to toggle source
# File lib/ruby_lexer.rb, line 133 def arg_state in_arg_state? ? :expr_arg : :expr_beg end
beginning_of_line?()
click to toggle source
# File lib/ruby_lexer.rb, line 137 def beginning_of_line? ss.bol? end
Also aliased as: bol?
check(re)
click to toggle source
# File lib/ruby_lexer.rb, line 990 def check re ss.check re end
comments()
click to toggle source
# File lib/ruby_lexer.rb, line 142 def comments # TODO: remove this... maybe comment_string + attr_accessor c = @comments.join @comments.clear c end
d(o)
click to toggle source
# File lib/ruby_lexer.rb, line 1330 def d o $stderr.puts o.inspect end
dedent_string(string, width)
click to toggle source
# File lib/ruby_lexer.rb, line 236 def dedent_string(string, width) characters_skipped = 0 indentation_skipped = 0 string.chars.each do |char| break if indentation_skipped >= width if char == ' ' characters_skipped += 1 indentation_skipped += 1 elsif char == "\t" proposed = TAB_WIDTH * (indentation_skipped / TAB_WIDTH + 1) break if (proposed > width) characters_skipped += 1 indentation_skipped = proposed end end string[characters_skipped..-1] end
eat_whitespace()
click to toggle source
# File lib/ruby_lexer.rb, line 994 def eat_whitespace r = scan(/\s+/) self.extra_lineno += r.count("\n") if r r end
end_of_stream?()
click to toggle source
# File lib/ruby_lexer.rb, line 148 def end_of_stream? ss.eos? end
expr_dot?()
click to toggle source
# File lib/ruby_lexer.rb, line 152 def expr_dot? lex_state == :expr_dot end
expr_fname?()
click to toggle source
# File lib/ruby_lexer.rb, line 156 def expr_fname? lex_state == :expr_fname end
expr_result(token, text)
click to toggle source
# File lib/ruby_lexer.rb, line 160 def expr_result token, text cond.push false cmdarg.push false result :expr_beg, token, text end
fixup_lineno(extra = 0)
click to toggle source
# File lib/ruby_lexer.rb, line 1000 def fixup_lineno extra = 0 self.lineno += self.extra_lineno + extra self.extra_lineno = 0 end
heredoc(here)
click to toggle source
# File lib/ruby_lexer.rb, line 166 def heredoc here # TODO: rewrite / remove _, eos, func, last_line = here indent = (func & STR_FUNC_INDENT) != 0 ? "[ \t]*" : nil content_indent = (func & STR_FUNC_ICNTNT) != 0 expand = (func & STR_FUNC_EXPAND) != 0 eos_re = /#{indent}#{Regexp.escape eos}(\r*\n|\z)/ err_msg = "can't match #{eos_re.inspect} anywhere in " rb_compile_error err_msg if end_of_stream? if beginning_of_line? && scan(eos_re) then self.lineno += 1 ss.unread_many last_line # TODO: figure out how to remove this return :tSTRING_END, eos end self.string_buffer = [] if expand then case when scan(/#[$@]/) then ss.pos -= 1 # FIX omg stupid return :tSTRING_DVAR, matched when scan(/#[{]/) then return :tSTRING_DBEG, matched when scan(/#/) then string_buffer << '#' end begin c = tokadd_string func, "\n", nil rb_compile_error err_msg if c == RubyLexer::EOF if c != "\n" then return :tSTRING_CONTENT, string_buffer.join.delete("\r") else string_buffer << scan(/\n/) end rb_compile_error err_msg if end_of_stream? end until check(eos_re) else until check(eos_re) do string_buffer << scan(/.*(\n|\z)/) rb_compile_error err_msg if end_of_stream? end end self.lex_strterm = [:heredoc, eos, func, last_line] string_content = string_buffer.join.delete("\r") string_content = heredoc_dedent(string_content) if content_indent && ruby23plus? return :tSTRING_CONTENT, string_content end
heredoc_dedent(string_content)
click to toggle source
# File lib/ruby_lexer.rb, line 226 def heredoc_dedent(string_content) width = string_content.scan(/^[ \t]*(?=\S)/).map do |whitespace| heredoc_whitespace_indent_size whitespace end.min || 0 string_content.split("\n", -1).map do |line| dedent_string line, width end.join "\n" end
heredoc_identifier()
click to toggle source
# File lib/ruby_lexer.rb, line 265 def heredoc_identifier # TODO: remove / rewrite term, func = nil, STR_FUNC_BORING self.string_buffer = [] heredoc_indent_mods = '-' heredoc_indent_mods += '\~' if ruby23plus? case when scan(/([#{heredoc_indent_mods}]?)([\'\"\`])(.*?)\2/) then term = ss[2] func |= STR_FUNC_INDENT unless ss[1].empty? func |= STR_FUNC_ICNTNT if ss[1] == '~' func |= case term when "\'" then STR_SQUOTE when '"' then STR_DQUOTE else STR_XQUOTE end string_buffer << ss[3] when scan(/[#{heredoc_indent_mods}]?([\'\"\`])(?!\1*\Z)/) then rb_compile_error "unterminated here document identifier" when scan(/([#{heredoc_indent_mods}]?)(#{IDENT_CHAR}+)/) then term = '"' func |= STR_DQUOTE unless ss[1].empty? then func |= STR_FUNC_INDENT func |= STR_FUNC_ICNTNT if ss[1] == '~' end string_buffer << ss[2] else return nil end if scan(/.*\n/) then # TODO: think about storing off the char range instead line = matched else line = nil end self.lex_strterm = [:heredoc, string_buffer.join, func, line] if term == '`' then result nil, :tXSTRING_BEG, "`" else result nil, :tSTRING_BEG, "\"" end end
heredoc_whitespace_indent_size(whitespace)
click to toggle source
# File lib/ruby_lexer.rb, line 255 def heredoc_whitespace_indent_size(whitespace) whitespace.chars.inject 0 do |size, char| if char == "\t" size + TAB_WIDTH else size + 1 end end end
in_arg_state?()
click to toggle source
# File lib/ruby_lexer.rb, line 320 def in_arg_state? # TODO: rename is_after_operator? in_lex_state? :expr_fname, :expr_dot end
in_fname?()
click to toggle source
# File lib/ruby_lexer.rb, line 316 def in_fname? in_lex_state? :expr_fname end
in_lex_state?(*states)
click to toggle source
# File lib/ruby_lexer.rb, line 324 def in_lex_state?(*states) states.include? lex_state end
int_with_base(base)
click to toggle source
# File lib/ruby_lexer.rb, line 328 def int_with_base base rb_compile_error "Invalid numeric format" if matched =~ /__/ text = matched case when text.end_with?('ri') return result(:expr_end, :tIMAGINARY, Complex(0, Rational(text.chop.chop.to_i(base)))) when text.end_with?('r') return result(:expr_end, :tRATIONAL, Rational(text.chop.to_i(base))) when text.end_with?('i') return result(:expr_end, :tIMAGINARY, Complex(0, text.chop.to_i(base))) else return result(:expr_end, :tINTEGER, text.to_i(base)) end end
is_arg?()
click to toggle source
# File lib/ruby_lexer.rb, line 344 def is_arg? in_lex_state? :expr_arg, :expr_cmdarg end
is_beg?()
click to toggle source
# File lib/ruby_lexer.rb, line 348 def is_beg? in_lex_state? :expr_beg, :expr_value, :expr_mid, :expr_class, :expr_labelarg end
is_end?()
click to toggle source
# File lib/ruby_lexer.rb, line 352 def is_end? in_lex_state? :expr_end, :expr_endarg, :expr_endfn end
is_label_possible?()
click to toggle source
# File lib/ruby_lexer.rb, line 360 def is_label_possible? (in_lex_state?(:expr_beg, :expr_endfn) && !command_state) || is_arg? end
is_label_suffix?()
click to toggle source
# File lib/ruby_lexer.rb, line 364 def is_label_suffix? check(/:(?!:)/) end
is_space_arg?(c = "x")
click to toggle source
# File lib/ruby_lexer.rb, line 368 def is_space_arg? c = "x" is_arg? and space_seen and c !~ /\s/ end
lineno=(n)
click to toggle source
# File lib/ruby_lexer.rb, line 1334 def lineno= n self.old_lineno= n where = caller.first.split(/:/).first(2).join(":") d :lineno => [n, where, ss && ss.rest[0,40]] end
location()
click to toggle source
# File lib/ruby_lexer.rex.rb, line 65 def location [ (filename || "<input>"), ].compact.join(":") end
matched()
click to toggle source
# File lib/ruby_lexer.rb, line 372 def matched ss.matched end
matches()
click to toggle source
# File lib/ruby_lexer.rex.rb, line 37 def matches m = (1..9).map { |i| ss[i] } m.pop until m[-1] or m.empty? m end
next_token()
click to toggle source
# File lib/ruby_lexer.rex.rb, line 71 def next_token return process_string if lex_strterm self.command_state = self.command_start self.command_start = false self.space_seen = false self.last_state = lex_state token = nil until ss.eos? or token do token = case state when nil then case when ss.skip(/[\ \t\r\f\v]/) then action { self.space_seen = true; next } when text = ss.scan(/\n|\#/) then process_newline_or_comment text when text = ss.scan(/[\]\)\}]/) then process_bracing text when ss.match?(/\!/) then case when in_arg_state? && (ss.skip(/\!\@/)) then action { result :expr_arg, :tUBANG, "!@" } when text = ss.scan(/\![=~]?/) then action { result :arg_state, TOKENS[text], text } end # group /\!/ when ss.match?(/\./) then case when text = ss.scan(/\.\.\.?/) then action { result :expr_beg, TOKENS[text], text } when ss.skip(/\.\d/) then action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" } when ss.skip(/\./) then action { result :expr_dot, :tDOT, "." } end # group /\./ when text = ss.scan(/\(/) then process_paren text when text = ss.scan(/\,/) then action { result :expr_beg, TOKENS[text], text } when ss.match?(/=/) then case when text = ss.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then action { result arg_state, TOKENS[text], text } when bol? && (text = ss.scan(/\=begin(?=\s)/)) then process_begin text when text = ss.scan(/\=(?=begin\b)/) then action { result arg_state, TOKENS[text], text } end # group /=/ when ruby22_label? && (text = ss.scan(/\"#{SIMPLE_STRING}\":/o)) then process_label text when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then action { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } } when text = ss.scan(/\"/) then action { string STR_DQUOTE; result nil, :tSTRING_BEG, text } when text = ss.scan(/\@\@?\d/) then action { rb_compile_error "`#{text}` is not allowed as a variable name" } when text = ss.scan(/\@\@?#{IDENT_CHAR}+/o) then process_ivar text when ss.match?(/:/) then case when not_end? && (text = ss.scan(/:([a-zA-Z_]#{IDENT_CHAR}*(?:[?]|[!](?!=)|=(?==>)|=(?![=>]))?)/o)) then process_symbol text when not_end? && (text = ss.scan(/\:\"(#{SIMPLE_STRING})\"/o)) then process_symbol text when not_end? && (text = ss.scan(/\:\'(#{SSTRING})\'/o)) then process_symbol text when text = ss.scan(/\:\:/) then process_colon2 text when text = ss.scan(/\:/) then process_colon1 text end # group /:/ when ss.skip(/->/) then action { result :expr_endfn, :tLAMBDA, nil } when text = ss.scan(/[+-]/) then process_plus_minus text when ss.match?(/[+\d]/) then case when ss.skip(/#{NUM_BAD}/o) then action { rb_compile_error "Invalid numeric format" } when ss.skip(/#{INT_DEC}/o) then action { int_with_base 10 } when ss.skip(/#{INT_HEX}/o) then action { int_with_base 16 } when ss.skip(/#{INT_BIN}/o) then action { int_with_base 2 } when ss.skip(/#{INT_OCT_BAD}/o) then action { rb_compile_error "Illegal octal digit." } when ss.skip(/#{INT_OCT}/o) then action { int_with_base 8 } when ss.skip(/#{FLOAT_BAD}/o) then action { rb_compile_error "Trailing '_' in number." } when text = ss.scan(/#{FLOAT}/o) then process_float text when ss.skip(/#{INT_DEC2}/o) then action { int_with_base 10 } when ss.skip(/[0-9]/) then action { rb_compile_error "Bad number format" } end # group /[+\d]/ when text = ss.scan(/\[/) then process_square_bracket text when was_label? && (text = ss.scan(/\'#{SSTRING}\':?/o)) then process_label_or_string text when ss.match?(/\|/) then case when ss.skip(/\|\|\=/) then action { result :expr_beg, :tOP_ASGN, "||" } when ss.skip(/\|\|/) then action { result :expr_beg, :tOROP, "||" } when ss.skip(/\|\=/) then action { result :expr_beg, :tOP_ASGN, "|" } when ss.skip(/\|/) then action { result :arg_state, :tPIPE, "|" } end # group /\|/ when text = ss.scan(/\{/) then process_curly_brace text when ss.match?(/\*/) then case when ss.skip(/\*\*=/) then action { result :expr_beg, :tOP_ASGN, "**" } when ss.skip(/\*\*/) then action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") } when ss.skip(/\*\=/) then action { result(:expr_beg, :tOP_ASGN, "*") } when ss.skip(/\*/) then action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") } end # group /\*/ when ss.match?(/</) then case when ss.skip(/\<\=\>/) then action { result :arg_state, :tCMP, "<=>" } when ss.skip(/\<\=/) then action { result :arg_state, :tLEQ, "<=" } when ss.skip(/\<\<\=/) then action { result :arg_state, :tOP_ASGN, "<<" } when text = ss.scan(/\<\</) then process_lchevron text when ss.skip(/\</) then action { result :arg_state, :tLT, "<" } end # group /</ when ss.match?(/>/) then case when ss.skip(/\>\=/) then action { result :arg_state, :tGEQ, ">=" } when ss.skip(/\>\>=/) then action { result :arg_state, :tOP_ASGN, ">>" } when ss.skip(/\>\>/) then action { result :arg_state, :tRSHFT, ">>" } when ss.skip(/\>/) then action { result :arg_state, :tGT, ">" } end # group />/ when ss.match?(/\`/) then case when expr_fname? && (ss.skip(/\`/)) then action { result(:expr_end, :tBACK_REF2, "`") } when expr_dot? && (ss.skip(/\`/)) then action { result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") } when ss.skip(/\`/) then action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") } end # group /\`/ when text = ss.scan(/\?/) then process_questionmark text when ss.match?(/&/) then case when ss.skip(/\&\&\=/) then action { result(:expr_beg, :tOP_ASGN, "&&") } when ss.skip(/\&\&/) then action { result(:expr_beg, :tANDOP, "&&") } when ss.skip(/\&\=/) then action { result(:expr_beg, :tOP_ASGN, "&" ) } when ss.skip(/\&\./) then action { result(:expr_dot, :tLONELY, "&.") } when text = ss.scan(/\&/) then process_amper text end # group /&/ when text = ss.scan(/\//) then process_slash text when ss.match?(/\^/) then case when ss.skip(/\^=/) then action { result(:expr_beg, :tOP_ASGN, "^") } when ss.skip(/\^/) then action { result(:arg_state, :tCARET, "^") } end # group /\^/ when ss.skip(/\;/) then action { self.command_start = true; result(:expr_beg, :tSEMI, ";") } when ss.match?(/~/) then case when in_arg_state? && (ss.skip(/\~@/)) then action { result(:arg_state, :tTILDE, "~") } when ss.skip(/\~/) then action { result(:arg_state, :tTILDE, "~") } end # group /~/ when ss.match?(/\\/) then case when ss.skip(/\\\r?\n/) then action { self.lineno += 1; self.space_seen = true; next } when ss.skip(/\\/) then action { rb_compile_error "bare backslash only allowed before newline" } end # group /\\/ when text = ss.scan(/\%/) then process_percent text when ss.match?(/\$/) then case when text = ss.scan(/\$_\w+/) then process_gvar text when text = ss.scan(/\$_/) then process_gvar text when text = ss.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then process_gvar text when in_fname? && (text = ss.scan(/\$([\&\`\'\+])/)) then process_gvar text when text = ss.scan(/\$([\&\`\'\+])/) then process_backref text when in_fname? && (text = ss.scan(/\$([1-9]\d*)/)) then process_gvar text when text = ss.scan(/\$([1-9]\d*)/) then process_nthref text when text = ss.scan(/\$0/) then process_gvar text when text = ss.scan(/\$\W|\$\z/) then process_gvar_oddity text when text = ss.scan(/\$\w+/) then process_gvar text end # group /\$/ when text = ss.scan(/\_/) then process_underscore text when text = ss.scan(/#{IDENT}/o) then process_token text when ss.skip(/\004|\032|\000|\Z/) then action { [RubyLexer::EOF, RubyLexer::EOF] } when text = ss.scan(/./) then action { rb_compile_error "Invalid char #{text.inspect} in expression" } else text = ss.string[ss.pos .. -1] raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'" end else raise ScanError, "undefined state at #{location}: '#{state}'" end # token = case state next unless token # allow functions to trigger redo w/ nil end # while raise LexerError, "bad lexical result at #{location}: #{token.inspect}" unless token.nil? || (Array === token && token.size >= 2) # auto-switch state self.state = token.last if token && token.first == :state token end
not_end?()
click to toggle source
# File lib/ruby_lexer.rb, line 376 def not_end? not is_end? end
parse(str)
click to toggle source
# File lib/ruby_lexer.rex.rb, line 51 def parse str self.ss = scanner_class.new str self.state ||= nil do_parse end
parse_file(path)
click to toggle source
# File lib/ruby_lexer.rex.rb, line 58 def parse_file path self.filename = path open path do |f| parse f.read end end
parse_quote()
click to toggle source
# File lib/ruby_lexer.rb, line 1210 def parse_quote # TODO: remove / rewrite beg, nnd, short_hand, c = nil, nil, false, nil if scan(/[a-z0-9]{1,2}/i) then # Long-hand (e.g. %Q{}). rb_compile_error "unknown type of %string" if ss.matched_size == 2 c, beg, short_hand = matched, ss.getch, false else # Short-hand (e.g. %{, %., %!, etc) c, beg, short_hand = 'Q', ss.getch, true end if end_of_stream? or c == RubyLexer::EOF or beg == RubyLexer::EOF then rb_compile_error "unterminated quoted string meets end of file" end # Figure nnd-char. "\0" is special to indicate beg=nnd and that no nesting? nnd = { "(" => ")", "[" => "]", "{" => "}", "<" => ">" }[beg] nnd, beg = beg, "\0" if nnd.nil? token_type, text = nil, "%#{c}#{beg}" token_type, string_type = case c when 'Q' then ch = short_hand ? nnd : c + beg text = "%#{ch}" [:tSTRING_BEG, STR_DQUOTE] when 'q' then [:tSTRING_BEG, STR_SQUOTE] when 'W' then eat_whitespace [:tWORDS_BEG, STR_DQUOTE | STR_FUNC_QWORDS] when 'w' then eat_whitespace [:tQWORDS_BEG, STR_SQUOTE | STR_FUNC_QWORDS] when 'x' then [:tXSTRING_BEG, STR_XQUOTE] when 'r' then [:tREGEXP_BEG, STR_REGEXP] when 's' then self.lex_state = :expr_fname [:tSYMBEG, STR_SSYM] when 'I' then eat_whitespace [:tSYMBOLS_BEG, STR_DQUOTE | STR_FUNC_QWORDS] when 'i' then eat_whitespace [:tQSYMBOLS_BEG, STR_SQUOTE | STR_FUNC_QWORDS] end rb_compile_error "Bad %string type. Expected [QqWwIixrs], found '#{c}'." if token_type.nil? raise "huh" unless string_type string string_type, nnd, beg return token_type, text end
parse_string(quote)
click to toggle source
# File lib/ruby_lexer.rb, line 1267 def parse_string quote # TODO: rewrite / remove _, string_type, term, open = quote space = false # FIX: remove these func = string_type paren = open term_re = @@regexp_cache[term] qwords = (func & STR_FUNC_QWORDS) != 0 regexp = (func & STR_FUNC_REGEXP) != 0 expand = (func & STR_FUNC_EXPAND) != 0 unless func then # nil'ed from qwords below. *sigh* return :tSTRING_END, nil end space = true if qwords and eat_whitespace if self.string_nest == 0 && scan(/#{term_re}/) then if qwords then quote[1] = nil return :tSPACE, nil elsif regexp then return :tREGEXP_END, self.regx_options else return :tSTRING_END, term end end return :tSPACE, nil if space self.string_buffer = [] if expand case when scan(/#(?=\$(-.|[a-zA-Z_0-9~\*\$\?!@\/\\;,\.=:<>\"\&\`\'+]))/) then # TODO: !ISASCII # ?! see parser_peek_variable_name return :tSTRING_DVAR, nil when scan(/#(?=\@\@?[a-zA-Z_])/) then # TODO: !ISASCII return :tSTRING_DVAR, nil when scan(/#[{]/) then return :tSTRING_DBEG, nil when scan(/#/) then string_buffer << '#' end end if tokadd_string(func, term, paren) == RubyLexer::EOF then rb_compile_error "unterminated string meets end of file" end return :tSTRING_CONTENT, string_buffer.join end
process_amper(text)
click to toggle source
# File lib/ruby_lexer.rb, line 380 def process_amper text token = if is_arg? && space_seen && !check(/\s/) then warning("`&' interpreted as argument prefix") :tAMPER elsif in_lex_state? :expr_beg, :expr_mid then :tAMPER else :tAMPER2 end return result(:arg_state, token, "&") end
process_backref(text)
click to toggle source
# File lib/ruby_lexer.rb, line 393 def process_backref text token = ss[1].to_sym # TODO: can't do lineno hack w/ symbol result :expr_end, :tBACK_REF, token end
process_begin(text)
click to toggle source
# File lib/ruby_lexer.rb, line 399 def process_begin text @comments << matched unless scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then @comments.clear rb_compile_error("embedded document meets end of file") end @comments << matched self.lineno += matched.count("\n") nil # TODO end
process_bracing(text)
click to toggle source
# File lib/ruby_lexer.rb, line 413 def process_bracing text cond.lexpop cmdarg.lexpop case matched when "}" then self.brace_nest -= 1 self.lex_state = :expr_endarg # TODO # if (c == '}') { # if (!brace_nest--) c = tSTRING_DEND; # } return :tRCURLY, matched when "]" then self.paren_nest -= 1 self.lex_state = :expr_endarg return :tRBRACK, matched when ")" then self.paren_nest -= 1 self.lex_state = :expr_endfn return :tRPAREN, matched else raise "Unknown bracing: #{matched.inspect}" end end
process_colon1(text)
click to toggle source
# File lib/ruby_lexer.rb, line 441 def process_colon1 text # ?: / then / when if is_end? || check(/\s/) then return result :expr_beg, :tCOLON, text end case when scan(/\'/) then string STR_SSYM when scan(/\"/) then string STR_DSYM end result :expr_fname, :tSYMBEG, text end
process_colon2(text)
click to toggle source
# File lib/ruby_lexer.rb, line 457 def process_colon2 text if is_beg? || in_lex_state?(:expr_class) || is_space_arg? then result :expr_beg, :tCOLON3, text else result :expr_dot, :tCOLON2, text end end
process_curly_brace(text)
click to toggle source
# File lib/ruby_lexer.rb, line 465 def process_curly_brace text self.brace_nest += 1 if lpar_beg && lpar_beg == paren_nest then self.lpar_beg = nil self.paren_nest -= 1 return expr_result(:tLAMBEG, "{") end token = if is_arg? || in_lex_state?(:expr_end, :expr_endfn) then :tLCURLY # block (primary) elsif in_lex_state?(:expr_endarg) then :tLBRACE_ARG # block (expr) else :tLBRACE # hash end self.command_start = true unless token == :tLBRACE return expr_result(token, "{") end
process_float(text)
click to toggle source
# File lib/ruby_lexer.rb, line 487 def process_float text rb_compile_error "Invalid numeric format" if text =~ /__/ case when text.end_with?('ri') return result(:expr_end, :tIMAGINARY, Complex(0, Rational(text.chop.chop))) when text.end_with?('r') return result(:expr_end, :tRATIONAL, Rational(text.chop)) when text.end_with?('i') return result(:expr_end, :tIMAGINARY, Complex(0, text.chop.to_f)) else return result(:expr_end, :tFLOAT, text.to_f) end end
process_gvar(text)
click to toggle source
# File lib/ruby_lexer.rb, line 502 def process_gvar text text.lineno = self.lineno result(:expr_end, :tGVAR, text) end
process_gvar_oddity(text)
click to toggle source
# File lib/ruby_lexer.rb, line 507 def process_gvar_oddity text return result :expr_end, "$", "$" if text == "$" # TODO: wtf is this? rb_compile_error "#{text.inspect} is not allowed as a global variable name" end
process_ivar(text)
click to toggle source
# File lib/ruby_lexer.rb, line 512 def process_ivar text tok_id = text =~ /^@@/ ? :tCVAR : :tIVAR text.lineno = self.lineno return result(:expr_end, tok_id, text) end
process_label(text)
click to toggle source
# File lib/ruby_lexer.rb, line 773 def process_label text symbol = text[1..-3].gsub(ESC) { unescape $1 } result(:expr_labelarg, :tLABEL, [symbol, self.lineno]) end
process_label_or_string(text)
click to toggle source
# File lib/ruby_lexer.rb, line 761 def process_label_or_string text if @was_label && text =~ /:\Z/ then @was_label = nil return process_label text elsif text =~ /:\Z/ then ss.pos -= 1 # put back ":" text = text[0..-2] end result :expr_end, :tSTRING, text[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'") end
process_lchevron(text)
click to toggle source
# File lib/ruby_lexer.rb, line 518 def process_lchevron text if (!in_lex_state?(:expr_dot, :expr_class) && !is_end? && (!is_arg? || space_seen)) then tok = self.heredoc_identifier return tok if tok end return result(:arg_state, :tLSHFT, "\<\<") end
process_newline_or_comment(text)
click to toggle source
# File lib/ruby_lexer.rb, line 529 def process_newline_or_comment text c = matched hit = false if c == '#' then ss.pos -= 1 while scan(/\s*\#.*(\n+|\z)/) do hit = true self.lineno += matched.lines.to_a.size @comments << matched.gsub(/^ +#/, '#').gsub(/^ +$/, '') end return nil if end_of_stream? end self.lineno += 1 unless hit # Replace a string of newlines with a single one self.lineno += matched.lines.to_a.size if scan(/\n+/) return if in_lex_state?(:expr_beg, :expr_value, :expr_class, :expr_fname, :expr_dot, :expr_labelarg) if scan(/([\ \t\r\f\v]*)(\.|&)/) then self.space_seen = true unless ss[1].empty? ss.pos -= 1 return unless check(/\.\./) end self.command_start = true return result(:expr_beg, :tNL, nil) end
process_nthref(text)
click to toggle source
# File lib/ruby_lexer.rb, line 565 def process_nthref text # TODO: can't do lineno hack w/ number result :expr_end, :tNTH_REF, ss[1].to_i end
process_paren(text)
click to toggle source
# File lib/ruby_lexer.rb, line 570 def process_paren text token = if ruby18 then process_paren18 else process_paren19 end self.paren_nest += 1 # TODO: add :expr_label to :expr_beg (set in expr_result below) return expr_result(token, "(") end
process_paren18()
click to toggle source
# File lib/ruby_lexer.rb, line 583 def process_paren18 self.command_start = true token = :tLPAREN2 if in_lex_state? :expr_beg, :expr_mid then token = :tLPAREN elsif space_seen then if in_lex_state? :expr_cmdarg then token = :tLPAREN_ARG elsif in_lex_state? :expr_arg then warning "don't put space before argument parentheses" end else # not a ternary -- do nothing? end token end
process_paren19()
click to toggle source
# File lib/ruby_lexer.rb, line 602 def process_paren19 if is_beg? then :tLPAREN elsif is_space_arg? then :tLPAREN_ARG else :tLPAREN2 # plain '(' in parse.y end end
process_percent(text)
click to toggle source
# File lib/ruby_lexer.rb, line 612 def process_percent text return parse_quote if is_beg? return result(:expr_beg, :tOP_ASGN, "%") if scan(/\=/) return parse_quote if is_arg? && space_seen && ! check(/\s/) return result(:arg_state, :tPERCENT, "%") end
process_plus_minus(text)
click to toggle source
# File lib/ruby_lexer.rb, line 622 def process_plus_minus text sign = matched utype, type = if sign == "+" then [:tUPLUS, :tPLUS] else [:tUMINUS, :tMINUS] end if in_arg_state? then if scan(/@/) then return result(:expr_arg, utype, "#{sign}@") else return result(:expr_arg, type, sign) end end return result(:expr_beg, :tOP_ASGN, sign) if scan(/\=/) if (is_beg? || (is_arg? && space_seen && !check(/\s/))) then arg_ambiguous if is_arg? if check(/\d/) then return nil if utype == :tUPLUS return result(:expr_beg, :tUMINUS_NUM, sign) end return result(:expr_beg, utype, sign) end return result(:expr_beg, type, sign) end
process_questionmark(text)
click to toggle source
# File lib/ruby_lexer.rb, line 654 def process_questionmark text if is_end? then state = ruby18 ? :expr_beg : :expr_value # HACK? return result(state, :tEH, "?") end if end_of_stream? then rb_compile_error "incomplete character syntax: parsed #{text.inspect}" end if check(/\s|\v/) then unless is_arg? then c2 = { " " => 's', "\n" => 'n', "\t" => 't', "\v" => 'v', "\r" => 'r', "\f" => 'f' }[matched] if c2 then warning("invalid character syntax; use ?\\" + c2) end end # ternary state = ruby18 ? :expr_beg : :expr_value # HACK? return result(state, :tEH, "?") elsif check(/\w(?=\w)/) then # ternary, also return result(:expr_beg, :tEH, "?") end c = if scan(/\\/) then self.read_escape else ss.getch end if version == 18 then return result(:expr_end, :tINTEGER, c[0].ord & 0xff) else return result(:expr_end, :tSTRING, c) end end
process_slash(text)
click to toggle source
# File lib/ruby_lexer.rb, line 698 def process_slash text if is_beg? then string STR_REGEXP return result(nil, :tREGEXP_BEG, "/") end if scan(/\=/) then return result(:expr_beg, :tOP_ASGN, "/") end if is_arg? && space_seen then unless scan(/\s/) then arg_ambiguous string STR_REGEXP, "/" return result(nil, :tREGEXP_BEG, "/") end end return result(:arg_state, :tDIVIDE, "/") end
process_square_bracket(text)
click to toggle source
# File lib/ruby_lexer.rb, line 720 def process_square_bracket text self.paren_nest += 1 token = nil if in_arg_state? then case when scan(/\]\=/) then self.paren_nest -= 1 # HACK? I dunno, or bug in MRI return result(:expr_arg, :tASET, "[]=") when scan(/\]/) then self.paren_nest -= 1 # HACK? I dunno, or bug in MRI return result(:expr_arg, :tAREF, "[]") else rb_compile_error "unexpected '['" end elsif is_beg? then token = :tLBRACK elsif is_arg? && space_seen then token = :tLBRACK else token = :tLBRACK2 end return expr_result(token, "[") end
process_string()
click to toggle source
# File lib/ruby_lexer.rb, line 1184 def process_string # TODO: rewrite / remove token = if lex_strterm[0] == :heredoc then self.heredoc lex_strterm else self.parse_string lex_strterm end token_type, c = token if ruby22plus? && token_type == :tSTRING_END && ["'", '"'].include?(c) then if (([:expr_beg, :expr_endfn].include?(lex_state) && !cond.is_in_state) || is_arg?) && is_label_suffix? then scan(/:/) token_type = token[0] = :tLABEL_END end end if [:tSTRING_END, :tREGEXP_END, :tLABEL_END].include? token_type then self.lex_strterm = nil self.lex_state = (token_type == :tLABEL_END) ? :expr_labelarg : :expr_end end return token end
process_symbol(text)
click to toggle source
# File lib/ruby_lexer.rb, line 747 def process_symbol text symbol = match[1].gsub(ESC) { unescape $1 } rb_compile_error "symbol cannot contain '\\0'" if ruby18 && symbol =~ /\0/ return result(:expr_end, :tSYMBOL, symbol) end
process_token(text)
click to toggle source
# File lib/ruby_lexer.rb, line 779 def process_token text # TODO: make this always return [token, lineno] token = self.token = text token << matched if scan(/[\!\?](?!=)/) tok_id = case when token =~ /[!?]$/ then :tFID when in_lex_state?(:expr_fname) && scan(/=(?:(?![~>=])|(?==>))/) then # ident=, not =~ => == or followed by => # TODO test lexing of a=>b vs a==>b token << matched :tIDENTIFIER when token =~ /^[A-Z]/ then :tCONSTANT else :tIDENTIFIER end if !ruby18 and is_label_possible? and is_label_suffix? then scan(/:/) return result(:expr_labelarg, :tLABEL, [token, self.lineno]) end unless in_lex_state? :expr_dot then # See if it is a reserved word. keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses RubyParserStuff::Keyword.keyword18 token else RubyParserStuff::Keyword.keyword19 token end return process_token_keyword keyword if keyword end # unless in_lex_state? :expr_dot # TODO: # if (mb == ENC_CODERANGE_7BIT && lex_state != EXPR_DOT) { state = if is_beg? or is_arg? or in_lex_state? :expr_dot then command_state ? :expr_cmdarg : :expr_arg elsif not ruby18 and in_lex_state? :expr_fname then :expr_endfn else :expr_end end if not [:expr_dot, :expr_fname].include? last_state and self.parser.env[token.to_sym] == :lvar then state = :expr_end end token.lineno = self.lineno # yes, on a string. I know... I know... return result(state, tok_id, token) end
process_token_keyword(keyword)
click to toggle source
# File lib/ruby_lexer.rb, line 836 def process_token_keyword keyword state = keyword.state value = [token, self.lineno] self.command_start = true if state == :expr_beg and lex_state != :expr_fname case when lex_state == :expr_fname then result(state, keyword.id0, keyword.name) when keyword.id0 == :kDO then case when lpar_beg && lpar_beg == paren_nest then self.lpar_beg = nil self.paren_nest -= 1 expr_result(:kDO_LAMBDA, value) when cond.is_in_state then result(state, :kDO_COND, value) when cmdarg.is_in_state && lex_state != :expr_cmdarg then result(state, :kDO_BLOCK, value) when in_lex_state?(:expr_beg, :expr_endarg) then result(state, :kDO_BLOCK, value) else result(state, :kDO, value) end when in_lex_state?(:expr_beg, :expr_value, :expr_labelarg) then result(state, keyword.id0, value) when keyword.id0 != keyword.id1 then result(:expr_beg, keyword.id1, value) else result(state, keyword.id1, value) end end
process_underscore(text)
click to toggle source
# File lib/ruby_lexer.rb, line 870 def process_underscore text ss.unscan # put back "_" if beginning_of_line? && scan(/\__END__(\r?\n|\Z)/) then return [RubyLexer::EOF, RubyLexer::EOF] elsif scan(/\_\w*/) then return process_token matched end end
rb_compile_error(msg)
click to toggle source
# File lib/ruby_lexer.rb, line 880 def rb_compile_error msg msg += ". near line #{self.lineno}: #{ss.rest[/^.*/].inspect}" raise RubyParser::SyntaxError, msg end
read_escape()
click to toggle source
# File lib/ruby_lexer.rb, line 885 def read_escape # TODO: remove / rewrite case when scan(/\\/) then # Backslash '\\' when scan(/n/) then # newline self.extra_lineno -= 1 "\n" when scan(/t/) then # horizontal tab "\t" when scan(/r/) then # carriage-return "\r" when scan(/f/) then # form-feed "\f" when scan(/v/) then # vertical tab "\13" when scan(/a/) then # alarm(bell) "\007" when scan(/e/) then # escape "\033" when scan(/b/) then # backspace "\010" when scan(/s/) then # space " " when scan(/[0-7]{1,3}/) then # octal constant (matched.to_i(8) & 0xFF).chr when scan(/x([0-9a-fA-F]{1,2})/) then # hex constant ss[1].to_i(16).chr when check(/M-\\[\\MCc]/) then scan(/M-\\/) # eat it c = self.read_escape c[0] = (c[0].ord | 0x80).chr c when scan(/M-(.)/) then c = ss[1] c[0] = (c[0].ord | 0x80).chr c when check(/(C-|c)\\[\\MCc]/) then scan(/(C-|c)\\/) # eat it c = self.read_escape c[0] = (c[0].ord & 0x9f).chr c when scan(/C-\?|c\?/) then 127.chr when scan(/(C-|c)(.)/) then c = ss[2] c[0] = (c[0].ord & 0x9f).chr c when scan(/^[89]/i) then # bad octal or hex... MRI ignores them :( matched when scan(/u([0-9a-fA-F]{2,4}|\{[0-9a-fA-F]{2,6}\})/) then [ss[1].delete("{}").to_i(16)].pack("U") when scan(/[McCx0-9]/) || end_of_stream? then rb_compile_error("Invalid escape character syntax") else ss.getch end end
regx_options()
click to toggle source
# File lib/ruby_lexer.rb, line 943 def regx_options # TODO: rewrite / remove good, bad = [], [] if scan(/[a-z]+/) then good, bad = matched.split(//).partition { |s| s =~ /^[ixmonesu]$/ } end unless bad.empty? then rb_compile_error("unknown regexp option%s - %s" % [(bad.size > 1 ? "s" : ""), bad.join.inspect]) end return good.join end
reset()
click to toggle source
# File lib/ruby_lexer.rb, line 958 def reset self.brace_nest = 0 self.command_start = true self.comments = [] self.lex_state = :expr_none self.lex_strterm = nil self.lineno = 1 self.lpar_beg = nil self.paren_nest = 0 self.space_seen = false self.string_nest = 0 self.token = nil self.extra_lineno = 0 self.cmdarg.reset self.cond.reset end
ruby18()
click to toggle source
# File lib/ruby_lexer.rb, line 982 def ruby18 RubyParser::V18 === parser end
ruby22_label?()
click to toggle source
# File lib/ruby_lexer.rb, line 356 def ruby22_label? ruby22plus? and is_label_possible? end
ruby22plus?()
click to toggle source
# File lib/ruby_lexer.rb, line 1176 def ruby22plus? parser.class.version >= 22 end
ruby23plus?()
click to toggle source
# File lib/ruby_lexer.rb, line 1180 def ruby23plus? parser.class.version >= 23 end
scan(re)
click to toggle source
# File lib/ruby_lexer.rb, line 986 def scan re ss.scan re end
scanner_class()
click to toggle source
# File lib/ruby_lexer.rb, line 1005 def scanner_class # TODO: design this out of oedipus_lex. or something. RPStringScanner end
space_vs_beginning(space_type, beg_type, fallback)
click to toggle source
# File lib/ruby_lexer.rb, line 1009 def space_vs_beginning space_type, beg_type, fallback if is_space_arg? check(/./m) then warning "`**' interpreted as argument prefix" space_type elsif is_beg? then beg_type else # TODO: warn_balanced("**", "argument prefix"); fallback end end
string(type, beg = matched, nnd = "\0")
click to toggle source
# File lib/ruby_lexer.rb, line 1021 def string type, beg = matched, nnd = "\0" self.lex_strterm = [:strterm, type, beg, nnd] end
tokadd_escape(term)
click to toggle source
TODO: consider def src= src
raise "bad src: #{src.inspect}" unless String === src @src = RPStringScanner.new(src)
end
# File lib/ruby_lexer.rb, line 1031 def tokadd_escape term # TODO: rewrite / remove case when scan(/\\\n/) then # just ignore when scan(/\\([0-7]{1,3}|x[0-9a-fA-F]{1,2})/) then self.string_buffer << matched when scan(/\\([MC]-|c)(?=\\)/) then self.string_buffer << matched self.tokadd_escape term when scan(/\\([MC]-|c)(.)/) then self.string_buffer << matched when scan(/\\[McCx]/) then rb_compile_error "Invalid escape character syntax" when scan(/\\(.)/m) then self.string_buffer << matched else rb_compile_error "Invalid escape character syntax" end end
tokadd_string(func, term, paren)
click to toggle source
# File lib/ruby_lexer.rb, line 1051 def tokadd_string(func, term, paren) # TODO: rewrite / remove qwords = (func & STR_FUNC_QWORDS) != 0 escape = (func & STR_FUNC_ESCAPE) != 0 expand = (func & STR_FUNC_EXPAND) != 0 regexp = (func & STR_FUNC_REGEXP) != 0 symbol = (func & STR_FUNC_SYMBOL) != 0 paren_re = @@regexp_cache[paren] term_re = @@regexp_cache[term] until end_of_stream? do c = nil handled = true case when paren_re && scan(paren_re) then self.string_nest += 1 when scan(term_re) then if self.string_nest == 0 then ss.pos -= 1 break else self.string_nest -= 1 end when expand && scan(/#(?=[\$\@\{])/) then ss.pos -= 1 break when qwords && scan(/\s/) then ss.pos -= 1 break when expand && scan(/#(?!\n)/) then # do nothing when check(/\\/) then case when qwords && scan(/\\\n/) then string_buffer << "\n" next when qwords && scan(/\\\s/) then c = ' ' when expand && scan(/\\\n/) then next when regexp && check(/\\/) then self.tokadd_escape term next when expand && scan(/\\/) then c = self.read_escape when scan(/\\\n/) then # do nothing when scan(/\\\\/) then string_buffer << '\\' if escape c = '\\' when scan(/\\/) then unless scan(term_re) || paren.nil? || scan(paren_re) then string_buffer << "\\" end else handled = false end # inner /\\/ case else handled = false end # top case unless handled then t = Regexp.escape term x = Regexp.escape(paren) if paren && paren != "\000" re = if qwords then if HAS_ENC then /[^#{t}#{x}\#\0\\\s]+|./ # |. to pick up whatever else /[^#{t}#{x}\#\0\\\s\v]+|./ # argh. 1.8's \s doesn't pick up \v end else /[^#{t}#{x}\#\0\\]+|./ end scan re c = matched rb_compile_error "symbol cannot contain '\\0'" if symbol && c =~ /\0/ end # unless handled c ||= matched string_buffer << c end # until c ||= matched c = RubyLexer::EOF if end_of_stream? return c end
unescape(s)
click to toggle source
# File lib/ruby_lexer.rb, line 1142 def unescape s r = ESCAPES[s] self.extra_lineno += 1 if s == "\n" # eg backslash newline strings self.extra_lineno -= 1 if r && s == "n" # literal \n, not newline return r if r x = case s when /^[0-7]{1,3}/ then ($&.to_i(8) & 0xFF).chr when /^x([0-9a-fA-F]{1,2})/ then $1.to_i(16).chr when /^M-(.)/ then ($1[0].ord | 0x80).chr when /^(C-|c)(.)/ then ($2[0].ord & 0x9f).chr when /^[89a-f]/i then # bad octal or hex... ignore? that's what MRI does :( s when /^[McCx0-9]/ then rb_compile_error("Invalid escape character syntax") when /u([0-9a-fA-F]{2,4}|\{[0-9a-fA-F]{2,6}\})/ then [$1.delete("{}").to_i(16)].pack("U") else s end x.force_encoding "UTF-8" if HAS_ENC x end
warning(s)
click to toggle source
# File lib/ruby_lexer.rb, line 1172 def warning s # do nothing for now end
was_label?()
click to toggle source
# File lib/ruby_lexer.rb, line 756 def was_label? @was_label = ruby22_label? true end