From e4c414691d42f374c1802e3ac9a34e43228284e8 Mon Sep 17 00:00:00 2001 From: murphy Date: Sat, 29 Oct 2005 04:56:28 +0000 Subject: scanners/ruby.rb, ruby_helper.rb: Fixes, changes. --- lib/coderay/scanners/helpers/ruby_helper.rb | 17 ++++---- lib/coderay/scanners/ruby.rb | 63 ++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 19 deletions(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/helpers/ruby_helper.rb b/lib/coderay/scanners/helpers/ruby_helper.rb index a44ca79..ad649da 100644 --- a/lib/coderay/scanners/helpers/ruby_helper.rb +++ b/lib/coderay/scanners/helpers/ruby_helper.rb @@ -12,9 +12,11 @@ module CodeRay module Scanners ] DEF_KEYWORDS = %w[ def ] + UNDEF_KEYWORDS = %w[ undef ] MODULE_KEYWORDS = %w[class module] DEF_NEW_STATE = WordList.new(:initial). add(DEF_KEYWORDS, :def_expected). + add(UNDEF_KEYWORDS, :undef_expected). add(MODULE_KEYWORDS, :module_expected) IDENTS_ALLOWING_REGEXP = %w[ @@ -32,13 +34,11 @@ module CodeRay module Scanners add(RESERVED_WORDS, :reserved). add(PREDEFINED_CONSTANTS, :pre_constant) -# IDENT = /[a-zA-Z_][a-zA-Z_0-9]*/ IDENT = /[a-z_][\w_]*/i METHOD_NAME = / #{IDENT} [?!]? /ox - METHOD_NAME_EX = / - #{IDENT}[?!=]? # common methods: split, foo=, empty?, gsub! - | \*\*? # multiplication and power + METHOD_NAME_OPERATOR = / + \*\*? # multiplication and power | [-+]@? # plus, minus | [\/%&|^`~] # division, modulo or format strings, &and, |or, ^xor, `system`, tilde | \[\]=? # array getter and setter @@ -46,10 +46,11 @@ module CodeRay module Scanners | <=?>? | >=? # comparison, rocket operator | ===? # simple equality and case equality /ox + METHOD_NAME_EX = / #{IDENT} [?!=]? | #{METHOD_NAME_OPERATOR} /ox INSTANCE_VARIABLE = / @ #{IDENT} /ox CLASS_VARIABLE = / @@ #{IDENT} /ox OBJECT_VARIABLE = / @@? #{IDENT} /ox - GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9] | 0[a-zA-Z_0-9]* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox + GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox @@ -144,7 +145,7 @@ module CodeRay module Scanners FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q'] class StringState < Struct.new :type, :interpreted, :delim, :heredoc, - :paren, :paren_depth, :pattern + :paren, :paren_depth, :pattern, :next_state CLOSING_PAREN = Hash[ *%w[ ( ) @@ -186,7 +187,7 @@ module CodeRay module Scanners delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x h[k] = if interpreted - / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx + / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx # $1 set == end of heredoc else / (?= #{delim_pattern}() | \\ ) /mx end @@ -203,7 +204,7 @@ module CodeRay module Scanners else pattern = STRING_PATTERN[ [delim, interpreted] ] end - super kind, interpreted, delim, heredoc, paren, paren_depth, pattern + super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial end end unless defined? StringState diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 4cb1e58..0946bbf 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -51,7 +51,7 @@ module CodeRay module Scanners match = getch + scan_until(/$/) tokens << [match, :delimiter] tokens << [:close, state.type] - state = :initial + state = state.next_state next end @@ -102,7 +102,7 @@ module CodeRay module Scanners end tokens << [:close, state.type] fancy_allowed = regexp_allowed = false - state = :initial + state = state.next_state when '\\' if state.interpreted @@ -161,7 +161,10 @@ module CodeRay module Scanners type = :space when ?\n, ?\\ type = :space - regexp_allowed = m == ?\n + if m == ?\n + regexp_allowed = true + state = :initial if state == :undef_comma_expected + end if heredocs unscan # heredoc scanning needs \n at start state = heredocs.shift @@ -181,7 +184,10 @@ module CodeRay module Scanners next elsif state == :initial - if match = scan(/ \.\.?\.? | [-+*=>;,|&!\(\)\[\]~^]+ | [\{\}] | :: /x) + + # OPERATORS # + if (not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [-+;,!\(\)\[\]~^] | [*|&>]{1,2} | [\{\}] | :: /x)) or + (last_token_dot and match = scan(/#{METHOD_NAME_OPERATOR}/o)) if match !~ / [.\)\]\}] \z/x or match =~ /\.\.\.?/ regexp_allowed = fancy_allowed = :set end @@ -202,12 +208,13 @@ module CodeRay module Scanners end end + # IDENTS # elsif match = scan(/#{METHOD_NAME}/o) if last_token_dot type = if match[/^[A-Z]/] then :constant else :ident end else type = IDENT_KIND[match] - if type == :ident and match[/^[A-Z]/] + if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/] type = :constant elsif type == :reserved state = DEF_NEW_STATE[match] @@ -219,7 +226,7 @@ module CodeRay module Scanners elsif match = scan(/ ['"] /mx) tokens << [:open, :string] type = :delimiter - state = StringState.new :string, match != '\'', match # important for streaming + state = StringState.new :string, match == '"', match # important for streaming elsif match = scan(/#{INSTANCE_VARIABLE}/o) type = :instance_variable @@ -237,13 +244,17 @@ module CodeRay module Scanners elsif match = scan(/#{NUMERIC}/o) type = if self[1] then :float else :integer end - elsif fancy_allowed and match = scan(/#{SYMBOL}/o) - case match[1] + elsif match = scan(/#{SYMBOL}/o) + case delim = match[1] when ?', ?" tokens << [:open, :symbol] - state = StringState.new :symbol, match[1] == ?", match[1,1] + tokens << [':', :symbol] + match = delim.chr + type = :delimiter + state = StringState.new :symbol, delim == ?", match + else + type = :symbol end - type = :symbol elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o) indented = self[1] == '-' @@ -294,9 +305,39 @@ module CodeRay module Scanners elsif state == :def_expected state = :initial + if match = scan(/(?>#{METHOD_NAME_EX})(?!\.|::)/o) + type = :method + else + next + end + + elsif state == :undef_expected + state = :undef_comma_expected if match = scan(/#{METHOD_NAME_EX}/o) type = :method + elsif match = scan(/#{SYMBOL}/o) + case delim = match[1] + when ?', ?" + tokens << [:open, :symbol] + tokens << [':', :symbol] + match = delim.chr + type = :delimiter + state = StringState.new :symbol, delim == ?", match + state.next_state = :undef_comma_expected + else + type = :symbol + end else + state = :initial + next + end + + elsif state == :undef_comma_expected + if match = scan(/,/) + type = :operator + state = :undef_expected + else + state = :initial next end @@ -319,7 +360,7 @@ module CodeRay module Scanners last_token_dot = last_token_dot == :set if $DEBUG - raise_inspect 'error token %p in line %d' % [tokens.last, line], tokens if not type or type == :error + raise_inspect 'error token %p in line %d' % [[match, type], line], tokens if not type or type == :error end tokens << [match, type] -- cgit v1.2.1