diff options
Diffstat (limited to 'lib/coderay')
| -rw-r--r-- | lib/coderay/scanners/ruby.rb | 33 | ||||
| -rw-r--r-- | lib/coderay/scanners/ruby/patterns.rb | 8 |
2 files changed, 30 insertions, 11 deletions
diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 721f0e4..b8cba97 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -21,6 +21,10 @@ module Scanners file_extension 'rb' helper :patterns + + if not defined? EncodingError + EncodingError = Class.new Exception + end private def scan_tokens tokens, options @@ -31,9 +35,10 @@ module Scanners state = :initial depth = nil inline_block_stack = [] - + unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8' + patterns = Patterns # avoid constant lookup - + until eos? match = nil kind = nil @@ -161,7 +166,8 @@ module Scanners elsif state == :initial # IDENTS # - if match = scan(/#{patterns::METHOD_NAME}/o) + if match = scan(unicode ? /#{patterns::METHOD_NAME}/uo : + /#{patterns::METHOD_NAME}/o) if last_token_dot kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end else @@ -175,7 +181,7 @@ module Scanners ## experimental! value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o) - elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o) + elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o) kind = :ident value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o) @@ -281,13 +287,19 @@ module Scanners else kind = :error - match = getch + match = (scan(/./mu) rescue nil) || getch + if !unicode && match.size > 1 + unicode = true + unscan + next + end end elsif state == :def_expected state = :initial - if match = scan(/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o) + if match = scan(unicode ? /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/uo : + /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o) kind = :method else next @@ -327,7 +339,14 @@ module Scanners end elsif state == :alias_expected - if match = scan(/(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o) + begin + match = scan(unicode ? /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/uo : + /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o) + rescue EncodingError + raise if $DEBUG + end + + if match tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)] tokens << [self[2], :space] tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)] diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index e303894..cf5f8c1 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -31,18 +31,18 @@ module Scanners add(RESERVED_WORDS, :reserved). add(PREDEFINED_CONSTANTS, :pre_constant) - IDENT = /[a-z_][\w_]*/i + IDENT = /[^\W\d]\w*/ METHOD_NAME = / #{IDENT} [?!]? /ox METHOD_NAME_OPERATOR = / \*\*? # multiplication and power - | [-+~]@? # plus, minus, tilde with and without @ - | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system` + | [-+~]@? # plus, minus, tilde with and without at sign + | [\/%&|^`] # division, modulo or format strings, and, or, xor, system | \[\]=? # array getter and setter | << | >> # append or shift left, shift right | <=?>? | >=? # comparison, rocket operator | ===? | =~ # simple equality, case equality, match - | ![~=@]? # negation with and without @, not-equal and not-match + | ![~=@]? # negation with and without at sign, not-equal and not-match /ox METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox INSTANCE_VARIABLE = / @ #{IDENT} /ox |