scanners/ruby.rb, ruby_helper.rb: Fixes, changes.

2 files changed, 61 insertions, 19 deletions

diff --git a/lib/coderay/scanners/helpers/ruby_helper.rb b/lib/coderay/scanners/helpers/ruby_helper.rb
index a44ca79..ad649da 100644
--- a/lib/coderay/scanners/helpers/ruby_helper.rb
+++ b/lib/coderay/scanners/helpers/ruby_helper.rb
@@ -12,9 +12,11 @@ module CodeRay module Scanners
 		]

 

 		DEF_KEYWORDS = %w[ def ]

+		UNDEF_KEYWORDS = %w[ undef ]

 		MODULE_KEYWORDS = %w[class module]

 		DEF_NEW_STATE = WordList.new(:initial).

 			add(DEF_KEYWORDS, :def_expected).

+			add(UNDEF_KEYWORDS, :undef_expected).

 			add(MODULE_KEYWORDS, :module_expected)

 

 		IDENTS_ALLOWING_REGEXP = %w[

@@ -32,13 +34,11 @@ module CodeRay module Scanners
 			add(RESERVED_WORDS, :reserved).

 			add(PREDEFINED_CONSTANTS, :pre_constant)

 

-#		IDENT = /[a-zA-Z_][a-zA-Z_0-9]*/

 		IDENT = /[a-z_][\w_]*/i

 

 		METHOD_NAME = / #{IDENT} [?!]? /ox

-		METHOD_NAME_EX = /

-			#{IDENT}[?!=]?  # common methods: split, foo=, empty?, gsub!

-			| \*\*?         # multiplication and power

+		METHOD_NAME_OPERATOR = /

+			\*\*?           # multiplication and power

 			| [-+]@?        # plus, minus

 			| [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde

 			| \[\]=?        # array getter and setter

@@ -46,10 +46,11 @@ module CodeRay module Scanners
 			| <=?>? | >=?   # comparison, rocket operator

 			| ===?          # simple equality and case equality

 		/ox

+		METHOD_NAME_EX = / #{IDENT} [?!=]? | #{METHOD_NAME_OPERATOR} /ox

 		INSTANCE_VARIABLE = / @ #{IDENT} /ox

 		CLASS_VARIABLE = / @@ #{IDENT} /ox

 		OBJECT_VARIABLE = / @@? #{IDENT} /ox

-		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9] | 0[a-zA-Z_0-9]* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox

+		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox

 		PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox

 		VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox

 

@@ -144,7 +145,7 @@ module CodeRay module Scanners
 		FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']

 			

 		class StringState < Struct.new :type, :interpreted, :delim, :heredoc,

-			:paren, :paren_depth, :pattern

+			:paren, :paren_depth, :pattern, :next_state

 			

 			CLOSING_PAREN = Hash[ *%w[

 				( )

@@ -186,7 +187,7 @@ module CodeRay module Scanners
 				delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x

 				h[k] =

 					if interpreted

-						/ (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx

+				/ (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc

 					else

 						/ (?= #{delim_pattern}() | \\ ) /mx

 					end

@@ -203,7 +204,7 @@ module CodeRay module Scanners
 				else

 					pattern = STRING_PATTERN[ [delim, interpreted] ]

 				end

-				super kind, interpreted, delim, heredoc, paren, paren_depth, pattern

+				super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial

 			end

 		end unless defined? StringState

 	

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index 4cb1e58..0946bbf 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -51,7 +51,7 @@ module CodeRay module Scanners
 						match = getch + scan_until(/$/)

 						tokens << [match, :delimiter]

 						tokens << [:close, state.type]

-						state = :initial

+						state = state.next_state

 						next

 					end

 					

@@ -102,7 +102,7 @@ module CodeRay module Scanners
 						end

 						tokens << [:close, state.type]

 						fancy_allowed = regexp_allowed = false

-						state = :initial

+						state = state.next_state

 						

 					when '\\'

 						if state.interpreted

@@ -161,7 +161,10 @@ module CodeRay module Scanners
 							type = :space

 						when ?\n, ?\\

 							type = :space

-							regexp_allowed = m == ?\n

+							if m == ?\n

+								regexp_allowed = true

+								state = :initial if state == :undef_comma_expected

+							end

 							if heredocs

 								unscan  # heredoc scanning needs \n at start

 								state = heredocs.shift

@@ -181,7 +184,10 @@ module CodeRay module Scanners
 						next

 

 					elsif state == :initial

-						if match = scan(/ \.\.?\.? | [-+*=>;,|&!\(\)\[\]~^]+ | [\{\}] | :: /x)

+						

+						# OPERATORS #

+						if (not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [-+;,!\(\)\[\]~^] | [*|&>]{1,2} | [\{\}] | :: /x)) or

+							(last_token_dot and match = scan(/#{METHOD_NAME_OPERATOR}/o))

 							if match !~ / [.\)\]\}] \z/x or match =~ /\.\.\.?/

 								regexp_allowed = fancy_allowed = :set

 							end

@@ -202,12 +208,13 @@ module CodeRay module Scanners
 								end

 							end

 							

+						# IDENTS #

 						elsif match = scan(/#{METHOD_NAME}/o)

 							if last_token_dot

 								type = if match[/^[A-Z]/] then :constant else :ident end

 							else

 								type = IDENT_KIND[match]

-								if type == :ident and match[/^[A-Z]/]

+								if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/]

 									type = :constant

 								elsif type == :reserved

 									state = DEF_NEW_STATE[match]

@@ -219,7 +226,7 @@ module CodeRay module Scanners
 						elsif match = scan(/ ['"] /mx)

 							tokens << [:open, :string]

 							type = :delimiter

-							state = StringState.new :string, match != '\'', match  # important for streaming

+							state = StringState.new :string, match == '"', match  # important for streaming

 							

 						elsif match = scan(/#{INSTANCE_VARIABLE}/o)

 							type = :instance_variable

@@ -237,13 +244,17 @@ module CodeRay module Scanners
 						elsif match = scan(/#{NUMERIC}/o)

 							type = if self[1] then :float else :integer end

 

-						elsif fancy_allowed and match = scan(/#{SYMBOL}/o)

-							case match[1]

+						elsif match = scan(/#{SYMBOL}/o)

+							case delim = match[1]

 							when ?', ?"

 								tokens << [:open, :symbol]

-								state = StringState.new :symbol, match[1] == ?", match[1,1]

+								tokens << [':', :symbol]

+								match = delim.chr

+								type = :delimiter

+								state = StringState.new :symbol, delim == ?", match

+							else

+								type = :symbol

 							end

-							type = :symbol

 							

 						elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)

 							indented = self[1] == '-'

@@ -294,9 +305,39 @@ module CodeRay module Scanners
 						

 					elsif state == :def_expected

 						state = :initial

+						if match = scan(/(?>#{METHOD_NAME_EX})(?!\.|::)/o)

+							type = :method

+						else

+							next

+						end

+

+					elsif state == :undef_expected

+						state = :undef_comma_expected

 						if match = scan(/#{METHOD_NAME_EX}/o)

 							type = :method

+						elsif match = scan(/#{SYMBOL}/o)

+							case delim = match[1]

+							when ?', ?"

+								tokens << [:open, :symbol]

+								tokens << [':', :symbol]

+								match = delim.chr

+								type = :delimiter

+								state = StringState.new :symbol, delim == ?", match

+								state.next_state = :undef_comma_expected

+							else

+								type = :symbol

+							end

 						else

+							state = :initial

+							next

+						end

+	

+					elsif state == :undef_comma_expected

+						if match = scan(/,/)

+							type = :operator

+							state = :undef_expected

+						else

+							state = :initial

 							next

 						end

 

@@ -319,7 +360,7 @@ module CodeRay module Scanners
 					last_token_dot = last_token_dot == :set

 

 					if $DEBUG

-						raise_inspect 'error token %p in line %d' % [tokens.last, line], tokens if not type or type == :error

+						raise_inspect 'error token %p in line %d' % [[match, type], line], tokens if not type or type == :error

 					end

 

 					tokens << [match, type]