Two new encoders: debug and xml.

encoder.rb: new token handling
encoders/statistic.rb: using new handling

ruby_helper.rb: small improvements
ruby.rb:
- escapes in subtoken
- Float detection changed
- some multi-char operators are now scanned as one token
- def and module definition handling changed

bin/coderay: improved, new interface (still in progress)

plugin.rb: more expressive load error message

1 files changed, 9 insertions, 9 deletions

diff --git a/lib/coderay/scanners/helpers/ruby_helper.rb b/lib/coderay/scanners/helpers/ruby_helper.rb
index 241b392..a44ca79 100644
--- a/lib/coderay/scanners/helpers/ruby_helper.rb
+++ b/lib/coderay/scanners/helpers/ruby_helper.rb
@@ -60,19 +60,17 @@ module CodeRay module Scanners
 		QUOTE_TO_TYPE.default = :string

 		

 		REGEXP_MODIFIERS = /[mixounse]*/

-		REGEXP_SYMBOLS = /

-			[|?*+?(){}\[\].^$]

-		/x

+		REGEXP_SYMBOLS = /[|?*+?(){}\[\].^$]/

 

-		DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error

+		DECIMAL = /\d+(?:_\d+)*/

 		OCTAL = /0_?[0-7]+(?:_[0-7]+)*/

 		HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/

 		BINARY = /0b[01]+(?:_[01]+)*/

 

 		EXPONENT = / [eE] [+-]? #{DECIMAL} /ox

-		FLOAT_OR_INT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? )? /ox

-		FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /ox

-		NUMERIC = / #{OCTAL} | #{HEXADECIMAL} | #{BINARY} | #{FLOAT_OR_INT} /ox

+		FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox

+		FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox

+		NUMERIC = / (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} /ox

 

 		SYMBOL = /

 			:

@@ -103,7 +101,7 @@ module CodeRay module Scanners
 			)

 		/mx

 

-		# NOTE: This is not completel correct, but

+		# NOTE: This is not completely correct, but

 		# nobody needs heredoc delimiters ending with \n.

 		HEREDOC_OPEN = /

 			<< (-)?              # $1 = float

@@ -115,7 +113,7 @@ module CodeRay module Scanners
 			)

 		/mx

 

-		RDOC = /

+		RUBYDOC = /

 			=begin (?!\S)

 			.*?

 			(?: \Z | ^=end (?!\S) [^\n]* )

@@ -127,6 +125,8 @@ module CodeRay module Scanners
 			(?: \Z | (?=^\#CODE) )

 		/mx

 

+		RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo

+

 		RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x

 

 		FANCY_START = / % ( [qQwWxsr] | (?![\w\s=]) ) (.) /mox