All scanners revisited not to produce errors unless in DEBUG mode. [thx@Daniel]

Fixed numerization for input not ending with \n. Added test cases in C scanner tests.
Added WoNáDo as contributer for last Ruby scanner fix.

Updated TODO.

15 files changed, 47 insertions, 24 deletions

diff --git a/README b/README
index a7ef42d..d9ef55a 100644
--- a/README
+++ b/README
@@ -74,7 +74,7 @@ Please report errors in this documentation to <coderay cycnus de>.
 * Doug Kearns and everyone else who worked on ruby.vim - it not only helped me coding CodeRay,
   but also gave me a wonderful target to reach for the Ruby scanner.
 * everyone who used CodeRay on http://www.rubyforen.de and http://www.infhu.de/mx
-* iGEL, magichisoka, manveru and everyone I forgot from rubyforen.de
+* iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de
 * Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de
 * matz, nobu, why, dave, dhh, etc... you know, those Ruby gods and gurus
 * The inventors of: the computer, the internet, the true color display, HTML & CSS, VIM, RUBY,
diff --git a/TODO b/TODO
index 58fa9a9..405af2b 100644
--- a/TODO
+++ b/TODO
@@ -14,12 +14,10 @@ Project:
 		1	2				Code Cleanup: Indentation

 L		2	2				Rewrite Tools:

 							coderay, bench.rb, highlight.rb

-L		2	1				Cleanup Rakefile

 

 

 

 Plugins:

-X		1	1	Sa			Plugin mapping (:cpp --> Scanners::C etc.)

 

 

 Scanners:

diff --git a/lib/coderay/encoders/html/numerization.rb b/lib/coderay/encoders/html/numerization.rb
index 19c760e..1e1f952 100644
--- a/lib/coderay/encoders/html/numerization.rb
+++ b/lib/coderay/encoders/html/numerization.rb
@@ -48,9 +48,6 @@ module Encoders
 						raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every

 					end

 				

-				line_count = count("\n")

-				line_count += 1 unless self[-1] == ?\n

-

 				case mode				

 				when :inline

 					max_width = (start + line_count).to_s.size

@@ -106,6 +103,17 @@ module Encoders
 				self

 			end

 

+			def line_count

+				line_count = count("\n")

+				position_of_last_newline = rindex(?\n)

+				if position_of_last_newline

+					after_last_newline = self[position_of_last_newline + 1 .. -1]

+					ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/]

+					line_count += 1 if not ends_with_newline

+				end

+				line_count

+			end

+

 		end

 

 	end

diff --git a/lib/coderay/scanners/c.rb b/lib/coderay/scanners/c.rb
index ae7ef83..ae0af73 100644
--- a/lib/coderay/scanners/c.rb
+++ b/lib/coderay/scanners/c.rb
@@ -135,7 +135,11 @@ module CodeRay module Scanners
 				end

 				

 				match ||= matched

-				raise_inspect [match, kind], tokens if kind == :error

+				if $DEBUG and (not kind or kind == :error)

+					raise_inspect 'Error token %p in line %d' %

+						[[match, kind], line], tokens

+				end

+				raise_inspect 'Empty token', tokens unless match

 

 				tokens << [match, kind]

 				

diff --git a/lib/coderay/scanners/delphi.rb b/lib/coderay/scanners/delphi.rb
index 77c3839..d02c632 100644
--- a/lib/coderay/scanners/delphi.rb
+++ b/lib/coderay/scanners/delphi.rb
@@ -109,7 +109,11 @@ module CodeRay module Scanners
 				end

 				

 				match ||= matched

-				raise [match, kind], tokens if kind == :error

+				if $DEBUG and (not kind or kind == :error)

+					raise_inspect 'Error token %p in line %d' %

+						[[match, kind], line], tokens

+				end

+				raise_inspect 'Empty token', tokens unless match

 

 				tokens << [match, kind]

 				

diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index a1efa9e..e8eea61 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -145,7 +145,7 @@ module CodeRay module Scanners
 				match ||= matched

 				if $DEBUG and (not kind or kind == :error)

 					raise_inspect 'Error token %p in line %d' %

-					[[match, kind], line], tokens

+						[[match, kind], line], tokens

 				end

 				raise_inspect 'Empty token', tokens unless match

 				

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index 8127944..1dbcb17 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -366,9 +366,11 @@ module CodeRay module Scanners
 					fancy_allowed = fancy_allowed == :set

 					last_token_dot = last_token_dot == :set

 

-					if $DEBUG

-						raise_inspect 'error token %p in line %d' % [[match, type], line], tokens if not type or type == :error

+					if $DEBUG and (not kind or kind == :error)

+						raise_inspect 'Error token %p in line %d' %

+							[[match, kind], line], tokens

 					end

+					raise_inspect 'Empty token', tokens unless match

 

 					tokens << [match, type]

 					

diff --git a/rake_helpers/code_statistics.rb b/rake_helpers/code_statistics.rb
index 28571c4..4c4b65c 100644
--- a/rake_helpers/code_statistics.rb
+++ b/rake_helpers/code_statistics.rb
@@ -60,25 +60,28 @@ private
 			p "Scanning #{file_name}..." if $DEBUG
 			next unless file_name =~ pattern
 
-			lines = codelines = classes = methods = 0
+			lines = codelines = classes = modules = methods = 0
 			empty_lines = comment_lines = 0
-			comment_block = nil
+			in_comment_block = false
 			
 			File.readlines(file_name).each do |line|
 				lines += 1
 				case line
-				when /^=end\s/
+				when /^=end\b/
 					comment_lines += 1
-					comment_block = nil
-				when comment_block
+					in_comment_block = false
+				when in_comment_block
 					comment_lines += 1
 				when /^\s*class\b/: classes += 1
+				when /^\s*module\b/: modules += 1
 				when /^\s*def\b/: methods += 1
 				when /^\s*#/: comment_lines += 1
 				when /^\s*$/: empty_lines += 1
-				when /^=begin\s/
-					comment_block = lines
+				when /^=begin\b/
+					in_comment_block = false
 					comment_lines += 1
+				when /^__END__$/
+					in_comment_block = true
 				end
 			end
 
@@ -87,6 +90,7 @@ private
 			stats[:lines] += lines
 			stats[:codelines] += codelines
 			stats[:classes] += classes
+			stats[:modules] += modules
 			stats[:methods] += methods
 			stats[:files] += 1
 		end
@@ -114,16 +118,16 @@ private
 
 	def print_header
 		print_splitter
-		puts "| T=Test  Name              | Files | Lines |   LOC | Classes | Methods | M/C | LOC/M |"
+		puts "| T=Test  Name              | Files | Lines |   LOC | Classes | Modules | Methods | M/C | LOC/M |"
 		print_splitter
 	end
 
 	def print_splitter
-		puts "+---------------------------+-------+-------+-------+---------+---------+-----+-------+"
+		puts "+---------------------------+-------+-------+-------+---------+---------+---------+-----+-------+"
 	end
 
 	def print_line name, statistics
-		m_over_c = (statistics[:methods] / statistics[:classes]) rescue m_over_c = 0
+		m_over_c = (statistics[:methods] / (statistics[:classes] + statistics[:modules])) rescue m_over_c = 0
 		loc_over_m = (statistics[:codelines] / statistics[:methods]) - 2 rescue loc_over_m = 0
 
 		if name[TEST_TYPES]
@@ -132,8 +136,8 @@ private
 			name = "  #{name}" 
 		end
 
-		line = "| %-25s | %5d | %5d | %5d | %7d | %7d | %3d | %5d |" % (
-			[name, *statistics.values_at(:files, :lines, :codelines, :classes, :methods)] +
+		line = "| %-25s | %5d | %5d | %5d | %7d | %7d | %7d | %3d | %5d |" % (
+			[name, *statistics.values_at(:files, :lines, :codelines, :classes, :modules, :methods)] +
 			[m_over_c, loc_over_m] )
 
 		puts line
diff --git a/test/c/empty.in.c b/test/c/empty.in.c
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/c/empty.in.c
diff --git a/test/c/empty.out.raydebug b/test/c/empty.out.raydebug
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/c/empty.out.raydebug
diff --git a/test/c/error.in.c b/test/c/error.in.c
new file mode 100644
index 0000000..9cad53e
--- /dev/null
+++ b/test/c/error.in.c
@@ -0,0 +1 @@
+\\

diff --git a/test/c/error.out.raydebug b/test/c/error.out.raydebug
new file mode 100644
index 0000000..bc3c350
--- /dev/null
+++ b/test/c/error.out.raydebug
@@ -0,0 +1 @@
+error(\\)\

diff --git a/test/c/error2.in.c b/test/c/error2.in.c
new file mode 100644
index 0000000..1910281
--- /dev/null
+++ b/test/c/error2.in.c
@@ -0,0 +1 @@
+foo
\ No newline at end of file
diff --git a/test/c/error2.out.raydebug b/test/c/error2.out.raydebug
new file mode 100644
index 0000000..ccd2125
--- /dev/null
+++ b/test/c/error2.out.raydebug
@@ -0,0 +1 @@
+ident(foo)
\ No newline at end of file
diff --git a/test/html/tolkien.html b/test/html/tolkien.html
index fd7ae2d..f4dd841 100644
--- a/test/html/tolkien.html
+++ b/test/html/tolkien.html
@@ -284,7 +284,6 @@ ol.CodeRay li { white-space: pre }
 </tt>167<tt>

 </tt>168<tt>

 </tt>169<tt>

-</tt><strong>170</strong><tt>

 </tt></pre></td>

 	<td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><span title="[:preprocessor]" class="pp">&lt;!DOCTYPE html PUBLIC &quot;-//W3C//DTD XHTML 1.0 Transitional//EN&quot; &quot;http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd&quot;&gt;</span><span title="[:space]" class="NO_HIGHLIGHT"><tt>

 </tt></span><span title="[:tag]" class="ta">&lt;html</span><span title="[:space]" class="NO_HIGHLIGHT"> </span><span title="[:attribute_name]" class="an">xmlns</span><span title="[:operator]" class="NO_HIGHLIGHT">=</span><span title="[:string]" class="s"><span title="[:string, :delimiter]" class="dl">&quot;</span><span title="[:string, :content]" class="k">http://www.w3.org/1999/xhtml</span><span title="[:string, :delimiter]" class="dl">&quot;</span></span><span title="[:space]" class="NO_HIGHLIGHT"> </span><span title="[:attribute_name]" class="an">xml:lang</span><span title="[:operator]" class="NO_HIGHLIGHT">=</span><span title="[:string]" class="s"><span title="[:string, :delimiter]" class="dl">&quot;</span><span title="[:string, :content]" class="k">de</span><span title="[:string, :delimiter]" class="dl">&quot;</span></span><span title="[:space]" class="NO_HIGHLIGHT"> </span><span title="[:attribute_name]" class="an">lang</span><span title="[:operator]" class="NO_HIGHLIGHT">=</span><span title="[:string]" class="s"><span title="[:string, :delimiter]" class="dl">&quot;</span><span title="[:string, :content]" class="k">de</span><span title="[:string, :delimiter]" class="dl">&quot;</span></span><span title="[:tag]" class="ta">&gt;</span><span title="[:space]" class="NO_HIGHLIGHT"><tt>