From 46b83c9c5384a299cdc8d8c5381259b1f6d0e4aa Mon Sep 17 00:00:00 2001 From: murphy Date: Thu, 31 Dec 2009 02:56:55 +0000 Subject: Fixed example tests. * test:clean task also deletes .expected.html files * LinesOfCode encoder can deal with tokens that have no scanner. Tests were added for this. * JSON encoder load rubygems if necessary. * NEW :loc as an alias for :lines_of_code * NEW Scanner methods marshal_dump, marshal_load FIXED Tokens dumping (failed while trying to dump @scanner) --- lib/coderay/encoders/_map.rb | 4 ++- lib/coderay/encoders/json.rb | 7 +++- lib/coderay/encoders/lines_of_code.rb | 60 +++++++++++++++++++++++++++++++++-- lib/coderay/scanner.rb | 11 ++++++- rake_tasks/test.rake | 5 ++- sample/css.expected | 6 ++-- sample/div.expected | 6 ++-- sample/dump.expected | 2 +- sample/encoder.expected | 14 ++------ sample/encoder.rb | 2 +- sample/highlight.expected | 21 ++++++------ sample/html.expected | 17 ++++++---- sample/html2.expected | 17 ++++++---- sample/html_list.expected | 17 ++++++---- sample/more.expected | 2 +- sample/suite.rb | 11 ++++--- sample/tokens.expected | 15 +-------- sample/tokens.rb | 2 +- 18 files changed, 144 insertions(+), 75 deletions(-) diff --git a/lib/coderay/encoders/_map.rb b/lib/coderay/encoders/_map.rb index 8e9732b..a66fcb8 100644 --- a/lib/coderay/encoders/_map.rb +++ b/lib/coderay/encoders/_map.rb @@ -1,8 +1,10 @@ module CodeRay module Encoders - map :stats => :statistic, + map \ + :loc => :lines_of_code, :plain => :text, + :stats => :statistic, :tex => :latex end diff --git a/lib/coderay/encoders/json.rb b/lib/coderay/encoders/json.rb index 072f334..7aa077c 100644 --- a/lib/coderay/encoders/json.rb +++ b/lib/coderay/encoders/json.rb @@ -10,7 +10,12 @@ module Encoders protected def setup options - require 'json' + begin + require 'json' + rescue LoadError + require 'rubygems' + require 'json' + end @out = [] end diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb index 32c0192..b24145e 100644 --- a/lib/coderay/encoders/lines_of_code.rb +++ b/lib/coderay/encoders/lines_of_code.rb @@ -1,8 +1,11 @@ +($:.unshift '../..'; require 'coderay') unless defined? CodeRay module CodeRay module Encoders # Counts the LoC (Lines of Code). Returns an Integer >= 0. # + # Alias: :loc + # # Everything that is not comment, markup, doctype/shebang, or an empty line, # is considered to be code. # @@ -11,7 +14,7 @@ module Encoders # * in a Java class without comments, LoC is the number of non-empty lines # # A Scanner class should define the token kinds that are not code in the - # KINDS_NOT_LOC constant. + # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype]. class LinesOfCode < Encoder register_for :lines_of_code @@ -19,7 +22,12 @@ module Encoders NON_EMPTY_LINE = /^\s*\S.*$/ def compile tokens, options - kinds_not_loc = tokens.scanner.class::KINDS_NOT_LOC + if scanner = tokens.scanner + kinds_not_loc = scanner.class::KINDS_NOT_LOC + else + warn ArgumentError, 'Tokens have no scanner.' if $DEBUG + kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC + end code = tokens.token_class_filter :exclude => kinds_not_loc @loc = code.text.scan(NON_EMPTY_LINE).size end @@ -32,3 +40,51 @@ module Encoders end end + +if $0 == __FILE__ + $VERBOSE = true + $: << File.join(File.dirname(__FILE__), '..') + eval DATA.read, nil, $0, __LINE__ + 4 +end + +__END__ +require 'test/unit' + +class LinesOfCodeTest < Test::Unit::TestCase + + def test_creation + assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder + filter = nil + assert_nothing_raised do + filter = CodeRay.encoder :loc + end + assert_kind_of CodeRay::Encoders::LinesOfCode, filter + assert_nothing_raised do + filter = CodeRay.encoder :lines_of_code + end + assert_kind_of CodeRay::Encoders::LinesOfCode, filter + end + + def test_lines_of_code + tokens = CodeRay.scan <<-RUBY, :ruby +#!/usr/bin/env ruby + +# a minimal Ruby program +puts "Hello world!" + RUBY + assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens) + assert_equal 1, tokens.lines_of_code + assert_equal 1, tokens.loc + end + + def test_filtering_block_tokens + tokens = CodeRay::Tokens.new + tokens << ["Hello\n", :world] + tokens << ["Hello\n", :space] + tokens << ["Hello\n", :comment] + assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens) + assert_equal 2, tokens.lines_of_code + assert_equal 2, tokens.loc + end + +end \ No newline at end of file diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 229ba19..8233b2c 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -114,9 +114,10 @@ module CodeRay # # Else, a Tokens object is used. def initialize code='', options = {}, &block - @options = self.class::DEFAULT_OPTIONS.merge options raise "I am only the basic Scanner class. I can't scan "\ "anything. :( Use my subclasses." if self.class == Scanner + + @options = self.class::DEFAULT_OPTIONS.merge options super Scanner.normify(code) @@ -199,6 +200,14 @@ module CodeRay end pos - (string.rindex(?\n, pos) || 0) end + + def marshal_dump + @options + end + + def marshal_load options + @options = options + end protected diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index f9619c7..bb22f36 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -46,6 +46,9 @@ namespace :test do for file in Dir['test/scanners/**/*.actual.*'] rm file end + for file in Dir['test/scanners/**/*.expected.html'] + rm file + end end desc 'run all tests on all supported Ruby platforms' @@ -68,4 +71,4 @@ namespace :test do end task :test => %w( test:functional test:scanners ) -task :samples => 'test:samples' +task :samples => 'test:samples' \ No newline at end of file diff --git a/sample/css.expected b/sample/css.expected index 7c02f53..09709ff 100644 --- a/sample/css.expected +++ b/sample/css.expected @@ -57,7 +57,7 @@ ol.CodeRay li { white-space: pre } .CodeRay .e { color:#666; font-weight:bold } .CodeRay .en { color:#800; font-weight:bold } .CodeRay .er { color:#F00; background-color:#FAA } -.CodeRay .ex { color:#F00; font-weight:bold } +.CodeRay .ex { color:#C00; font-weight:bold } .CodeRay .fl { color:#60E; font-weight:bold } .CodeRay .fu { color:#06B; font-weight:bold } .CodeRay .gv { color:#d70; font-weight:bold } @@ -82,8 +82,8 @@ ol.CodeRay li { white-space: pre } .CodeRay .pc { color:#038; font-weight:bold } .CodeRay .pd { color:#369; font-weight:bold } .CodeRay .pp { color:#579; } -.CodeRay .ps { color:#00C; font-weight: bold; } -.CodeRay .pt { color:#349; font-weight:bold } +.CodeRay .ps { color:#00C; font-weight:bold } +.CodeRay .pt { color:#074; font-weight:bold } .CodeRay .r, .kw { color:#080; font-weight:bold } .CodeRay .ke { color: #808; } diff --git a/sample/div.expected b/sample/div.expected index 9ad26cb..f28ede3 100644 --- a/sample/div.expected +++ b/sample/div.expected @@ -2,14 +2,14 @@ 
for a in 0..255
         a = a.chr
         begin
-                x = eval("?\\#{a}")
+                x = eval("?\\#{a}")
                 if x == a[0]
                         next
                 else
-                        print "#{a}: #{x}"
+                        print "#{a}: #{x}"
                 end
         rescue SyntaxError => boom
-                print "#{a}: error"
+                print "#{a}: error"
         end
         puts
 end
diff --git a/sample/dump.expected b/sample/dump.expected
index 2c24962..a451686 100644
--- a/sample/dump.expected
+++ b/sample/dump.expected
@@ -1,5 +1,5 @@
 YAML: 2358 bytes
-Dump: 1058 bytes
+Dump: 1109 bytes
 undumped:
 

   
require 'coderay'
diff --git a/sample/encoder.expected b/sample/encoder.expected
index c221d25..438032a 100644
--- a/sample/encoder.expected
+++ b/sample/encoder.expected
@@ -19,15 +19,7 @@ Token Types (4):
 
 
 Original text:
-ident	puts
-space	 
-integer	17
-space	 
-operator	+
-space	 
-integer	4
-space	\
-
+[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}]
 
 YAML:
 --- 
@@ -51,8 +43,8 @@ YAML:
   - :space
 
 Dump:
-"x\332\355\314;\n\302@\024\205aP\311c\320\316\005\004[+A\020\356\224\331\201\330\245\n\346\"A\230\0312c\341\356M\"\242k\220\277\272\217\303wVE-\333\332wzn\237\"\027\177W\027\233E\265l\362]\031\036)\212\351;ui<\263JL\f\355U\307=?\234d\335\273\2447\035\346\310\346\323\330\313\306\a\035\332\344\177\277G[L\303\314\327\\j\263oputs <<HTML
 <html>
 <head>
-#{output.stylesheet true}
+#{output.stylesheet true}
 <body>
-#{output}
+#{output}
 </body>
 </html>
 HTML
diff --git a/sample/html.expected b/sample/html.expected
index 25616dc..e98d589 100644
--- a/sample/html.expected
+++ b/sample/html.expected
@@ -50,9 +50,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .ch .dl { color:#039 }
 
 .CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
 .CodeRay .co { color:#036; font-weight:bold }
 .CodeRay .cr { color:#0A0 }
 .CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
 .CodeRay .df { color:#099; font-weight:bold }
 .CodeRay .di { color:#088; font-weight:bold }
 .CodeRay .dl { color:black }
@@ -62,7 +64,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -70,10 +72,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .i  { color:#00D; font-weight:bold }
 .CodeRay .ic { color:#B44; font-weight:bold }
 
-.CodeRay .il { background: #eee; color: black }
-.CodeRay .il .il { background: #ddd }
-.CodeRay .il .il .il { background: #ccc }
-.CodeRay .il .idl { font-weight: bold; color: #777 }
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .CodeRay .im { color:#f00; }
 .CodeRay .in { color:#B2B; font-weight:bold }
@@ -86,8 +89,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
diff --git a/sample/html2.expected b/sample/html2.expected
index 829ffbf..c8ae56a 100644
--- a/sample/html2.expected
+++ b/sample/html2.expected
@@ -50,9 +50,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .ch .dl { color:#039 }
 
 .CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
 .CodeRay .co { color:#036; font-weight:bold }
 .CodeRay .cr { color:#0A0 }
 .CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
 .CodeRay .df { color:#099; font-weight:bold }
 .CodeRay .di { color:#088; font-weight:bold }
 .CodeRay .dl { color:black }
@@ -62,7 +64,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -70,10 +72,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .i  { color:#00D; font-weight:bold }
 .CodeRay .ic { color:#B44; font-weight:bold }
 
-.CodeRay .il { background: #eee; color: black }
-.CodeRay .il .il { background: #ddd }
-.CodeRay .il .il .il { background: #ccc }
-.CodeRay .il .idl { font-weight: bold; color: #777 }
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .CodeRay .im { color:#f00; }
 .CodeRay .in { color:#B2B; font-weight:bold }
@@ -86,8 +89,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
diff --git a/sample/html_list.expected b/sample/html_list.expected
index 5351672..a4092c8 100644
--- a/sample/html_list.expected
+++ b/sample/html_list.expected
@@ -50,9 +50,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .ch .dl { color:#039 }
 
 .CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
 .CodeRay .co { color:#036; font-weight:bold }
 .CodeRay .cr { color:#0A0 }
 .CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
 .CodeRay .df { color:#099; font-weight:bold }
 .CodeRay .di { color:#088; font-weight:bold }
 .CodeRay .dl { color:black }
@@ -62,7 +64,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -70,10 +72,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .i  { color:#00D; font-weight:bold }
 .CodeRay .ic { color:#B44; font-weight:bold }
 
-.CodeRay .il { background: #eee; color: black }
-.CodeRay .il .il { background: #ddd }
-.CodeRay .il .il .il { background: #ccc }
-.CodeRay .il .idl { font-weight: bold; color: #777 }
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .CodeRay .im { color:#f00; }
 .CodeRay .in { color:#B2B; font-weight:bold }
@@ -86,8 +89,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
diff --git a/sample/more.expected b/sample/more.expected
index 61fabe6..196904d 100644
--- a/sample/more.expected
+++ b/sample/more.expected
@@ -1,2 +1,2 @@
-Input: 4983B, Output: 23316B
+Input: 4983B, Output: 23484B
 Take a look with your browser.
diff --git a/sample/suite.rb b/sample/suite.rb
index c586697..fa24114 100644
--- a/sample/suite.rb
+++ b/sample/suite.rb
@@ -28,16 +28,19 @@ class CodeRaySuite < TestCase
         name = File.basename(input, ".rb")
         output = name + '.expected'
         code = File.open(input, 'rb') { |f| break f.read }
-
+        
         result = `ruby -wI../lib #{input}`
-
+        
+        diff = output.sub '.expected', '.diff'
+        File.delete diff if File.exist? diff
+        computed = output.sub '.expected', '.actual'
         if File.exist? output
           expected = File.read output
           ok = expected == result
-          computed = output.sub('.expected', '.actual')
           unless ok
             File.open(computed, 'w') { |f| f.write result }
-            print `diff #{output} #{computed}` if $DEBUG
+            `diff #{output} #{computed} > #{diff}` if $DEBUG
+            puts "Test failed; output written to #{diff}."
           end
           assert(ok, "Output error: #{computed} != #{output}") unless $DEBUG
         else
diff --git a/sample/tokens.expected b/sample/tokens.expected
index bbc2b94..747904e 100644
--- a/sample/tokens.expected
+++ b/sample/tokens.expected
@@ -1,14 +1 @@
-ident	puts
-space	 
-integer	3
-space	 
-operator	+
-space	 
-integer	4
-operator	,
-space	 
-:open	string	
-delimiter	'
-content	3 + 4
-delimiter	'
-:close	string	
+[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]]
diff --git a/sample/tokens.rb b/sample/tokens.rb
index eb8d448..91b8abb 100644
--- a/sample/tokens.rb
+++ b/sample/tokens.rb
@@ -1,3 +1,3 @@
 require 'coderay'
 
-puts CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby).tokens
+p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby)
-- 
cgit v1.2.1