Debug scanner supports line tokens; more cleanups.

4 files changed, 130 insertions, 16 deletions

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index f43564e..f206f4e 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -1,15 +1,19 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Scanners
 
   # = Debug Scanner
+  # 
+  # Parses the output of the Encoders::Debug encoder.
   class Debug < Scanner
 
     include Streamable
     register_for :debug
     file_extension 'raydebug'
     title 'CodeRay Token Dump'
-
+    
   protected
+    
     def scan_tokens tokens, options
 
       opened_tokens = []
@@ -27,23 +31,34 @@ module Scanners
             kind = self[1].to_sym
             match = self[2].gsub(/\\(.)/, '\1')
             
-          elsif scan(/ (\w+) < /x)
+          elsif scan(/ (\w+) ([<\[]) /x)
             kind = self[1].to_sym
             opened_tokens << kind
-            match = :open
+            case self[2]
+            when '<'
+              match = :open
+            when '['
+              match = :begin_line
+            else
+              raise
+            end
             
-          elsif scan(/ > /x)
+          elsif !opened_tokens.empty? && scan(/ > /x)
             kind = opened_tokens.pop
             match = :close
             
+          elsif !opened_tokens.empty? && scan(/ \] /x)
+            kind = opened_tokens.pop
+            match = :end_line
+            
           else
-            kind = :error
+            kind = :space
             getch
 
           end
                   
         match ||= matched
-        if $DEBUG and not kind
+        if $CODERAY_DEBUG and not kind
           raise_inspect 'Error token %p in line %d' %
             [[match, kind], line], tokens
         end
@@ -60,3 +75,48 @@ module Scanners
 
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class DebugScannerTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Scanners::Debug < CodeRay::Scanners::Scanner
+    debug = nil
+    assert_nothing_raised do
+      debug = CodeRay.scanner :debug
+    end
+    assert_kind_of CodeRay::Scanners::Scanner, debug
+  end
+  
+  TEST_INPUT = <<-'DEBUG'.chomp
+integer(10)operator((\\\))string<content(test)>test[
+
+  	   
+method([])]
+  DEBUG
+  TEST_OUTPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:open, :string],
+    ['test', :content],
+    [:close, :string],
+    [:begin_line, :test],
+    ["\n\n  \t   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ]
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Scanners::Debug.new.tokenize(TEST_INPUT)
+    assert_equal TEST_OUTPUT, CodeRay.scan(TEST_INPUT, :debug)
+  end
+  
+end
diff --git a/test/scanners/debug/class.expected.raydebug b/test/scanners/debug/class.expected.raydebug
index c0df683..501c99f 100644
--- a/test/scanners/debug/class.expected.raydebug
+++ b/test/scanners/debug/class.expected.raydebug
@@ -1,14 +1,14 @@
-error(3)error(7)error(,)error(3)error(8)error(c)error(3)error(7)error(,)error(3)error(8)
-error(<) 			reserved(for) ident(input) reserved(in) constant(Dir)operator([)string<delimiter(")content(*.)inline<delimiter(#{)ident(extension)delimiter(})>delimiter(")>operator(])
-error(<) 				ident(name) operator(=) constant(File)operator(.)ident(basename)operator(()ident(input)operator(,) string<delimiter(")content(.)inline<delimiter(#{)ident(extension)delimiter(})>delimiter(")>operator(\))
-error(-)error(-)error(-)
+37,38c37,38
+< 			reserved(for) ident(input) reserved(in) constant(Dir)operator([)string<delimiter(")content(*.)inline<delimiter(#{)ident(extension)delimiter(})>delimiter(")>operator(])
+< 				ident(name) operator(=) constant(File)operator(.)ident(basename)operator(()ident(input)operator(,) string<delimiter(")content(.)inline<delimiter(#{)ident(extension)delimiter(})>delimiter(")>operator(\))
+---
 > 			reserved(for) ident(input) reserved(in) constant(Dir)operator([)string<delimiter(")content(*.)inline<inline_delimiter(#{)ident(extension)inline_delimiter(})>delimiter(")>operator(])
 > 				ident(name) operator(=) constant(File)operator(.)ident(basename)operator(()ident(input)operator(,) string<delimiter(")content(.)inline<inline_delimiter(#{)ident(extension)inline_delimiter(})>delimiter(")>operator(\))
-error(4)error(9)error(c)error(4)error(9)
-error(<) 					ident(puts) string<delimiter(")content(New test: )inline<delimiter(#{)ident(output)delimiter(})>delimiter(")>
-error(-)error(-)error(-)
+49c49
+< 					ident(puts) string<delimiter(")content(New test: )inline<delimiter(#{)ident(output)delimiter(})>delimiter(")>
+---
 > 					ident(puts) string<delimiter(")content(New test: )inline<inline_delimiter(#{)ident(output)inline_delimiter(})>delimiter(")>
-error(6)error(9)error(c)error(6)error(9)
-error(<) error(!)error(!) error(F)error(o)error(l)error(d)error(e)error(r) error(\))inline<delimiter(#{)constant(File)operator(.)ident(split)operator(()pre_constant(__FILE__)operator(\))operator(.)ident(first) operator(+) string<delimiter(')content(/)delimiter(')> operator(+) ident(name)delimiter(})>content( not found
+69c69
+< !! Folder )inline<delimiter(#{)constant(File)operator(.)ident(split)operator(()pre_constant(__FILE__)operator(\))operator(.)ident(first) operator(+) string<delimiter(')content(/)delimiter(')> operator(+) ident(name)delimiter(})>content( not found
 ---
-> !! Folder )inline<inline_delimiter(#{)constant(File)operator(.)ident(split)operator(()pre_constant(__FILE__)operator(\))operator(.)ident(first) operator(+) string<delimiter(')content(/)delimiter(')> operator(+) ident(name)inline_delimiter(})>error(c)error(o)error(n)error(t)error(e)error(n)error(t)error(() error(n)error(o)error(t) error(f)error(o)error(u)error(n)error(d)
+> !! Folder )inline<inline_delimiter(#{)constant(File)operator(.)ident(split)operator(()pre_constant(__FILE__)operator(\))operator(.)ident(first) operator(+) string<delimiter(')content(/)delimiter(')> operator(+) ident(name)inline_delimiter(})>content( not found
diff --git a/test/scanners/debug/lines.expected.raydebug b/test/scanners/debug/lines.expected.raydebug
new file mode 100644
index 0000000..3e836fa
--- /dev/null
+++ b/test/scanners/debug/lines.expected.raydebug
@@ -0,0 +1,27 @@
+head[head(===================================================================)]
+head[head(--- )plain(/Users/jgoebel/rails/pastie/app/controllers/pastes_controller.rb  (revision 1431\))]
+head[head(+++ )plain(/Users/jgoebel/rails/pastie/app/controllers/pastes_controller.rb  (revision 1437\))]
+change[change(@@)plain( -1,6 +1,10 )change(@@)]
+insert[insert(+)plain(require 'login_system')]
+comment( require 'coderay')
+comment( )
+comment( class PastesController < ApplicationController)
+insert[insert(+)plain(  include LoginSystem)]
+insert[insert(+)]
+insert[insert(+)plain(  before_filter :attempt_cookie_login)]
+comment( )
+comment( #  caches_action :recent)
+comment( )
+change[change(@@)plain( -10,11 +14,7 )change(@@)]
+comment( )
+comment(   def show)
+comment(     @paste = Paste.find(params[:id]\))
+delete[delete(-)plain(    if params[:key] and params[:key]==User.new(@paste.nick\).magic_mojo)]
+delete[delete(-)plain(        session[:login]=@paste.nick)]
+delete[delete(-)plain(        return redirect_to(:action => 'show', :id => @paste.id\))]
+delete[delete(-)plain(    end)]
+delete[delete(-)plain(    )]
+insert[insert(+)plain(    attempt_key_login if not logged_in?)]
+comment(     unless @paste.asset or not @paste.body.blank?)
+comment(       render :action => "edit")
+comment(     end)
\ No newline at end of file
diff --git a/test/scanners/debug/lines.in.raydebug b/test/scanners/debug/lines.in.raydebug
new file mode 100644
index 0000000..3e836fa
--- /dev/null
+++ b/test/scanners/debug/lines.in.raydebug
@@ -0,0 +1,27 @@
+head[head(===================================================================)]
+head[head(--- )plain(/Users/jgoebel/rails/pastie/app/controllers/pastes_controller.rb  (revision 1431\))]
+head[head(+++ )plain(/Users/jgoebel/rails/pastie/app/controllers/pastes_controller.rb  (revision 1437\))]
+change[change(@@)plain( -1,6 +1,10 )change(@@)]
+insert[insert(+)plain(require 'login_system')]
+comment( require 'coderay')
+comment( )
+comment( class PastesController < ApplicationController)
+insert[insert(+)plain(  include LoginSystem)]
+insert[insert(+)]
+insert[insert(+)plain(  before_filter :attempt_cookie_login)]
+comment( )
+comment( #  caches_action :recent)
+comment( )
+change[change(@@)plain( -10,11 +14,7 )change(@@)]
+comment( )
+comment(   def show)
+comment(     @paste = Paste.find(params[:id]\))
+delete[delete(-)plain(    if params[:key] and params[:key]==User.new(@paste.nick\).magic_mojo)]
+delete[delete(-)plain(        session[:login]=@paste.nick)]
+delete[delete(-)plain(        return redirect_to(:action => 'show', :id => @paste.id\))]
+delete[delete(-)plain(    end)]
+delete[delete(-)plain(    )]
+insert[insert(+)plain(    attempt_key_login if not logged_in?)]
+comment(     unless @paste.asset or not @paste.body.blank?)
+comment(       render :action => "edit")
+comment(     end)
\ No newline at end of file