From 5c23a731ca55729fc65630eca3b37a5b1a71e5b1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 20 Jul 2013 18:12:04 +0200
Subject: mark possibly problematic spots with FIXME

---
 lib/coderay/scanners/debug.rb | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib/coderay/scanners/debug.rb')

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index 566bfa7..9d10864 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -21,6 +21,7 @@ module Scanners
           encoder.text_token match, :space
           
         elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
+          # FIXME: cache attack
           kind = self[1].to_sym
           match = self[2].gsub(/\\(.)/m, '\1')
           unless TokenKinds.has_key? kind
@@ -30,6 +31,7 @@ module Scanners
           encoder.text_token match, kind
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
+          # FIXME: cache attack
           kind = self[1].to_sym
           opened_tokens << kind
           case self[2]
-- 
cgit v1.2.1


From 2ab42c7b5e674453fac0320fe0c4a40daf6197e1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 18:53:41 +0200
Subject: prevent Symbol attack in Debug scanner

---
 lib/coderay/scanners/debug.rb | 39 ++++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

(limited to 'lib/coderay/scanners/debug.rb')

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index 9d10864..ac12c16 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -1,3 +1,5 @@
+require 'set'
+
 module CodeRay
 module Scanners
   
@@ -11,6 +13,11 @@ module Scanners
     
   protected
     
+    def setup
+      super
+      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
+    end
+    
     def scan_tokens encoder, options
       
       opened_tokens = []
@@ -21,26 +28,24 @@ module Scanners
           encoder.text_token match, :space
           
         elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
-          # FIXME: cache attack
-          kind = self[1].to_sym
-          match = self[2].gsub(/\\(.)/m, '\1')
-          unless TokenKinds.has_key? kind
-            kind = :error
-            match = matched
+          if @known_token_kinds.include? self[1]
+            encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
+          else
+            encoder.text_token matched, :error
           end
-          encoder.text_token match, kind
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          # FIXME: cache attack
-          kind = self[1].to_sym
-          opened_tokens << kind
-          case self[2]
-          when '<'
-            encoder.begin_group kind
-          when '['
-            encoder.begin_line kind
-          else
-            raise 'CodeRay bug: This case should not be reached.'
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
+            opened_tokens << kind
+            case self[2]
+            when '<'
+              encoder.begin_group kind
+            when '['
+              encoder.begin_line kind
+            else
+              raise 'CodeRay bug: This case should not be reached.'
+            end
           end
           
         elsif !opened_tokens.empty? && match = scan(/ > /x)
-- 
cgit v1.2.1


From 5d6bee7f5caced1383e6aac427fb356a4788794b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 20:14:21 +0200
Subject: tweak Debug scanners again, introduce :unknown token kind

---
 lib/coderay/scanners/debug.rb | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

(limited to 'lib/coderay/scanners/debug.rb')

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index ac12c16..83ede9a 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -5,7 +5,7 @@ module Scanners
   
   # = Debug Scanner
   # 
-  # Interprets the output of the Encoders::Debug encoder.
+  # Interprets the output of the Encoders::Debug encoder (basically the inverse function).
   class Debug < Scanner
     
     register_for :debug
@@ -31,21 +31,24 @@ module Scanners
           if @known_token_kinds.include? self[1]
             encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
           else
-            encoder.text_token matched, :error
+            encoder.text_token matched, :unknown
           end
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
           if @known_token_kinds.include? self[1]
             kind = self[1].to_sym
-            opened_tokens << kind
-            case self[2]
-            when '<'
-              encoder.begin_group kind
-            when '['
-              encoder.begin_line kind
-            else
-              raise 'CodeRay bug: This case should not be reached.'
-            end
+          else
+            kind = :unknown
+          end
+          
+          opened_tokens << kind
+          case self[2]
+          when '<'
+            encoder.begin_group kind
+          when '['
+            encoder.begin_line kind
+          else
+            raise 'CodeRay bug: This case should not be reached.'
           end
           
         elsif !opened_tokens.empty? && match = scan(/ > /x)
-- 
cgit v1.2.1