From d9d6dd5f4a73363ea5d353ecda142f77ed4eba5a Mon Sep 17 00:00:00 2001
From: Gavin Kistner <gkistner@nvidia.com>
Date: Thu, 6 Jun 2013 10:43:19 -0600
Subject: Support CDATA blocks in HTML/XML

---
 lib/coderay/scanners/html.rb | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 3ba3b79..fcf249a 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -99,7 +99,17 @@ module Scanners
           case state
           
           when :initial
-            if match = scan(/<!--(?:.*?-->|.*)/m)
+            if match = scan(/<!\[CDATA\[/)
+              encoder.begin_group :string
+              encoder.text_token match, :delimiter
+              if match = scan(/.*?\]\]>/m)
+                encoder.text_token match[0..-4], :content
+                encoder.text_token ']]>', :delimiter
+              else
+                encoder.text_token scan(/.*/m), :error
+              end
+              encoder.end_group :string
+            elsif match = scan(/<!--(?:.*?-->|.*)/m)
               encoder.text_token match, :comment
             elsif match = scan(/<!(\w+)(?:.*?>|.*)|\]>/m)
               encoder.text_token match, :doctype
-- 
cgit v1.2.1


From fa9848b6dbd95a4a97a3e63cb99ad2d4c26516df Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:37:06 +0200
Subject: avoid empty tokens in CSS

---
 lib/coderay/scanners/css.rb  | 2 +-
 lib/coderay/scanners/sass.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 732f9c5..6c52c8a 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -148,7 +148,7 @@ module Scanners
             encoder.text_token match[start.size..-2], :content
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if start.size < match.size
           end
           encoder.end_group :function
           
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 167051d..65d40b0 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -176,7 +176,7 @@ module Scanners
             encoder.text_token match[start.size..-2], :content
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if start.size < match.size
           end
           encoder.end_group :function
           
-- 
cgit v1.2.1


From bedd4e4fb5a18270a742e71b65f88fb8e3182050 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:40:56 +0200
Subject: avoid empty tokens in Raydebug

---
 lib/coderay/scanners/raydebug.rb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index 7a21354..d39d962 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -1,11 +1,11 @@
 module CodeRay
 module Scanners
-
+  
   # = Debug Scanner
   # 
   # Parses the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
-
+    
     register_for :raydebug
     file_extension 'raydebug'
     title 'CodeRay Token Dump'
@@ -13,11 +13,11 @@ module Scanners
   protected
     
     def scan_tokens encoder, options
-
+      
       opened_tokens = []
-
+      
       until eos?
-
+        
         if match = scan(/\s+/)
           encoder.text_token match, :space
           
@@ -26,7 +26,7 @@ module Scanners
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
-          encoder.text_token match, kind.to_sym
+          encoder.text_token match, kind.to_sym unless match.empty?
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
@@ -59,8 +59,8 @@ module Scanners
       
       encoder
     end
-
+    
   end
-
+  
 end
 end
-- 
cgit v1.2.1


From 39c074f171eeca1c91aa89757a3f3a12fc78323b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:42:32 +0200
Subject: avoid empty tokens in YAML

---
 lib/coderay/scanners/yaml.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb
index 96f4e93..358a3f1 100644
--- a/lib/coderay/scanners/yaml.rb
+++ b/lib/coderay/scanners/yaml.rb
@@ -47,7 +47,7 @@ module Scanners
           when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/)
             encoder.begin_group :string
             encoder.text_token match, :delimiter
-            encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
+            encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty?
             encoder.text_token match, :delimiter if match = scan(/"/)
             encoder.end_group :string
             next
-- 
cgit v1.2.1


From e7df328cb49fd09a4e5d0f02e877e3270a8703ed Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 05:53:53 +0200
Subject: DebugLint check open tokens at the end

---
 lib/coderay/encoders/debug_lint.rb | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
index 0ac89ef..17a0795 100644
--- a/lib/coderay/encoders/debug_lint.rb
+++ b/lib/coderay/encoders/debug_lint.rb
@@ -19,11 +19,6 @@ module Encoders
     EmptyToken = Class.new InvalidTokenStream
     IncorrectTokenGroupNesting = Class.new InvalidTokenStream
     
-    def initialize options = {}
-      super
-      @opened = []
-    end
-    
     def text_token text, kind
       raise EmptyToken, 'empty token' if text.empty?
       super
@@ -35,7 +30,8 @@ module Encoders
     end
     
     def end_group kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
       super
     end
     
@@ -45,7 +41,20 @@ module Encoders
     end
     
     def end_line kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+      super
+    end
+    
+    protected
+    
+    def setup options
+      super
+      @opened = []
+    end
+    
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
       super
     end
     
-- 
cgit v1.2.1


From d6cc5767c9a9bfb4c08bcd5127145b38cb1d29de Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 05:54:31 +0200
Subject: HTML encoder shouldn't warn about open tokens

---
 lib/coderay/encoders/html.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index b897f5e..20f2409 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -193,7 +193,6 @@ module Encoders
     
     def finish options
       unless @opened.empty?
-        warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
         @out << '</span>' while @opened.pop
         @last_opened = nil
       end
-- 
cgit v1.2.1


From b059a63e7ff3cd7c26dd5664bf048aa42338fbe1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 05:54:56 +0200
Subject: close open tokens in Groovy scanner

---
 lib/coderay/scanners/groovy.rb | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb
index cf55daf..d04a535 100644
--- a/lib/coderay/scanners/groovy.rb
+++ b/lib/coderay/scanners/groovy.rb
@@ -36,9 +36,12 @@ module Scanners
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     def scan_tokens encoder, options
-      
-      state = :initial
+      state = options[:state] || @state
       inline_block_stack = []
       inline_block_paren_depth = nil
       string_delimiter = nil
@@ -246,6 +249,16 @@ module Scanners
         encoder.end_group state
       end
       
+      if options[:keep_state]
+        @state = state
+      end
+      
+      until inline_block_stack.empty?
+        state, = *inline_block_stack.pop
+        encoder.end_group :inline
+        encoder.end_group state
+      end
+      
       encoder
     end
     
-- 
cgit v1.2.1


From 3c3060acaf2422c23784539b270e365decc9cb2e Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 05:55:23 +0200
Subject: close open tokens in PHP scanner

---
 lib/coderay/scanners/php.rb | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
index 6c68834..40037f9 100644
--- a/lib/coderay/scanners/php.rb
+++ b/lib/coderay/scanners/php.rb
@@ -500,6 +500,10 @@ module Scanners
         
       end
       
+      while state = states.pop
+        encoder.end_group :string if [:sqstring, :dqstring].include? state
+      end
+      
       encoder
     end
     
-- 
cgit v1.2.1


From 7f45168efd1abe679b261a7a91f3641984091d12 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:06:16 +0200
Subject: remove instance variables from Lua scanner

---
 lib/coderay/scanners/lua.rb | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 25bebbe..185c133 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -59,13 +59,15 @@ module Scanners
     # CodeRay entry hook. Starts parsing.
     def scan_tokens(encoder, options)
       state = options[:state] || @state
+      brace_depth = @brace_depth
+      num_equals = nil
       
       until eos?
         case state
         
         when :initial
           if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
-            @num_equals = match.count("=") # Number must match for comment end
+            num_equals = match.count("=") # Number must match for comment end
             encoder.begin_group(:comment)
             encoder.text_token(match, :delimiter)
             state = :long_comment
@@ -74,7 +76,7 @@ module Scanners
             encoder.text_token(match, :comment)
           
           elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
-            @num_equals = match.count("=") # Number must match for comment end
+            num_equals = match.count("=") # Number must match for comment end
             encoder.begin_group(:string)
             encoder.text_token(match, :delimiter)
             state = :long_string
@@ -101,19 +103,19 @@ module Scanners
           
           elsif match = scan(/\{/) # Opening table brace {
             encoder.begin_group(:map)
-            encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
-            @brace_depth += 1
+            encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter)
+            brace_depth += 1
             state        = :map
           
           elsif match = scan(/\}/) # Closing table brace }
-            if @brace_depth == 1
-              @brace_depth = 0
+            if brace_depth == 1
+              brace_depth = 0
               encoder.text_token(match, :delimiter)
               encoder.end_group(:map)
-            elsif @brace_depth == 0 # Mismatched brace
+            elsif brace_depth == 0 # Mismatched brace
               encoder.text_token(match, :error)
             else
-              @brace_depth -= 1
+              brace_depth -= 1
               encoder.text_token(match, :inline_delimiter)
               encoder.end_group(:map)
               state = :map
@@ -122,7 +124,7 @@ module Scanners
           elsif match = scan(/["']/) # String delimiters " and '
             encoder.begin_group(:string)
             encoder.text_token(match, :delimiter)
-            @start_delim = match
+            start_delim = match
             state       = :string
           
                             # ↓Prefix                hex number ←|→ decimal number
@@ -146,7 +148,7 @@ module Scanners
           # It may be that we’re scanning a full-blown subexpression of a table
           # (tables can contain full expressions in parts).
           # If this is the case, return to :map scanning state.
-          state = :map if state == :initial && @brace_depth >= 1
+          state = :map if state == :initial && brace_depth >= 1
         
         when :function_expected
           if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
@@ -198,10 +200,10 @@ module Scanners
           end
         
         when :long_comment
-          if match = scan(/.*?(?=\]={#@num_equals}\])/m)
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m)
             encoder.text_token(match, :content)
             
-            delim = scan(/\]={#@num_equals}\]/)
+            delim = scan(/\]={#{num_equals}}\]/)
             encoder.text_token(delim, :delimiter)
           else # No terminator found till EOF
             encoder.text_token(rest, :error)
@@ -211,10 +213,10 @@ module Scanners
           state = :initial
         
         when :long_string
-          if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences
             encoder.text_token(match, :content)
             
-            delim = scan(/\]={#@num_equals}\]/)
+            delim = scan(/\]={#{num_equals}}\]/)
             encoder.text_token(delim, :delimiter)
           else # No terminator found till EOF
             encoder.text_token(rest, :error)
@@ -224,11 +226,11 @@ module Scanners
           state = :initial
         
         when :string
-          if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+          if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
             encoder.text_token(match, :content)
           elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
             encoder.text_token(match, :char)
-          elsif match = scan(Regexp.compile(@start_delim))
+          elsif match = scan(Regexp.compile(start_delim))
             encoder.text_token(match, :delimiter)
             encoder.end_group(:string)
             state = :initial
-- 
cgit v1.2.1


From 45bb0c576b67b7a3c0dead02078b3a16b5583154 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:06:52 +0200
Subject: close open token groups in Lua scanner

---
 lib/coderay/scanners/lua.rb | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 185c133..fb1e45a 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -268,6 +268,9 @@ module Scanners
         @state = state
       end
       
+      encoder.end_group :string if [:string].include? state
+      brace_depth.times { encoder.end_group :map }
+      
       encoder
     end
     
-- 
cgit v1.2.1


From dc6071129cdc1bcd15129147bbc4d92ba870f007 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:13:32 +0200
Subject: close open token groups in Sass scanner

---
 lib/coderay/scanners/sass.rb | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 65d40b0..0ba383f 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -218,6 +218,14 @@ module Scanners
         @state = states
       end
       
+      while state = states.pop
+        if state == :sass_inline
+          encoder.end_group :inline
+        elsif state == :string
+          encoder.end_group :string
+        end
+      end
+      
       encoder
     end
     
-- 
cgit v1.2.1


From 4327fcbe932ac913f7eb8e92497f97913fb398c5 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:15:55 +0200
Subject: fix Sass regexp modifier

---
 lib/coderay/scanners/sass.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 0ba383f..e20bebe 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -195,7 +195,7 @@ module Scanners
         elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
           encoder.text_token match, :color
           
-        elsif match = scan(/@else if\b|#{RE::AtKeyword}/)
+        elsif match = scan(/@else if\b|#{RE::AtKeyword}/o)
           encoder.text_token match, :directive
           value_expected = true
           
-- 
cgit v1.2.1


From 755904f7046d8a8d6208fe8367aaa3b9a19ecd65 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:30:24 +0200
Subject: avoid empty tokens in Diff scanner

---
 lib/coderay/scanners/diff.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index af0f755..fd1aed6 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -69,7 +69,7 @@ module Scanners
             state = :added
           elsif match = scan(/\\ .*/)
             encoder.text_token match, :comment
-          elsif match = scan(/@@(?>[^@\n]*)@@/)
+          elsif match = scan(/@@(?>[^@\n]+)@@/)
             content_scanner.state = :initial unless match?(/\n\+/)
             content_scanner_entry_state = nil
             if check(/\n|$/)
-- 
cgit v1.2.1


From e1d5e111d968639fa03a6074cf90535ecc90d0dd Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:30:44 +0200
Subject: close correct token groups in Groovy scanner

---
 lib/coderay/scanners/groovy.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb
index d04a535..c64454f 100644
--- a/lib/coderay/scanners/groovy.rb
+++ b/lib/coderay/scanners/groovy.rb
@@ -226,7 +226,7 @@ module Scanners
             encoder.text_token match, :content  # TODO: Shouldn't this be :error?
             
           elsif match = scan(/ \\ | \n /x)
-            encoder.end_group state
+            encoder.end_group state == :regexp ? :regexp : :string
             encoder.text_token match, :error
             after_def = value_expected = false
             state = :initial
@@ -246,7 +246,7 @@ module Scanners
       end
       
       if [:multiline_string, :string, :regexp].include? state
-        encoder.end_group state
+        encoder.end_group state == :regexp ? :regexp : :string
       end
       
       if options[:keep_state]
@@ -256,7 +256,7 @@ module Scanners
       until inline_block_stack.empty?
         state, = *inline_block_stack.pop
         encoder.end_group :inline
-        encoder.end_group state
+        encoder.end_group state == :regexp ? :regexp : :string
       end
       
       encoder
-- 
cgit v1.2.1


From 603ff7d0b14521cfd0408aa68e2e1cb6ea9086bc Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:36:46 +0200
Subject: avoid empty tokens in YAML scanner

---
 lib/coderay/scanners/yaml.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb
index 358a3f1..32c8e2c 100644
--- a/lib/coderay/scanners/yaml.rb
+++ b/lib/coderay/scanners/yaml.rb
@@ -84,7 +84,7 @@ module Scanners
           when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/)
             encoder.begin_group :key
             encoder.text_token match[0,1], :delimiter
-            encoder.text_token match[1..-2], :content
+            encoder.text_token match[1..-2], :content if match.size > 2
             encoder.text_token match[-1,1], :delimiter
             encoder.end_group :key
             key_indent = column(pos - match.size) - 1
-- 
cgit v1.2.1


From 253a616924000cc5d522a2d4b8030f5693c98e43 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:39:30 +0200
Subject: close open string token groups in SQL scanner

---
 lib/coderay/scanners/sql.rb | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb
index b757278..93aeaf3 100644
--- a/lib/coderay/scanners/sql.rb
+++ b/lib/coderay/scanners/sql.rb
@@ -1,8 +1,9 @@
-module CodeRay module Scanners
+module CodeRay
+module Scanners
   
   # by Josh Goebel
   class SQL < Scanner
-
+    
     register_for :sql
     
     KEYWORDS = %w(
@@ -149,6 +150,7 @@ module CodeRay module Scanners
               string_content = ''
             end
             encoder.text_token match, :error unless match.empty?
+            encoder.end_group :string
             state = :initial
           else
             raise "else case \" reached; %p not handled." % peek(1), encoder
@@ -171,4 +173,5 @@ module CodeRay module Scanners
     
   end
   
-end end
\ No newline at end of file
+end
+end
-- 
cgit v1.2.1


From c8751fbc09d4a8f43b0b037e3a7dc7ddbfbcacb1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 13:15:10 +0200
Subject: fix unclosed token group in Ruby scanner

---
 lib/coderay/scanners/ruby.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index c282f31..80165ca 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -269,7 +269,7 @@ module Scanners
             end
             
             if last_state
-              state = last_state
+              state = last_state unless state.is_a?(StringState)  # otherwise, a simple 'def"' results in unclosed tokens
               last_state = nil
             end
             
-- 
cgit v1.2.1


From c34ad739bc7342aa40830945a5e19e90e9814e04 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 13:27:43 +0200
Subject: fix token nesting in PHP scanner

---
 lib/coderay/scanners/php.rb | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
index 40037f9..7a8d75d 100644
--- a/lib/coderay/scanners/php.rb
+++ b/lib/coderay/scanners/php.rb
@@ -265,7 +265,7 @@ module Scanners
             @html_scanner.tokenize match unless match.empty?
           end
         
-        when :php
+        when :php, :php_inline
           if match = scan(/\s+/)
             encoder.text_token match, :space
           
@@ -332,7 +332,7 @@ module Scanners
             if states.size == 1
               encoder.text_token match, :error
             else
-              states.pop
+              state = states.pop
               if states.last.is_a?(::Array)
                 delimiter = states.last[1]
                 states[-1] = states.last[0]
@@ -340,6 +340,7 @@ module Scanners
                 encoder.end_group :inline
               else
                 encoder.text_token match, :operator
+                encoder.end_group :inline if state == :php_inline
                 label_expected = true
               end
             end
@@ -350,7 +351,14 @@ module Scanners
           
           elsif match = scan(RE::PHP_END)
             encoder.text_token match, :inline_delimiter
-            states = [:initial]
+            while state = states.pop
+              encoder.end_group :string if [:sqstring, :dqstring].include? state
+              if state.is_a? Array
+                encoder.end_group :inline
+                encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+              end
+            end
+            states << :initial
           
           elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
             encoder.begin_group :string
@@ -400,6 +408,7 @@ module Scanners
           elsif match = scan(/\\/)
             encoder.text_token match, :error
           else
+            encoder.end_group :string
             states.pop
           end
         
@@ -459,7 +468,7 @@ module Scanners
               encoder.begin_group :inline
               states[-1] = [states.last, delimiter]
               delimiter = nil
-              states.push :php
+              states.push :php_inline
               encoder.text_token match, :delimiter
             else
               encoder.text_token match, :content
@@ -469,6 +478,7 @@ module Scanners
           elsif match = scan(/\$/)
             encoder.text_token match, :content
           else
+            encoder.end_group :string
             states.pop
           end
         
@@ -502,6 +512,10 @@ module Scanners
       
       while state = states.pop
         encoder.end_group :string if [:sqstring, :dqstring].include? state
+        if state.is_a? Array
+          encoder.end_group :inline
+          encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+        end
       end
       
       encoder
-- 
cgit v1.2.1


From 14f3a4f28341237503a51bfa0764c922bb02bdfa Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 14:13:15 +0200
Subject: tweak HTML CDATA token kinds

---
 lib/coderay/scanners/html.rb | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index fcf249a..530ce44 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -1,13 +1,13 @@
 module CodeRay
 module Scanners
-
+  
   # HTML Scanner
   # 
   # Alias: +xhtml+
   # 
   # See also: Scanners::XML
   class HTML < Scanner
-
+    
     register_for :html
     
     KINDS_NOT_LOC = [
@@ -100,15 +100,13 @@ module Scanners
           
           when :initial
             if match = scan(/<!\[CDATA\[/)
-              encoder.begin_group :string
-              encoder.text_token match, :delimiter
+              encoder.text_token match, :inline_delimiter
               if match = scan(/.*?\]\]>/m)
-                encoder.text_token match[0..-4], :content
-                encoder.text_token ']]>', :delimiter
-              else
-                encoder.text_token scan(/.*/m), :error
+                encoder.text_token match[0..-4], :plain
+                encoder.text_token ']]>', :inline_delimiter
+              elsif match = scan(/.+/)
+                encoder.text_token match, :error
               end
-              encoder.end_group :string
             elsif match = scan(/<!--(?:.*?-->|.*)/m)
               encoder.text_token match, :comment
             elsif match = scan(/<!(\w+)(?:.*?>|.*)|\]>/m)
-- 
cgit v1.2.1


From 56f4163e99689912c3797e5d3b2b97244ce65782 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 14:33:22 +0200
Subject: fix another CSS empty token issue

---
 lib/coderay/scanners/css.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 6c52c8a..9ed4618 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -145,10 +145,10 @@ module Scanners
           start = match[/^\w+\(/]
           encoder.text_token start, :delimiter
           if match[-1] == ?)
-            encoder.text_token match[start.size..-2], :content
+            encoder.text_token match[start.size..-2], :content if match.size > start.size + 1
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content if start.size < match.size
+            encoder.text_token match[start.size..-1], :content if match.size > start.size
           end
           encoder.end_group :function
           
-- 
cgit v1.2.1