From 501df7613c67bcd743eaa00071f0dbb179607394 Mon Sep 17 00:00:00 2001
From: Quintus <sutniuq@gmx.net>
Date: Sun, 22 Apr 2012 22:10:42 +0200
Subject: Lua scanner for CodeRay. Meta-commit. This commit is a super-commit
 containing all the subcommits for implementing the Lua scanner.

---
 lib/coderay/scanners/lua.rb | 267 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 267 insertions(+)
 create mode 100644 lib/coderay/scanners/lua.rb

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
new file mode 100644
index 0000000..e7706fc
--- /dev/null
+++ b/lib/coderay/scanners/lua.rb
@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+
+# Scanner for the Lua[http://lua.org] programming lanuage.
+#
+# The language’s complete syntax is defined in
+# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
+# which is what this scanner tries to conform to.
+class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+
+  register_for :lua
+  file_extension "lua"
+  title "Lua"
+
+  # Keywords used in Lua.
+  KEYWORDS = %w[and break do else elseif end
+  for function goto if in
+  local not or repeat return
+  then until while
+  ]
+
+  # Constants set by the Lua core.
+  PREDEFINED_CONSTANTS = %w[false true nil]
+
+  # The expressions contained in this array are parts of Lua’s `basic'
+  # library. Although it’s not entirely necessary to load that library,
+  # it is highly recommended and one would have to provide own implementations
+  # of some of these expressions if one does not do so. They however aren’t
+  # keywords, neither are they constants, but nearly predefined, so they
+  # get tagged as `predefined' rather than anything else.
+  #
+  # This list excludes values of form `_UPPERCASE' because the Lua manual
+  # requires such identifiers to be reserved by Lua anyway and they are
+  # highlighted directly accordingly, without the need for specific
+  # identifiers to be listed here.
+  PREDEFINED_EXPRESSIONS = %w[
+  assert collectgarbage dofile error getmetatable
+  ipairs load loadfile next pairs pcall print
+  rawequal rawget rawlen rawset select setmetatable
+  tonumber tostring type xpcall
+  ]
+
+  # Automatic token kind selection for normal words.
+  IDENT_KIND = CodeRay::WordList.new(:ident)
+    .add(KEYWORDS, :keyword)
+    .add(PREDEFINED_CONSTANTS, :predefined_constant)
+    .add(PREDEFINED_EXPRESSIONS, :predefined)
+
+  protected
+
+  # Scanner initialization.
+  def setup
+    @state       = :initial
+    @brace_depth = 0
+  end
+
+  # CodeRay entry hook. Starts parsing.
+  def scan_tokens(encoder, options)
+    @encoder = encoder
+    @options = options
+
+    send(:"handle_state_#@state") until eos?
+
+    @encoder
+  end
+
+  def handle_state_initial
+    if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
+      @num_equals = match.count("=") # Number must match for comment end
+      @encoder.begin_group(:comment)
+      @encoder.text_token(match, :delimiter)
+      @state = :long_comment
+
+    elsif match = scan(/--.*?$/) # --Lua comment
+      @encoder.text_token(match, :comment)
+
+    elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
+      @num_equals = match.count("=") # Number must match for comment end
+      @encoder.begin_group(:string)
+      @encoder.text_token(match, :delimiter)
+      @state = :long_string
+
+    elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label::
+      @encoder.text_token(match, :label)
+
+    elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua
+      @encoder.text_token(match, :predefined)
+
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits)
+      kind = IDENT_KIND[match]
+
+      # Extra highlighting for entities following certain keywords
+      if kind == :keyword and match == "function"
+        @state = :function_expected
+      elsif kind == :keyword and match == "goto"
+        @state = :goto_label_expected
+      elsif kind == :keyword and match == "local"
+        @state = :local_var_expected
+      end
+
+      @encoder.text_token(match, kind)
+
+    elsif match = scan(/{/) # Opening table brace {
+      @encoder.begin_group(:table)
+      @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
+      @brace_depth += 1
+      @state        = :table
+
+    elsif match = scan(/}/) # Closing table brace }
+      if @brace_depth == 1
+        @brace_depth = 0
+        @encoder.text_token(match, :delimiter)
+      elsif @brace_depth == 0 # Mismatched brace
+        @encoder.text_token(match, :error)
+      else
+        @brace_depth -= 1
+        @encoder.text_token(match, :inline_delimiter)
+        @state = :table
+      end
+      @encoder.end_group(:table)
+
+    elsif match = scan(/["']/) # String delimiters " and '
+      @encoder.begin_group(:string)
+      @encoder.text_token(match, :delimiter)
+      @start_delim = match
+      @state       = :string
+
+                      # ↓Prefix                hex number ←|→ decimal number
+    elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+      @encoder.text_token(match, :float)
+
+                      # ↓Prefix         hex number ←|→ decimal number
+    elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+      @encoder.text_token(match, :integer)
+
+    elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators
+      @encoder.text_token(match, :operator)
+
+    elsif match = scan(/\s+/) # Space
+      @encoder.text_token(match, :space)
+
+    else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors.
+      @encoder.text_token(getch, :error)
+    end
+
+    # It may be that we’re scanning a full-blown subexpression of a table
+    # (tables can contain full expressions in parts).
+    # If this is the case, return to :table scanning state.
+    @state = :table if @state == :initial && @brace_depth >= 1
+  end
+
+  def handle_state_function_expected
+    if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
+      @encoder.text_token(match, :operator)
+      @state = :initial
+    elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator
+      @encoder.text_token(match, :ident)
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo()
+      @encoder.text_token(match, :function)
+      @state = :initial
+    elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace
+      @encoder.text_token(match, :space)
+    else
+      @encoder.text_token(getch, :error)
+      @state = :initial
+    end
+  end
+
+  def handle_state_goto_label_expected
+    if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+      @encoder.text_token(match, :label)
+      @state = :initial
+    elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace
+      @encoder.text_token(match, :space)
+    else
+      @encoder.text_token(getch, :error)
+    end
+  end
+
+  def handle_state_local_var_expected
+    if match = scan(/function/) # local function ...
+      @encoder.text_token(match, :keyword)
+      @state = :function_expected
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+      @encoder.text_token(match, :local_variable)
+    elsif match = scan(/,/)
+      @encoder.text_token(match, :operator)
+    elsif match = scan(/=/)
+      @encoder.text_token(match, :operator)
+      # After encountering the equal sign, arbitrary expressions are
+      # allowed again, so just return to the main state for further
+      # parsing.
+      @state = :initial
+    elsif match = scan(/\n/)
+      @encoder.text_token(match, :space)
+      @state = :initial
+    elsif match = scan(/\s+/)
+      @encoder.text_token(match, :space)
+    else
+      @encoder.text_token(getch, :error)
+    end
+  end
+
+  def handle_state_long_comment
+    if match = scan(/.*?(?=\]={#@num_equals}\])/m)
+      @encoder.text_token(match, :content)
+
+      delim = scan(/\]={#@num_equals}\]/)
+      @encoder.text_token(delim, :delimiter)
+    else # No terminator found till EOF
+      @encoder.text_token(rest, :error)
+      terminate
+    end
+    @encoder.end_group(:comment)
+    @state = :initial
+  end
+
+  def handle_state_long_string
+    if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
+      @encoder.text_token(match, :content)
+
+      delim = scan(/\]={#@num_equals}\]/)
+      @encoder.text_token(delim, :delimiter)
+    else # No terminator found till EOF
+      @encoder.text_token(rest, :error)
+      terminate
+    end
+    @encoder.end_group(:string)
+    @state = :initial
+  end
+
+  def handle_state_string
+    if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+      @encoder.text_token(match, :content)
+    elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
+      @encoder.text_token(match, :char)
+    elsif match = scan(Regexp.compile(@start_delim))
+      @encoder.text_token(match, :delimiter)
+      @encoder.end_group(:string)
+      @state = :initial
+    elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings
+      @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings
+      @encoder.end_group(:string)
+      @state = :initial
+    else
+      @encoder.text_token(getch, :error)
+    end
+  end
+
+  def handle_state_table
+    if match = scan(/[,;]/)
+      @encoder.text_token(match, :operator)
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
+      @encoder.text_token(match, :key)
+      @encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
+      @encoder.text_token(scan(/=/), :operator)
+      @state = :initial
+    elsif match = scan(/\s+/m)
+      @encoder.text_token(match, :space)
+    else
+      # Note this clause doesn’t advance the scan pointer, it’s a kind of
+      # "retry with other options" (the :initial state then of course
+      # advances the pointer).
+      @state = :initial
+    end
+  end
+
+end
-- 
cgit v1.2.1


From 8979cc621431248fded86c341e2102a67c5344bb Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 19 Jun 2012 17:45:49 +0200
Subject: use case+when instead of send and methods

---
 lib/coderay/scanners/lua.rb | 64 ++++++++++++++++++++++-----------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index e7706fc..e712061 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
   ]
 
   # Automatic token kind selection for normal words.
-  IDENT_KIND = CodeRay::WordList.new(:ident)
-    .add(KEYWORDS, :keyword)
-    .add(PREDEFINED_CONSTANTS, :predefined_constant)
-    .add(PREDEFINED_EXPRESSIONS, :predefined)
+  IDENT_KIND = CodeRay::WordList.new(:ident).
+    add(KEYWORDS, :keyword).
+    add(PREDEFINED_CONSTANTS, :predefined_constant).
+    add(PREDEFINED_EXPRESSIONS, :predefined)
 
   protected
 
@@ -57,13 +57,11 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
   def scan_tokens(encoder, options)
     @encoder = encoder
     @options = options
-
-    send(:"handle_state_#@state") until eos?
-
-    @encoder
-  end
-
-  def handle_state_initial
+    
+    until eos?
+  case state
+        
+  when :initial
     if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
       @num_equals = match.count("=") # Number must match for comment end
       @encoder.begin_group(:comment)
@@ -99,13 +97,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
 
       @encoder.text_token(match, kind)
 
-    elsif match = scan(/{/) # Opening table brace {
+    elsif match = scan(/\{/) # Opening table brace {
       @encoder.begin_group(:table)
       @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
       @brace_depth += 1
       @state        = :table
 
-    elsif match = scan(/}/) # Closing table brace }
+    elsif match = scan(/\}/) # Closing table brace }
       if @brace_depth == 1
         @brace_depth = 0
         @encoder.text_token(match, :delimiter)
@@ -146,9 +144,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     # (tables can contain full expressions in parts).
     # If this is the case, return to :table scanning state.
     @state = :table if @state == :initial && @brace_depth >= 1
-  end
-
-  def handle_state_function_expected
+    
+  when :function_expected
     if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
       @encoder.text_token(match, :operator)
       @state = :initial
@@ -163,9 +160,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(getch, :error)
       @state = :initial
     end
-  end
 
-  def handle_state_goto_label_expected
+  when :goto_label_expected
     if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
       @encoder.text_token(match, :label)
       @state = :initial
@@ -174,9 +170,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  end
-
-  def handle_state_local_var_expected
+  
+  when :local_var_expected
     if match = scan(/function/) # local function ...
       @encoder.text_token(match, :keyword)
       @state = :function_expected
@@ -198,9 +193,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  end
-
-  def handle_state_long_comment
+    
+  when :long_comment
     if match = scan(/.*?(?=\]={#@num_equals}\])/m)
       @encoder.text_token(match, :content)
 
@@ -212,9 +206,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:comment)
     @state = :initial
-  end
-
-  def handle_state_long_string
+    
+  when :long_string
     if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
       @encoder.text_token(match, :content)
 
@@ -226,9 +219,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:string)
     @state = :initial
-  end
-
-  def handle_state_string
+    
+  when :string
     if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
       @encoder.text_token(match, :content)
     elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
@@ -244,9 +236,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  end
-
-  def handle_state_table
+  
+  when :table
     if match = scan(/[,;]/)
       @encoder.text_token(match, :operator)
     elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
@@ -262,6 +253,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       # advances the pointer).
       @state = :initial
     end
+  else
+    raise
+  end
+
+    end
+
+    @encoder
   end
 
 end
-- 
cgit v1.2.1


From e796a247e5bf2395ec72b0ae98d68bcb648f5368 Mon Sep 17 00:00:00 2001
From: Joel Holdbrooks <cjholdbrooks@gmail.com>
Date: Tue, 3 Jul 2012 18:19:05 -0700
Subject: use "id" instead of "constant" for css ids

---
 lib/coderay/scanners/css.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 7b731ef..c473574 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -8,7 +8,7 @@ module Scanners
     KINDS_NOT_LOC = [
       :comment,
       :class, :pseudo_class, :type,
-      :constant, :directive,
+      :id, :directive,
       :key, :value, :operator, :color, :float, :string,
       :error, :important,
     ]  # :nodoc:
@@ -73,7 +73,7 @@ module Scanners
               encoder.text_token match, :class
               next
             elsif match = scan(RE::Id)
-              encoder.text_token match, :constant
+              encoder.text_token match, :id
               next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
-- 
cgit v1.2.1


From abb92f30b12e11781afa76f43a344627520b5b34 Mon Sep 17 00:00:00 2001
From: Eric Guo <eric.guo@sandisk.com>
Date: Sun, 8 Jul 2012 14:32:28 +0800
Subject: New: *Go Encoder*

Draft version, copy from c
---
 lib/coderay/scanners/go.rb | 195 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 195 insertions(+)
 create mode 100644 lib/coderay/scanners/go.rb

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
new file mode 100644
index 0000000..4431ef2
--- /dev/null
+++ b/lib/coderay/scanners/go.rb
@@ -0,0 +1,195 @@
+module CodeRay
+module Scanners
+
+  # Scanner for Go, copy from c
+  class Go < Scanner
+
+    register_for :go
+    file_extension 'go'
+
+    # http://golang.org/ref/spec#Keywords
+    KEYWORDS = [
+      'break', 'default', 'func', 'interface', 'select',
+      'case', 'defer', 'go', 'map', 'struct',
+      'chan', 'else', 'goto', 'package', 'switch',
+      'const', 'fallthrough', 'if', 'range', 'type',
+      'continue', 'for', 'import', 'return', 'var',
+    ]  # :nodoc:
+
+    # http://golang.org/ref/spec#Types
+    PREDEFINED_TYPES = [
+      'bool',
+      'uint8', 'uint16', 'uint32', 'uint64',
+      'int8', 'int16', 'int32', 'int64',
+      'float32', 'float64',
+      'complex64', 'complex128',
+      'byte', 'rune',
+      'uint', 'int', 'uintptr',
+    ]  # :nodoc:
+
+    PREDEFINED_CONSTANTS = [
+      'nil', 'iota',
+      'true', 'false',
+    ]  # :nodoc:
+
+    DIRECTIVES = [
+      'go_no_directive',  # Seems no directive concept in Go?
+    ]  # :nodoc:
+
+    IDENT_KIND = WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_TYPES, :predefined_type).
+      add(DIRECTIVES, :directive).
+      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
+
+    ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+
+  protected
+
+    def scan_tokens encoder, options
+
+      state = :initial
+      label_expected = true
+      case_expected = false
+      label_expected_before_preproc_line = nil
+      in_preproc_line = false
+
+      until eos?
+
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            if in_preproc_line && match != "\\\n" && match.index(?\n)
+              in_preproc_line = false
+              label_expected = label_expected_before_preproc_line
+            end
+            encoder.text_token match, :space
+
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            encoder.text_token match, :comment
+
+          elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+            label_expected = match =~ /[;\{\}]/
+            if case_expected
+              label_expected = true if match == ':'
+              case_expected = false
+            end
+            encoder.text_token match, :operator
+
+          elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = IDENT_KIND[match]
+            if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
+              kind = :label
+              match << matched
+            else
+              label_expected = false
+              if kind == :keyword
+                case match
+                when 'case', 'default'
+                  case_expected = true
+                end
+              end
+            end
+            encoder.text_token match, kind
+
+          elsif match = scan(/L?"/)
+            encoder.begin_group :string
+            if match[0] == ?L
+              encoder.text_token 'L', :modifier
+              match = '"'
+            end
+            encoder.text_token match, :delimiter
+            state = :string
+
+          elsif match = scan(/ \# \s* if \s* 0 /x)
+            match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+            encoder.text_token match, :comment
+
+          elsif match = scan(/#[ \t]*(\w*)/)
+            encoder.text_token match, :preprocessor
+            in_preproc_line = true
+            label_expected_before_preproc_line = label_expected
+            state = :include_expected if self[1] == 'include'
+
+          elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+            label_expected = false
+            encoder.text_token match, :char
+
+          elsif match = scan(/\$/)
+            encoder.text_token match, :ident
+
+          elsif match = scan(/0[xX][0-9A-Fa-f]+/)
+            label_expected = false
+            encoder.text_token match, :hex
+
+          elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/)
+            label_expected = false
+            encoder.text_token match, :octal
+
+          elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
+            label_expected = false
+            encoder.text_token match, :integer
+
+          elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+            label_expected = false
+            encoder.text_token match, :float
+
+          else
+            encoder.text_token getch, :error
+
+          end
+
+        when :string
+          if match = scan(/[^\\\n"]+/)
+            encoder.text_token match, :content
+          elsif match = scan(/"/)
+            encoder.text_token match, :delimiter
+            encoder.end_group :string
+            state = :initial
+            label_expected = false
+          elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            encoder.text_token match, :char
+          elsif match = scan(/ \\ | $ /x)
+            encoder.end_group :string
+            encoder.text_token match, :error
+            state = :initial
+            label_expected = false
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
+          end
+
+        when :include_expected
+          if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+            encoder.text_token match, :include
+            state = :initial
+
+          elsif match = scan(/\s+/)
+            encoder.text_token match, :space
+            state = :initial if match.index ?\n
+
+          else
+            state = :initial
+
+          end
+
+        else
+          raise_inspect 'Unknown state', encoder
+
+        end
+
+      end
+
+      if state == :string
+        encoder.end_group :string
+      end
+
+      encoder
+    end
+
+  end
+
+end
+end
-- 
cgit v1.2.1


From 4eb4324c6fb20b9b5e7f73f17bc21e4a17fd3073 Mon Sep 17 00:00:00 2001
From: Joel Holdbrooks <cjholdbrooks@gmail.com>
Date: Tue, 24 Jul 2012 10:59:50 -0700
Subject: use :tag instead of :type for identifiers

---
 lib/coderay/scanners/css.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index c473574..503e14e 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -7,7 +7,7 @@ module Scanners
     
     KINDS_NOT_LOC = [
       :comment,
-      :class, :pseudo_class, :type,
+      :class, :pseudo_class, :tag,
       :id, :directive,
       :key, :value, :operator, :color, :float, :string,
       :error, :important,
@@ -67,7 +67,7 @@ module Scanners
         elsif case states.last
           when :initial, :media
             if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
@@ -102,7 +102,7 @@ module Scanners
             
           when :media_before_name
             if match = scan(RE::Ident)
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               states[-1] = :media_after_name
               next
             end
-- 
cgit v1.2.1


From b081c1f3ab670301199005c04ecbc11614593285 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Tue, 11 Sep 2012 15:04:45 -0600
Subject: escape = in regex (my editor gets confused:-)

---
 lib/coderay/scanners/lua.rb | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index e712061..e640397 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -57,10 +57,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
   def scan_tokens(encoder, options)
     @encoder = encoder
     @options = options
-    
+
     until eos?
   case state
-        
+
   when :initial
     if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
       @num_equals = match.count("=") # Number must match for comment end
@@ -144,7 +144,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     # (tables can contain full expressions in parts).
     # If this is the case, return to :table scanning state.
     @state = :table if @state == :initial && @brace_depth >= 1
-    
+
   when :function_expected
     if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
       @encoder.text_token(match, :operator)
@@ -170,7 +170,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  
+
   when :local_var_expected
     if match = scan(/function/) # local function ...
       @encoder.text_token(match, :keyword)
@@ -179,7 +179,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(match, :local_variable)
     elsif match = scan(/,/)
       @encoder.text_token(match, :operator)
-    elsif match = scan(/=/)
+    elsif match = scan(/\=/)
       @encoder.text_token(match, :operator)
       # After encountering the equal sign, arbitrary expressions are
       # allowed again, so just return to the main state for further
@@ -193,7 +193,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-    
+
   when :long_comment
     if match = scan(/.*?(?=\]={#@num_equals}\])/m)
       @encoder.text_token(match, :content)
@@ -206,7 +206,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:comment)
     @state = :initial
-    
+
   when :long_string
     if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
       @encoder.text_token(match, :content)
@@ -219,7 +219,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:string)
     @state = :initial
-    
+
   when :string
     if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
       @encoder.text_token(match, :content)
@@ -236,14 +236,14 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  
+
   when :table
     if match = scan(/[,;]/)
       @encoder.text_token(match, :operator)
     elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
       @encoder.text_token(match, :key)
       @encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
-      @encoder.text_token(scan(/=/), :operator)
+      @encoder.text_token(scan(/\=/), :operator)
       @state = :initial
     elsif match = scan(/\s+/m)
       @encoder.text_token(match, :space)
-- 
cgit v1.2.1


From aeac6672fbe70702288b259605cb863493ad8ac2 Mon Sep 17 00:00:00 2001
From: SHIMOMURA Sho <graueule@gmail.com>
Date: Sun, 21 Oct 2012 01:23:56 +0900
Subject: TaskPaper scanner for CodeRay

---
 lib/coderay/scanners/taskpaper.rb | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 lib/coderay/scanners/taskpaper.rb

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/taskpaper.rb b/lib/coderay/scanners/taskpaper.rb
new file mode 100644
index 0000000..c6eb3b3
--- /dev/null
+++ b/lib/coderay/scanners/taskpaper.rb
@@ -0,0 +1,34 @@
+module CodeRay
+module Scanners
+  
+  class Taskpaper < Scanner
+    
+    register_for :taskpaper
+    file_extension 'taskpaper'
+    
+  protected
+    
+    def scan_tokens encoder, options
+      until eos?
+        if match = scan(/^\S.*:.*$/)                  # project
+          encoder.text_token(match, :project)
+        elsif match = scan(/^-.+@done(?:\(.*)?.*$/)   # completed task
+          encoder.text_token(match, :complete)
+        elsif match = scan(/^-.+$/)                   # task
+          encoder.text_token(match, :plain)
+        elsif match = scan(/^.+$/)                    # comment
+          encoder.text_token(match, :comment)
+        elsif match = scan(/\s+/)                     # space
+          encoder.text_token(match, :space)
+        else                                          # other
+          encoder.text_token getch, :error
+        end
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end
-- 
cgit v1.2.1


From 279348d3c2159df4ce6ac5949ada0177aa7c7159 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Sat, 27 Oct 2012 23:41:52 -0600
Subject: :map token kind

Use :map instead of :table. It's more generic, and won't be confused with the :table rendering style.
---
 lib/coderay/scanners/lua.rb | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index e640397..64763dc 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -98,10 +98,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(match, kind)
 
     elsif match = scan(/\{/) # Opening table brace {
-      @encoder.begin_group(:table)
+      @encoder.begin_group(:map)
       @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
       @brace_depth += 1
-      @state        = :table
+      @state        = :map
 
     elsif match = scan(/\}/) # Closing table brace }
       if @brace_depth == 1
@@ -112,9 +112,9 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       else
         @brace_depth -= 1
         @encoder.text_token(match, :inline_delimiter)
-        @state = :table
+        @state = :map
       end
-      @encoder.end_group(:table)
+      @encoder.end_group(:map)
 
     elsif match = scan(/["']/) # String delimiters " and '
       @encoder.begin_group(:string)
@@ -142,8 +142,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
 
     # It may be that we’re scanning a full-blown subexpression of a table
     # (tables can contain full expressions in parts).
-    # If this is the case, return to :table scanning state.
-    @state = :table if @state == :initial && @brace_depth >= 1
+    # If this is the case, return to :map scanning state.
+    @state = :map if @state == :initial && @brace_depth >= 1
 
   when :function_expected
     if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
@@ -237,7 +237,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(getch, :error)
     end
 
-  when :table
+  when :map
     if match = scan(/[,;]/)
       @encoder.text_token(match, :operator)
     elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
-- 
cgit v1.2.1


From 51526173dd7ee640171ee597d3c380c5dda42762 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 10 Mar 2013 22:40:10 +0100
Subject: support multi-line comments in JavaScript diff

---
 lib/coderay/scanners/java_script.rb | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb
index 43ecb18..75881a0 100644
--- a/lib/coderay/scanners/java_script.rb
+++ b/lib/coderay/scanners/java_script.rb
@@ -54,10 +54,17 @@ module Scanners
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     def scan_tokens encoder, options
       
-      state = :initial
-      string_delimiter = nil
+      state, string_delimiter = options[:state] || @state
+      if string_delimiter
+        encoder.begin_group state
+      end
+      
       value_expected = true
       key_expected = false
       function_expected = false
@@ -72,9 +79,10 @@ module Scanners
             value_expected = true if !value_expected && match.index(?\n)
             encoder.text_token match, :space
             
-          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .*() ) !mx)
             value_expected = true
             encoder.text_token match, :comment
+            state = :open_multi_line_comment if self[1]
             
           elsif check(/\.?\d/)
             key_expected = value_expected = false
@@ -182,13 +190,28 @@ module Scanners
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
           
+        when :open_multi_line_comment
+          if match = scan(%r! .*? \*/ !mx)
+            state = :initial
+          else
+            match = scan(%r! .+ !mx)
+          end
+          value_expected = true
+          encoder.text_token match, :comment if match
+          
         else
-          raise_inspect 'Unknown state', encoder
+          #:nocov:
+          raise_inspect 'Unknown state: %p' % [state], encoder
+          #:nocov:
           
         end
         
       end
       
+      if options[:keep_state]
+        @state = state, string_delimiter
+      end
+      
       if [:string, :regexp].include? state
         encoder.end_group state
       end
-- 
cgit v1.2.1


From d9d6dd5f4a73363ea5d353ecda142f77ed4eba5a Mon Sep 17 00:00:00 2001
From: Gavin Kistner <gkistner@nvidia.com>
Date: Thu, 6 Jun 2013 10:43:19 -0600
Subject: Support CDATA blocks in HTML/XML

---
 lib/coderay/scanners/html.rb | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 3ba3b79..fcf249a 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -99,7 +99,17 @@ module Scanners
           case state
           
           when :initial
-            if match = scan(/<!--(?:.*?-->|.*)/m)
+            if match = scan(/<!\[CDATA\[/)
+              encoder.begin_group :string
+              encoder.text_token match, :delimiter
+              if match = scan(/.*?\]\]>/m)
+                encoder.text_token match[0..-4], :content
+                encoder.text_token ']]>', :delimiter
+              else
+                encoder.text_token scan(/.*/m), :error
+              end
+              encoder.end_group :string
+            elsif match = scan(/<!--(?:.*?-->|.*)/m)
               encoder.text_token match, :comment
             elsif match = scan(/<!(\w+)(?:.*?>|.*)|\]>/m)
               encoder.text_token match, :doctype
-- 
cgit v1.2.1


From a69d5d43b79a4f2cf5aaccc50b2275bf0dab4aba Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 23:29:14 +0200
Subject: accept all idents as Ruby 1.9 hash keys (#126)

---
 lib/coderay/scanners/ruby.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index c5cf1e2..c282f31 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -96,7 +96,7 @@ module Scanners
                                       /#{patterns::METHOD_NAME}/o)
               
               kind = patterns::IDENT_KIND[match]
-              if kind == :ident && value_expected != :colon_expected && scan(/:(?!:)/)
+              if value_expected != :colon_expected && scan(/:(?!:)/)
                 value_expected = true
                 encoder.text_token match, :key
                 encoder.text_token ':',   :operator
-- 
cgit v1.2.1


From 33f1a738ea85ea40a5fe760ac2591065f3e29033 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 23:59:47 +0200
Subject: reset string_delimiter when detecting open string

---
 lib/coderay/scanners/java_script.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb
index 8587b06..9eb0a0a 100644
--- a/lib/coderay/scanners/java_script.rb
+++ b/lib/coderay/scanners/java_script.rb
@@ -184,6 +184,7 @@ module Scanners
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group state
             encoder.text_token match, :error unless match.empty?
+            string_delimiter = nil
             key_expected = value_expected = false
             state = :initial
           else
-- 
cgit v1.2.1


From b159057934d2c84c0a25c83f6cbe59010345d0a7 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 11 Jun 2013 03:29:54 +0200
Subject: tweak CSS tag/id token kind patch from noprompt

---
 lib/coderay/scanners/css.rb  | 4 ++--
 lib/coderay/scanners/sass.rb | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 22c6bfc..732f9c5 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -10,7 +10,7 @@ module Scanners
       :class, :pseudo_class, :tag,
       :id, :directive,
       :key, :value, :operator, :color, :float, :string,
-      :error, :important,
+      :error, :important, :type,
     ]  # :nodoc:
     
     module RE  # :nodoc:
@@ -99,7 +99,7 @@ module Scanners
             
           when :media_before_name
             if match = scan(RE::Ident)
-              encoder.text_token match, :tag
+              encoder.text_token match, :type
               states[-1] = :media_after_name
               next
             end
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 0eb2caa..218ee57 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -38,16 +38,16 @@ module Scanners
         elsif case states.last
           when :initial, :media, :sass_inline
             if match = scan(/(?>#{RE::Ident})(?!\()/ox)
-              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :type)
+              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag)
               next
             elsif !value_expected && (match = scan(/\*/))
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
               next
             elsif match = scan(RE::Id)
-              encoder.text_token match, :constant
+              encoder.text_token match, :id
               next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
-- 
cgit v1.2.1


From 2e4e83bf84282d6ce7fbc9eeff2cbe1c79788a9b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 11 Jun 2013 16:30:26 +0200
Subject: tweak Sass scanner: comments, includes, functions

---
 lib/coderay/scanners/sass.rb | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 218ee57..167051d 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -7,8 +7,6 @@ module Scanners
     register_for :sass
     file_extension 'sass'
     
-    SASS_FUNCTION = /(?:inline-image|linear-gradient|color-stops|mix|lighten|darken|rotate|image-url|image-width|image-height|sprite-url|sprite-path|sprite-file|sprite-map|sprite-position|sprite|unquote|join|round|ceil|floor|nth)/
-    
     STRING_CONTENT_PATTERN = {
       "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
       '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
@@ -26,10 +24,22 @@ module Scanners
       
       until eos?
         
-        if match = scan(/\s+/)
+        if bol? && (match = scan(/(?>( +)?(\/[\*\/])(.+)?)(?=\n)/))
+          encoder.text_token self[1], :space if self[1]
+          encoder.begin_group :comment
+          encoder.text_token self[2], :delimiter
+          encoder.text_token self[3], :content if self[3]
+          if match = scan(/(?:\n+#{self[1]} .*)+/)
+            encoder.text_token match, :content
+          end
+          encoder.end_group :comment
+        elsif match = scan(/\n|[^\n\S]+\n?/)
           encoder.text_token match, :space
-          value_expected = false if match.index(/\n/)
-          
+          if match.index(/\n/)
+            value_expected = false
+            states.pop if states.last == :include
+          end
+        
         elsif states.last == :sass_inline && (match = scan(/\}/))
           encoder.text_token match, :inline_delimiter
           encoder.end_group :inline
@@ -61,7 +71,11 @@ module Scanners
             elsif match = scan(/(\=|@mixin +)#{RE::Ident}/o)
               encoder.text_token match, :function
               next
-            elsif match = scan(/@media/)
+            elsif match = scan(/@import\b/)
+              encoder.text_token match, :directive
+              states << :include
+              next
+            elsif match = scan(/@media\b/)
               encoder.text_token match, :directive
               # states.push :media_before_name
               next
@@ -90,13 +104,19 @@ module Scanners
               encoder.text_token match, :inline_delimiter
               states.push :sass_inline
             elsif match = scan(/ \\ | $ /x)
-              encoder.end_group state
+              encoder.end_group :string
               encoder.text_token match, :error unless match.empty?
               states.pop
             else
               raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
             end
           
+          when :include
+            if match = scan(/[^\s'",]+/)
+              encoder.text_token match, :include
+              next
+            end
+          
           else
             #:nocov:
             raise_inspect 'Unknown state', encoder
@@ -148,9 +168,6 @@ module Scanners
             states.push :string
           end
           
-        elsif match = scan(/#{SASS_FUNCTION}/o)
-          encoder.text_token match, :predefined
-          
         elsif match = scan(/#{RE::Function}/o)
           encoder.begin_group :function
           start = match[/^[-\w]+\(/]
@@ -163,6 +180,9 @@ module Scanners
           end
           encoder.end_group :function
           
+        elsif match = scan(/[a-z][-a-z_]*(?=\()/o)
+          encoder.text_token match, :predefined
+          
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
           encoder.text_token match, :float
           
-- 
cgit v1.2.1


From abdc3e4a18b664e9d6e20e06913a0aa342f8ca73 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Wed, 12 Jun 2013 15:49:44 +0200
Subject: tweak Taskpaper scanner, cleanup doc_string kind

---
 lib/coderay/scanners/taskpaper.rb | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/taskpaper.rb b/lib/coderay/scanners/taskpaper.rb
index c6eb3b3..42670bc 100644
--- a/lib/coderay/scanners/taskpaper.rb
+++ b/lib/coderay/scanners/taskpaper.rb
@@ -10,17 +10,19 @@ module Scanners
     
     def scan_tokens encoder, options
       until eos?
-        if match = scan(/^\S.*:.*$/)                  # project
-          encoder.text_token(match, :project)
-        elsif match = scan(/^-.+@done(?:\(.*)?.*$/)   # completed task
-          encoder.text_token(match, :complete)
-        elsif match = scan(/^-.+$/)                   # task
+        if match = scan(/\S.*:.*$/)                  # project
+          encoder.text_token(match, :namespace)
+        elsif match = scan(/-.+@done.*/)             # completed task
+          encoder.text_token(match, :done)
+        elsif match = scan(/-(?:[^@\n]+|@(?!due))*/) # task
           encoder.text_token(match, :plain)
-        elsif match = scan(/^.+$/)                    # comment
+        elsif match = scan(/@due.*/)                 # comment
+          encoder.text_token(match, :important)
+        elsif match = scan(/.+/)                     # comment
           encoder.text_token(match, :comment)
-        elsif match = scan(/\s+/)                     # space
+        elsif match = scan(/\s+/)                    # space
           encoder.text_token(match, :space)
-        else                                          # other
+        else                                         # other
           encoder.text_token getch, :error
         end
       end
-- 
cgit v1.2.1


From 65eeaca6c18bea379b966559084e47d6fb45c75c Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Wed, 12 Jun 2013 16:19:41 +0200
Subject: add python binary string

---
 lib/coderay/scanners/python.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index a9492ab..09c8b6e 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -157,12 +157,12 @@ module Scanners
             encoder.text_token match, :operator
           
           elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
+            modifiers = self[1]
             string_delimiter = self[2]
-            string_type = docstring_coming ? :docstring : :string
+            string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string)
             docstring_coming = false if docstring_coming
             encoder.begin_group string_type
             string_raw = false
-            modifiers = self[1]
             unless modifiers.empty?
               string_raw = !!modifiers.index(?r)
               encoder.text_token modifiers, :modifier
-- 
cgit v1.2.1


From fc21c91d7a89d98785e494ffd41e9f3adbdfed81 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Wed, 19 Jun 2013 11:50:04 +0200
Subject: use different key/value heuristic in JSON scanner

---
 lib/coderay/scanners/json.rb | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 4e0f462..3754a9b 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -14,15 +14,17 @@ module Scanners
     
     ESCAPE = / [bfnrt\\"\/] /x  # :nodoc:
     UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x  # :nodoc:
+    KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /x
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     # See http://json.org/ for a definition of the JSON lexic/grammar.
     def scan_tokens encoder, options
-      
-      state = :initial
-      stack = []
-      key_expected = false
+      state = options[:state] || @state
       
       until eos?
         
@@ -32,18 +34,11 @@ module Scanners
           if match = scan(/ \s+ /x)
             encoder.text_token match, :space
           elsif match = scan(/"/)
-            state = key_expected ? :key : :string
+            state = check(/#{KEY}/o) ? :key : :string
             encoder.begin_group state
             encoder.text_token match, :delimiter
           elsif match = scan(/ [:,\[{\]}] /x)
             encoder.text_token match, :operator
-            case match
-            when ':' then key_expected = false
-            when ',' then key_expected = true if stack.last == :object
-            when '{' then stack << :object; key_expected = true
-            when '[' then stack << :array
-            when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
-            end
           elsif match = scan(/ true | false | null /x)
             encoder.text_token match, :value
           elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x)
@@ -82,6 +77,10 @@ module Scanners
         end
       end
       
+      if options[:keep_state]
+        @state = state
+      end
+      
       if [:string, :key].include? state
         encoder.end_group state
       end
-- 
cgit v1.2.1


From da1425058e9b9e04bd988ddf606331d9cd0c827b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:06:33 +0200
Subject: cleanup Lua scanner, fix end_group(:map) issue - use local instead of
 instance variables

---
 lib/coderay/scanners/lua.rb | 524 ++++++++++++++++++++++----------------------
 1 file changed, 267 insertions(+), 257 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 64763dc..3bee275 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -1,265 +1,275 @@
-# -*- coding: utf-8 -*-
+# encoding: utf-8
 
-# Scanner for the Lua[http://lua.org] programming lanuage.
-#
-# The language’s complete syntax is defined in
-# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
-# which is what this scanner tries to conform to.
-class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+module CodeRay
+module Scanners
 
-  register_for :lua
-  file_extension "lua"
-  title "Lua"
-
-  # Keywords used in Lua.
-  KEYWORDS = %w[and break do else elseif end
-  for function goto if in
-  local not or repeat return
-  then until while
-  ]
-
-  # Constants set by the Lua core.
-  PREDEFINED_CONSTANTS = %w[false true nil]
-
-  # The expressions contained in this array are parts of Lua’s `basic'
-  # library. Although it’s not entirely necessary to load that library,
-  # it is highly recommended and one would have to provide own implementations
-  # of some of these expressions if one does not do so. They however aren’t
-  # keywords, neither are they constants, but nearly predefined, so they
-  # get tagged as `predefined' rather than anything else.
+  # Scanner for the Lua[http://lua.org] programming lanuage.
   #
-  # This list excludes values of form `_UPPERCASE' because the Lua manual
-  # requires such identifiers to be reserved by Lua anyway and they are
-  # highlighted directly accordingly, without the need for specific
-  # identifiers to be listed here.
-  PREDEFINED_EXPRESSIONS = %w[
-  assert collectgarbage dofile error getmetatable
-  ipairs load loadfile next pairs pcall print
-  rawequal rawget rawlen rawset select setmetatable
-  tonumber tostring type xpcall
-  ]
-
-  # Automatic token kind selection for normal words.
-  IDENT_KIND = CodeRay::WordList.new(:ident).
-    add(KEYWORDS, :keyword).
-    add(PREDEFINED_CONSTANTS, :predefined_constant).
-    add(PREDEFINED_EXPRESSIONS, :predefined)
-
-  protected
-
-  # Scanner initialization.
-  def setup
-    @state       = :initial
-    @brace_depth = 0
-  end
-
-  # CodeRay entry hook. Starts parsing.
-  def scan_tokens(encoder, options)
-    @encoder = encoder
-    @options = options
-
-    until eos?
-  case state
-
-  when :initial
-    if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
-      @num_equals = match.count("=") # Number must match for comment end
-      @encoder.begin_group(:comment)
-      @encoder.text_token(match, :delimiter)
-      @state = :long_comment
-
-    elsif match = scan(/--.*?$/) # --Lua comment
-      @encoder.text_token(match, :comment)
-
-    elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
-      @num_equals = match.count("=") # Number must match for comment end
-      @encoder.begin_group(:string)
-      @encoder.text_token(match, :delimiter)
-      @state = :long_string
-
-    elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label::
-      @encoder.text_token(match, :label)
-
-    elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua
-      @encoder.text_token(match, :predefined)
-
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits)
-      kind = IDENT_KIND[match]
-
-      # Extra highlighting for entities following certain keywords
-      if kind == :keyword and match == "function"
-        @state = :function_expected
-      elsif kind == :keyword and match == "goto"
-        @state = :goto_label_expected
-      elsif kind == :keyword and match == "local"
-        @state = :local_var_expected
+  # The language’s complete syntax is defined in
+  # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
+  # which is what this scanner tries to conform to.
+  class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+    
+    register_for :lua
+    file_extension "lua"
+    title "Lua"
+    
+    # Keywords used in Lua.
+    KEYWORDS = %w[and break do else elseif end
+    for function goto if in
+    local not or repeat return
+    then until while
+    ]
+    
+    # Constants set by the Lua core.
+    PREDEFINED_CONSTANTS = %w[false true nil]
+    
+    # The expressions contained in this array are parts of Lua’s `basic'
+    # library. Although it’s not entirely necessary to load that library,
+    # it is highly recommended and one would have to provide own implementations
+    # of some of these expressions if one does not do so. They however aren’t
+    # keywords, neither are they constants, but nearly predefined, so they
+    # get tagged as `predefined' rather than anything else.
+    #
+    # This list excludes values of form `_UPPERCASE' because the Lua manual
+    # requires such identifiers to be reserved by Lua anyway and they are
+    # highlighted directly accordingly, without the need for specific
+    # identifiers to be listed here.
+    PREDEFINED_EXPRESSIONS = %w[
+    assert collectgarbage dofile error getmetatable
+    ipairs load loadfile next pairs pcall print
+    rawequal rawget rawlen rawset select setmetatable
+    tonumber tostring type xpcall
+    ]
+    
+    # Automatic token kind selection for normal words.
+    IDENT_KIND = CodeRay::WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
+      add(PREDEFINED_EXPRESSIONS, :predefined)
+    
+    protected
+    
+    # Scanner initialization.
+    def setup
+      @state       = :initial
+      @brace_depth = 0
+    end
+    
+    # CodeRay entry hook. Starts parsing.
+    def scan_tokens(encoder, options)
+      state = options[:state] || @state
+      
+      until eos?
+        case state
+        
+        when :initial
+          if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
+            @num_equals = match.count("=") # Number must match for comment end
+            encoder.begin_group(:comment)
+            encoder.text_token(match, :delimiter)
+            state = :long_comment
+          
+          elsif match = scan(/--.*$/) # --Lua comment
+            encoder.text_token(match, :comment)
+          
+          elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
+            @num_equals = match.count("=") # Number must match for comment end
+            encoder.begin_group(:string)
+            encoder.text_token(match, :delimiter)
+            state = :long_string
+          
+          elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label::
+            encoder.text_token(match, :label)
+          
+          elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua
+            encoder.text_token(match, :predefined)
+          
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits)
+            kind = IDENT_KIND[match]
+            
+            # Extra highlighting for entities following certain keywords
+            if kind == :keyword and match == "function"
+              state = :function_expected
+            elsif kind == :keyword and match == "goto"
+              state = :goto_label_expected
+            elsif kind == :keyword and match == "local"
+              state = :local_var_expected
+            end
+            
+            encoder.text_token(match, kind)
+          
+          elsif match = scan(/\{/) # Opening table brace {
+            encoder.begin_group(:map)
+            encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
+            @brace_depth += 1
+            state        = :map
+          
+          elsif match = scan(/\}/) # Closing table brace }
+            if @brace_depth == 1
+              @brace_depth = 0
+              encoder.text_token(match, :delimiter)
+              encoder.end_group(:map)
+            elsif @brace_depth == 0 # Mismatched brace
+              encoder.text_token(match, :error)
+            else
+              @brace_depth -= 1
+              encoder.text_token(match, :inline_delimiter)
+              encoder.end_group(:map)
+              state = :map
+            end
+          
+          elsif match = scan(/["']/) # String delimiters " and '
+            encoder.begin_group(:string)
+            encoder.text_token(match, :delimiter)
+            @start_delim = match
+            state       = :string
+          
+                            # ↓Prefix                hex number ←|→ decimal number
+          elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+            encoder.text_token(match, :float)
+          
+                            # ↓Prefix         hex number ←|→ decimal number
+          elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+            encoder.text_token(match, :integer)
+          
+          elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators
+            encoder.text_token(match, :operator)
+          
+          elsif match = scan(/\s+/) # Space
+            encoder.text_token(match, :space)
+          
+          else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors.
+            encoder.text_token(getch, :error)
+          end
+          
+          # It may be that we’re scanning a full-blown subexpression of a table
+          # (tables can contain full expressions in parts).
+          # If this is the case, return to :map scanning state.
+          state = :map if state == :initial && @brace_depth >= 1
+        
+        when :function_expected
+          if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
+            encoder.text_token(match, :operator)
+            state = :initial
+          elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator
+            encoder.text_token(match, :ident)
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo()
+            encoder.text_token(match, :function)
+            state = :initial
+          elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+            state = :initial
+          end
+        
+        when :goto_label_expected
+          if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+            encoder.text_token(match, :label)
+            state = :initial
+          elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :local_var_expected
+          if match = scan(/function/) # local function ...
+            encoder.text_token(match, :keyword)
+            state = :function_expected
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+            encoder.text_token(match, :local_variable)
+          elsif match = scan(/,/)
+            encoder.text_token(match, :operator)
+          elsif match = scan(/\=/)
+            encoder.text_token(match, :operator)
+            # After encountering the equal sign, arbitrary expressions are
+            # allowed again, so just return to the main state for further
+            # parsing.
+            state = :initial
+          elsif match = scan(/\n/)
+            encoder.text_token(match, :space)
+            state = :initial
+          elsif match = scan(/\s+/)
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :long_comment
+          if match = scan(/.*?(?=\]={#@num_equals}\])/m)
+            encoder.text_token(match, :content)
+            
+            delim = scan(/\]={#@num_equals}\]/)
+            encoder.text_token(delim, :delimiter)
+          else # No terminator found till EOF
+            encoder.text_token(rest, :error)
+            terminate
+          end
+          encoder.end_group(:comment)
+          state = :initial
+        
+        when :long_string
+          if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
+            encoder.text_token(match, :content)
+            
+            delim = scan(/\]={#@num_equals}\]/)
+            encoder.text_token(delim, :delimiter)
+          else # No terminator found till EOF
+            encoder.text_token(rest, :error)
+            terminate
+          end
+          encoder.end_group(:string)
+          state = :initial
+        
+        when :string
+          if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+            encoder.text_token(match, :content)
+          elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
+            encoder.text_token(match, :char)
+          elsif match = scan(Regexp.compile(@start_delim))
+            encoder.text_token(match, :delimiter)
+            encoder.end_group(:string)
+            state = :initial
+          elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings
+            encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings
+            encoder.end_group(:string)
+            state = :initial
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :map
+          if match = scan(/[,;]/)
+            encoder.text_token(match, :operator)
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
+            encoder.text_token(match, :key)
+            encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
+            encoder.text_token(scan(/\=/), :operator)
+            state = :initial
+          elsif match = scan(/\s+/m)
+            encoder.text_token(match, :space)
+          else
+            # Note this clause doesn’t advance the scan pointer, it’s a kind of
+            # "retry with other options" (the :initial state then of course
+            # advances the pointer).
+            state = :initial
+          end
+        else
+          raise
+        end
+        
       end
-
-      @encoder.text_token(match, kind)
-
-    elsif match = scan(/\{/) # Opening table brace {
-      @encoder.begin_group(:map)
-      @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
-      @brace_depth += 1
-      @state        = :map
-
-    elsif match = scan(/\}/) # Closing table brace }
-      if @brace_depth == 1
-        @brace_depth = 0
-        @encoder.text_token(match, :delimiter)
-      elsif @brace_depth == 0 # Mismatched brace
-        @encoder.text_token(match, :error)
-      else
-        @brace_depth -= 1
-        @encoder.text_token(match, :inline_delimiter)
-        @state = :map
+      
+      if options[:keep_state]
+        @state = state
       end
-      @encoder.end_group(:map)
-
-    elsif match = scan(/["']/) # String delimiters " and '
-      @encoder.begin_group(:string)
-      @encoder.text_token(match, :delimiter)
-      @start_delim = match
-      @state       = :string
-
-                      # ↓Prefix                hex number ←|→ decimal number
-    elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
-      @encoder.text_token(match, :float)
-
-                      # ↓Prefix         hex number ←|→ decimal number
-    elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
-      @encoder.text_token(match, :integer)
-
-    elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators
-      @encoder.text_token(match, :operator)
-
-    elsif match = scan(/\s+/) # Space
-      @encoder.text_token(match, :space)
-
-    else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors.
-      @encoder.text_token(getch, :error)
-    end
-
-    # It may be that we’re scanning a full-blown subexpression of a table
-    # (tables can contain full expressions in parts).
-    # If this is the case, return to :map scanning state.
-    @state = :map if @state == :initial && @brace_depth >= 1
-
-  when :function_expected
-    if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
-      @encoder.text_token(match, :operator)
-      @state = :initial
-    elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator
-      @encoder.text_token(match, :ident)
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo()
-      @encoder.text_token(match, :function)
-      @state = :initial
-    elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace
-      @encoder.text_token(match, :space)
-    else
-      @encoder.text_token(getch, :error)
-      @state = :initial
-    end
-
-  when :goto_label_expected
-    if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
-      @encoder.text_token(match, :label)
-      @state = :initial
-    elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace
-      @encoder.text_token(match, :space)
-    else
-      @encoder.text_token(getch, :error)
-    end
-
-  when :local_var_expected
-    if match = scan(/function/) # local function ...
-      @encoder.text_token(match, :keyword)
-      @state = :function_expected
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
-      @encoder.text_token(match, :local_variable)
-    elsif match = scan(/,/)
-      @encoder.text_token(match, :operator)
-    elsif match = scan(/\=/)
-      @encoder.text_token(match, :operator)
-      # After encountering the equal sign, arbitrary expressions are
-      # allowed again, so just return to the main state for further
-      # parsing.
-      @state = :initial
-    elsif match = scan(/\n/)
-      @encoder.text_token(match, :space)
-      @state = :initial
-    elsif match = scan(/\s+/)
-      @encoder.text_token(match, :space)
-    else
-      @encoder.text_token(getch, :error)
-    end
-
-  when :long_comment
-    if match = scan(/.*?(?=\]={#@num_equals}\])/m)
-      @encoder.text_token(match, :content)
-
-      delim = scan(/\]={#@num_equals}\]/)
-      @encoder.text_token(delim, :delimiter)
-    else # No terminator found till EOF
-      @encoder.text_token(rest, :error)
-      terminate
-    end
-    @encoder.end_group(:comment)
-    @state = :initial
-
-  when :long_string
-    if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
-      @encoder.text_token(match, :content)
-
-      delim = scan(/\]={#@num_equals}\]/)
-      @encoder.text_token(delim, :delimiter)
-    else # No terminator found till EOF
-      @encoder.text_token(rest, :error)
-      terminate
-    end
-    @encoder.end_group(:string)
-    @state = :initial
-
-  when :string
-    if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
-      @encoder.text_token(match, :content)
-    elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
-      @encoder.text_token(match, :char)
-    elsif match = scan(Regexp.compile(@start_delim))
-      @encoder.text_token(match, :delimiter)
-      @encoder.end_group(:string)
-      @state = :initial
-    elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings
-      @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings
-      @encoder.end_group(:string)
-      @state = :initial
-    else
-      @encoder.text_token(getch, :error)
-    end
-
-  when :map
-    if match = scan(/[,;]/)
-      @encoder.text_token(match, :operator)
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
-      @encoder.text_token(match, :key)
-      @encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
-      @encoder.text_token(scan(/\=/), :operator)
-      @state = :initial
-    elsif match = scan(/\s+/m)
-      @encoder.text_token(match, :space)
-    else
-      # Note this clause doesn’t advance the scan pointer, it’s a kind of
-      # "retry with other options" (the :initial state then of course
-      # advances the pointer).
-      @state = :initial
+      
+      encoder
     end
-  else
-    raise
+    
   end
-
-    end
-
-    @encoder
-  end
-
+  
+end
 end
-- 
cgit v1.2.1


From 90c401c9001e0a670360a55d1189e9f814ae7592 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:17:49 +0200
Subject: some more code cleanups before merge

---
 lib/coderay/scanners/lua.rb | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 3bee275..25bebbe 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -8,17 +8,17 @@ module Scanners
   # The language’s complete syntax is defined in
   # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
   # which is what this scanner tries to conform to.
-  class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+  class Lua < Scanner
     
     register_for :lua
-    file_extension "lua"
-    title "Lua"
+    file_extension 'lua'
+    title 'Lua'
     
     # Keywords used in Lua.
     KEYWORDS = %w[and break do else elseif end
-    for function goto if in
-    local not or repeat return
-    then until while
+      for function goto if in
+      local not or repeat return
+      then until while
     ]
     
     # Constants set by the Lua core.
@@ -36,10 +36,10 @@ module Scanners
     # highlighted directly accordingly, without the need for specific
     # identifiers to be listed here.
     PREDEFINED_EXPRESSIONS = %w[
-    assert collectgarbage dofile error getmetatable
-    ipairs load loadfile next pairs pcall print
-    rawequal rawget rawlen rawset select setmetatable
-    tonumber tostring type xpcall
+      assert collectgarbage dofile error getmetatable
+      ipairs load loadfile next pairs pcall print
+      rawequal rawget rawlen rawset select setmetatable
+      tonumber tostring type xpcall
     ]
     
     # Automatic token kind selection for normal words.
@@ -52,7 +52,7 @@ module Scanners
     
     # Scanner initialization.
     def setup
-      @state       = :initial
+      @state = :initial
       @brace_depth = 0
     end
     
-- 
cgit v1.2.1


From fa9848b6dbd95a4a97a3e63cb99ad2d4c26516df Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:37:06 +0200
Subject: avoid empty tokens in CSS

---
 lib/coderay/scanners/css.rb  | 2 +-
 lib/coderay/scanners/sass.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 732f9c5..6c52c8a 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -148,7 +148,7 @@ module Scanners
             encoder.text_token match[start.size..-2], :content
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if start.size < match.size
           end
           encoder.end_group :function
           
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 167051d..65d40b0 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -176,7 +176,7 @@ module Scanners
             encoder.text_token match[start.size..-2], :content
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if start.size < match.size
           end
           encoder.end_group :function
           
-- 
cgit v1.2.1


From bedd4e4fb5a18270a742e71b65f88fb8e3182050 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:40:56 +0200
Subject: avoid empty tokens in Raydebug

---
 lib/coderay/scanners/raydebug.rb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index 7a21354..d39d962 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -1,11 +1,11 @@
 module CodeRay
 module Scanners
-
+  
   # = Debug Scanner
   # 
   # Parses the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
-
+    
     register_for :raydebug
     file_extension 'raydebug'
     title 'CodeRay Token Dump'
@@ -13,11 +13,11 @@ module Scanners
   protected
     
     def scan_tokens encoder, options
-
+      
       opened_tokens = []
-
+      
       until eos?
-
+        
         if match = scan(/\s+/)
           encoder.text_token match, :space
           
@@ -26,7 +26,7 @@ module Scanners
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
-          encoder.text_token match, kind.to_sym
+          encoder.text_token match, kind.to_sym unless match.empty?
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
@@ -59,8 +59,8 @@ module Scanners
       
       encoder
     end
-
+    
   end
-
+  
 end
 end
-- 
cgit v1.2.1


From 39c074f171eeca1c91aa89757a3f3a12fc78323b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:42:32 +0200
Subject: avoid empty tokens in YAML

---
 lib/coderay/scanners/yaml.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb
index 96f4e93..358a3f1 100644
--- a/lib/coderay/scanners/yaml.rb
+++ b/lib/coderay/scanners/yaml.rb
@@ -47,7 +47,7 @@ module Scanners
           when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/)
             encoder.begin_group :string
             encoder.text_token match, :delimiter
-            encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
+            encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty?
             encoder.text_token match, :delimiter if match = scan(/"/)
             encoder.end_group :string
             next
-- 
cgit v1.2.1


From b059a63e7ff3cd7c26dd5664bf048aa42338fbe1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 05:54:56 +0200
Subject: close open tokens in Groovy scanner

---
 lib/coderay/scanners/groovy.rb | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb
index cf55daf..d04a535 100644
--- a/lib/coderay/scanners/groovy.rb
+++ b/lib/coderay/scanners/groovy.rb
@@ -36,9 +36,12 @@ module Scanners
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     def scan_tokens encoder, options
-      
-      state = :initial
+      state = options[:state] || @state
       inline_block_stack = []
       inline_block_paren_depth = nil
       string_delimiter = nil
@@ -246,6 +249,16 @@ module Scanners
         encoder.end_group state
       end
       
+      if options[:keep_state]
+        @state = state
+      end
+      
+      until inline_block_stack.empty?
+        state, = *inline_block_stack.pop
+        encoder.end_group :inline
+        encoder.end_group state
+      end
+      
       encoder
     end
     
-- 
cgit v1.2.1


From 3c3060acaf2422c23784539b270e365decc9cb2e Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 05:55:23 +0200
Subject: close open tokens in PHP scanner

---
 lib/coderay/scanners/php.rb | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
index 6c68834..40037f9 100644
--- a/lib/coderay/scanners/php.rb
+++ b/lib/coderay/scanners/php.rb
@@ -500,6 +500,10 @@ module Scanners
         
       end
       
+      while state = states.pop
+        encoder.end_group :string if [:sqstring, :dqstring].include? state
+      end
+      
       encoder
     end
     
-- 
cgit v1.2.1


From 7f45168efd1abe679b261a7a91f3641984091d12 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:06:16 +0200
Subject: remove instance variables from Lua scanner

---
 lib/coderay/scanners/lua.rb | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 25bebbe..185c133 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -59,13 +59,15 @@ module Scanners
     # CodeRay entry hook. Starts parsing.
     def scan_tokens(encoder, options)
       state = options[:state] || @state
+      brace_depth = @brace_depth
+      num_equals = nil
       
       until eos?
         case state
         
         when :initial
           if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
-            @num_equals = match.count("=") # Number must match for comment end
+            num_equals = match.count("=") # Number must match for comment end
             encoder.begin_group(:comment)
             encoder.text_token(match, :delimiter)
             state = :long_comment
@@ -74,7 +76,7 @@ module Scanners
             encoder.text_token(match, :comment)
           
           elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
-            @num_equals = match.count("=") # Number must match for comment end
+            num_equals = match.count("=") # Number must match for comment end
             encoder.begin_group(:string)
             encoder.text_token(match, :delimiter)
             state = :long_string
@@ -101,19 +103,19 @@ module Scanners
           
           elsif match = scan(/\{/) # Opening table brace {
             encoder.begin_group(:map)
-            encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
-            @brace_depth += 1
+            encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter)
+            brace_depth += 1
             state        = :map
           
           elsif match = scan(/\}/) # Closing table brace }
-            if @brace_depth == 1
-              @brace_depth = 0
+            if brace_depth == 1
+              brace_depth = 0
               encoder.text_token(match, :delimiter)
               encoder.end_group(:map)
-            elsif @brace_depth == 0 # Mismatched brace
+            elsif brace_depth == 0 # Mismatched brace
               encoder.text_token(match, :error)
             else
-              @brace_depth -= 1
+              brace_depth -= 1
               encoder.text_token(match, :inline_delimiter)
               encoder.end_group(:map)
               state = :map
@@ -122,7 +124,7 @@ module Scanners
           elsif match = scan(/["']/) # String delimiters " and '
             encoder.begin_group(:string)
             encoder.text_token(match, :delimiter)
-            @start_delim = match
+            start_delim = match
             state       = :string
           
                             # ↓Prefix                hex number ←|→ decimal number
@@ -146,7 +148,7 @@ module Scanners
           # It may be that we’re scanning a full-blown subexpression of a table
           # (tables can contain full expressions in parts).
           # If this is the case, return to :map scanning state.
-          state = :map if state == :initial && @brace_depth >= 1
+          state = :map if state == :initial && brace_depth >= 1
         
         when :function_expected
           if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
@@ -198,10 +200,10 @@ module Scanners
           end
         
         when :long_comment
-          if match = scan(/.*?(?=\]={#@num_equals}\])/m)
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m)
             encoder.text_token(match, :content)
             
-            delim = scan(/\]={#@num_equals}\]/)
+            delim = scan(/\]={#{num_equals}}\]/)
             encoder.text_token(delim, :delimiter)
           else # No terminator found till EOF
             encoder.text_token(rest, :error)
@@ -211,10 +213,10 @@ module Scanners
           state = :initial
         
         when :long_string
-          if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences
             encoder.text_token(match, :content)
             
-            delim = scan(/\]={#@num_equals}\]/)
+            delim = scan(/\]={#{num_equals}}\]/)
             encoder.text_token(delim, :delimiter)
           else # No terminator found till EOF
             encoder.text_token(rest, :error)
@@ -224,11 +226,11 @@ module Scanners
           state = :initial
         
         when :string
-          if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+          if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
             encoder.text_token(match, :content)
           elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
             encoder.text_token(match, :char)
-          elsif match = scan(Regexp.compile(@start_delim))
+          elsif match = scan(Regexp.compile(start_delim))
             encoder.text_token(match, :delimiter)
             encoder.end_group(:string)
             state = :initial
-- 
cgit v1.2.1


From 45bb0c576b67b7a3c0dead02078b3a16b5583154 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:06:52 +0200
Subject: close open token groups in Lua scanner

---
 lib/coderay/scanners/lua.rb | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 185c133..fb1e45a 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -268,6 +268,9 @@ module Scanners
         @state = state
       end
       
+      encoder.end_group :string if [:string].include? state
+      brace_depth.times { encoder.end_group :map }
+      
       encoder
     end
     
-- 
cgit v1.2.1


From dc6071129cdc1bcd15129147bbc4d92ba870f007 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:13:32 +0200
Subject: close open token groups in Sass scanner

---
 lib/coderay/scanners/sass.rb | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 65d40b0..0ba383f 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -218,6 +218,14 @@ module Scanners
         @state = states
       end
       
+      while state = states.pop
+        if state == :sass_inline
+          encoder.end_group :inline
+        elsif state == :string
+          encoder.end_group :string
+        end
+      end
+      
       encoder
     end
     
-- 
cgit v1.2.1


From 4327fcbe932ac913f7eb8e92497f97913fb398c5 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:15:55 +0200
Subject: fix Sass regexp modifier

---
 lib/coderay/scanners/sass.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 0ba383f..e20bebe 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -195,7 +195,7 @@ module Scanners
         elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
           encoder.text_token match, :color
           
-        elsif match = scan(/@else if\b|#{RE::AtKeyword}/)
+        elsif match = scan(/@else if\b|#{RE::AtKeyword}/o)
           encoder.text_token match, :directive
           value_expected = true
           
-- 
cgit v1.2.1


From 755904f7046d8a8d6208fe8367aaa3b9a19ecd65 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:30:24 +0200
Subject: avoid empty tokens in Diff scanner

---
 lib/coderay/scanners/diff.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index af0f755..fd1aed6 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -69,7 +69,7 @@ module Scanners
             state = :added
           elsif match = scan(/\\ .*/)
             encoder.text_token match, :comment
-          elsif match = scan(/@@(?>[^@\n]*)@@/)
+          elsif match = scan(/@@(?>[^@\n]+)@@/)
             content_scanner.state = :initial unless match?(/\n\+/)
             content_scanner_entry_state = nil
             if check(/\n|$/)
-- 
cgit v1.2.1


From e1d5e111d968639fa03a6074cf90535ecc90d0dd Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:30:44 +0200
Subject: close correct token groups in Groovy scanner

---
 lib/coderay/scanners/groovy.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb
index d04a535..c64454f 100644
--- a/lib/coderay/scanners/groovy.rb
+++ b/lib/coderay/scanners/groovy.rb
@@ -226,7 +226,7 @@ module Scanners
             encoder.text_token match, :content  # TODO: Shouldn't this be :error?
             
           elsif match = scan(/ \\ | \n /x)
-            encoder.end_group state
+            encoder.end_group state == :regexp ? :regexp : :string
             encoder.text_token match, :error
             after_def = value_expected = false
             state = :initial
@@ -246,7 +246,7 @@ module Scanners
       end
       
       if [:multiline_string, :string, :regexp].include? state
-        encoder.end_group state
+        encoder.end_group state == :regexp ? :regexp : :string
       end
       
       if options[:keep_state]
@@ -256,7 +256,7 @@ module Scanners
       until inline_block_stack.empty?
         state, = *inline_block_stack.pop
         encoder.end_group :inline
-        encoder.end_group state
+        encoder.end_group state == :regexp ? :regexp : :string
       end
       
       encoder
-- 
cgit v1.2.1


From 603ff7d0b14521cfd0408aa68e2e1cb6ea9086bc Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:36:46 +0200
Subject: avoid empty tokens in YAML scanner

---
 lib/coderay/scanners/yaml.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb
index 358a3f1..32c8e2c 100644
--- a/lib/coderay/scanners/yaml.rb
+++ b/lib/coderay/scanners/yaml.rb
@@ -84,7 +84,7 @@ module Scanners
           when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/)
             encoder.begin_group :key
             encoder.text_token match[0,1], :delimiter
-            encoder.text_token match[1..-2], :content
+            encoder.text_token match[1..-2], :content if match.size > 2
             encoder.text_token match[-1,1], :delimiter
             encoder.end_group :key
             key_indent = column(pos - match.size) - 1
-- 
cgit v1.2.1


From 253a616924000cc5d522a2d4b8030f5693c98e43 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 06:39:30 +0200
Subject: close open string token groups in SQL scanner

---
 lib/coderay/scanners/sql.rb | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb
index b757278..93aeaf3 100644
--- a/lib/coderay/scanners/sql.rb
+++ b/lib/coderay/scanners/sql.rb
@@ -1,8 +1,9 @@
-module CodeRay module Scanners
+module CodeRay
+module Scanners
   
   # by Josh Goebel
   class SQL < Scanner
-
+    
     register_for :sql
     
     KEYWORDS = %w(
@@ -149,6 +150,7 @@ module CodeRay module Scanners
               string_content = ''
             end
             encoder.text_token match, :error unless match.empty?
+            encoder.end_group :string
             state = :initial
           else
             raise "else case \" reached; %p not handled." % peek(1), encoder
@@ -171,4 +173,5 @@ module CodeRay module Scanners
     
   end
   
-end end
\ No newline at end of file
+end
+end
-- 
cgit v1.2.1


From c8751fbc09d4a8f43b0b037e3a7dc7ddbfbcacb1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 13:15:10 +0200
Subject: fix unclosed token group in Ruby scanner

---
 lib/coderay/scanners/ruby.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index c282f31..80165ca 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -269,7 +269,7 @@ module Scanners
             end
             
             if last_state
-              state = last_state
+              state = last_state unless state.is_a?(StringState)  # otherwise, a simple 'def"' results in unclosed tokens
               last_state = nil
             end
             
-- 
cgit v1.2.1


From c34ad739bc7342aa40830945a5e19e90e9814e04 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 13:27:43 +0200
Subject: fix token nesting in PHP scanner

---
 lib/coderay/scanners/php.rb | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
index 40037f9..7a8d75d 100644
--- a/lib/coderay/scanners/php.rb
+++ b/lib/coderay/scanners/php.rb
@@ -265,7 +265,7 @@ module Scanners
             @html_scanner.tokenize match unless match.empty?
           end
         
-        when :php
+        when :php, :php_inline
           if match = scan(/\s+/)
             encoder.text_token match, :space
           
@@ -332,7 +332,7 @@ module Scanners
             if states.size == 1
               encoder.text_token match, :error
             else
-              states.pop
+              state = states.pop
               if states.last.is_a?(::Array)
                 delimiter = states.last[1]
                 states[-1] = states.last[0]
@@ -340,6 +340,7 @@ module Scanners
                 encoder.end_group :inline
               else
                 encoder.text_token match, :operator
+                encoder.end_group :inline if state == :php_inline
                 label_expected = true
               end
             end
@@ -350,7 +351,14 @@ module Scanners
           
           elsif match = scan(RE::PHP_END)
             encoder.text_token match, :inline_delimiter
-            states = [:initial]
+            while state = states.pop
+              encoder.end_group :string if [:sqstring, :dqstring].include? state
+              if state.is_a? Array
+                encoder.end_group :inline
+                encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+              end
+            end
+            states << :initial
           
           elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
             encoder.begin_group :string
@@ -400,6 +408,7 @@ module Scanners
           elsif match = scan(/\\/)
             encoder.text_token match, :error
           else
+            encoder.end_group :string
             states.pop
           end
         
@@ -459,7 +468,7 @@ module Scanners
               encoder.begin_group :inline
               states[-1] = [states.last, delimiter]
               delimiter = nil
-              states.push :php
+              states.push :php_inline
               encoder.text_token match, :delimiter
             else
               encoder.text_token match, :content
@@ -469,6 +478,7 @@ module Scanners
           elsif match = scan(/\$/)
             encoder.text_token match, :content
           else
+            encoder.end_group :string
             states.pop
           end
         
@@ -502,6 +512,10 @@ module Scanners
       
       while state = states.pop
         encoder.end_group :string if [:sqstring, :dqstring].include? state
+        if state.is_a? Array
+          encoder.end_group :inline
+          encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+        end
       end
       
       encoder
-- 
cgit v1.2.1


From 14f3a4f28341237503a51bfa0764c922bb02bdfa Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 14:13:15 +0200
Subject: tweak HTML CDATA token kinds

---
 lib/coderay/scanners/html.rb | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index fcf249a..530ce44 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -1,13 +1,13 @@
 module CodeRay
 module Scanners
-
+  
   # HTML Scanner
   # 
   # Alias: +xhtml+
   # 
   # See also: Scanners::XML
   class HTML < Scanner
-
+    
     register_for :html
     
     KINDS_NOT_LOC = [
@@ -100,15 +100,13 @@ module Scanners
           
           when :initial
             if match = scan(/<!\[CDATA\[/)
-              encoder.begin_group :string
-              encoder.text_token match, :delimiter
+              encoder.text_token match, :inline_delimiter
               if match = scan(/.*?\]\]>/m)
-                encoder.text_token match[0..-4], :content
-                encoder.text_token ']]>', :delimiter
-              else
-                encoder.text_token scan(/.*/m), :error
+                encoder.text_token match[0..-4], :plain
+                encoder.text_token ']]>', :inline_delimiter
+              elsif match = scan(/.+/)
+                encoder.text_token match, :error
               end
-              encoder.end_group :string
             elsif match = scan(/<!--(?:.*?-->|.*)/m)
               encoder.text_token match, :comment
             elsif match = scan(/<!(\w+)(?:.*?>|.*)|\]>/m)
-- 
cgit v1.2.1


From 5b0dc0f35090949d7512a85dadc8cf2871b91aac Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 14:15:01 +0200
Subject: scan_css in HTML scanner (tags)

---
 lib/coderay/scanners/html.rb | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 3ba3b79..06728e4 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -75,9 +75,14 @@ module Scanners
     def scan_java_script encoder, code
       if code && !code.empty?
         @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true
-        # encoder.begin_group :inline
         @java_script_scanner.tokenize code, :tokens => encoder
-        # encoder.end_group :inline
+      end
+    end
+    
+    def scan_css encoder, code
+      if code && !code.empty?
+        @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true
+        @css_scanner.tokenize code, :tokens => encoder
       end
     end
     
@@ -110,7 +115,7 @@ module Scanners
             elsif match = scan(/<\/[-\w.:]*>?/m)
               in_tag = nil
               encoder.text_token match, :tag
-            elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m)
+            elsif match = scan(/<(?:(script|style)|[-\w.:]+)(>)?/m)
               encoder.text_token match, :tag
               in_tag = self[1]
               if self[2]
@@ -206,19 +211,23 @@ module Scanners
             
           when :in_special_tag
             case in_tag
-            when 'script'
+            when 'script', 'style'
               encoder.text_token match, :space if match = scan(/[ \t]*\n/)
               if scan(/(\s*<!--)(?:(.*?)(-->)|(.*))/m)
                 code = self[2] || self[4]
                 closing = self[3]
                 encoder.text_token self[1], :comment
               else
-                code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/)
+                code = scan_until(/(?=(?:\n\s*)?<\/#{in_tag}>)|\z/)
                 closing = false
               end
               unless code.empty?
                 encoder.begin_group :inline
-                scan_java_script encoder, code
+                if in_tag == 'script'
+                  scan_java_script encoder, code
+                else
+                  scan_css encoder, code
+                end
                 encoder.end_group :inline
               end
               encoder.text_token closing, :comment if closing
-- 
cgit v1.2.1


From 7ada74c8f0f77815393e887db83e1d2c36ce7235 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 14:32:14 +0200
Subject: scan_css in HTML scanner (arguments), change token kind from :inline
 to :string

---
 lib/coderay/scanners/html.rb | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 06728e4..f1dfba0 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -33,7 +33,8 @@ module Scanners
     )
     
     IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil).
-      add(EVENT_ATTRIBUTES, :script)
+      add(EVENT_ATTRIBUTES, :script).
+      add(['style'], :style)
     
     ATTR_NAME = /[\w.:-]+/  # :nodoc:
     TAG_END = /\/?>/  # :nodoc:
@@ -79,10 +80,10 @@ module Scanners
       end
     end
     
-    def scan_css encoder, code
+    def scan_css encoder, code, state = [:initial]
       if code && !code.empty?
         @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true
-        @css_scanner.tokenize code, :tokens => encoder
+        @css_scanner.tokenize code, :tokens => encoder, :state => state
       end
     end
     
@@ -166,17 +167,21 @@ module Scanners
               encoder.text_token match, :attribute_value
               state = :attribute
             elsif match = scan(/["']/)
-              if in_attribute == :script
-                encoder.begin_group :inline
-                encoder.text_token match, :inline_delimiter
+              if in_attribute == :script || in_attribute == :style
+                encoder.begin_group :string
+                encoder.text_token match, :delimiter
                 if scan(/javascript:[ \t]*/)
                   encoder.text_token matched, :comment
                 end
                 code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/)
-                scan_java_script encoder, code
+                if in_attribute == :script
+                  scan_java_script encoder, code
+                else
+                  scan_css encoder, code, [:block]
+                end
                 match = scan(/["']/)
-                encoder.text_token match, :inline_delimiter if match
-                encoder.end_group :inline
+                encoder.text_token match, :delimiter if match
+                encoder.end_group :string
                 state = :attribute
                 in_attribute = nil
               else
-- 
cgit v1.2.1


From 56f4163e99689912c3797e5d3b2b97244ce65782 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 14:33:22 +0200
Subject: fix another CSS empty token issue

---
 lib/coderay/scanners/css.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 6c52c8a..9ed4618 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -145,10 +145,10 @@ module Scanners
           start = match[/^\w+\(/]
           encoder.text_token start, :delimiter
           if match[-1] == ?)
-            encoder.text_token match[start.size..-2], :content
+            encoder.text_token match[start.size..-2], :content if match.size > start.size + 1
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content if start.size < match.size
+            encoder.text_token match[start.size..-1], :content if match.size > start.size
           end
           encoder.end_group :function
           
-- 
cgit v1.2.1


From b89caf96d1cfc304c2114d8734ebe8b91337c799 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 14:53:14 +0200
Subject: fix diff/json token nesting issue

---
 lib/coderay/scanners/json.rb | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 3754a9b..b09970c 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -26,6 +26,10 @@ module Scanners
     def scan_tokens encoder, options
       state = options[:state] || @state
       
+      if [:string, :key].include? state
+        encoder.begin_group state
+      end
+      
       until eos?
         
         case state
-- 
cgit v1.2.1


From fd8c81f5c338e4f1448007c9100d15d912fadd27 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Sun, 23 Jun 2013 09:43:54 -0600
Subject: additional types: string, error

http://golang.org/ref/spec#Predeclared_identifiers
---
 lib/coderay/scanners/go.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 4431ef2..a66a5e3 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -23,7 +23,7 @@ module Scanners
       'int8', 'int16', 'int32', 'int64',
       'float32', 'float64',
       'complex64', 'complex128',
-      'byte', 'rune',
+      'byte', 'rune', 'string', 'error',
       'uint', 'int', 'uintptr',
     ]  # :nodoc:
 
-- 
cgit v1.2.1


From 004d0c83222113ff732101a3e25785ce0625bfa2 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 18:14:29 +0200
Subject: whitespace

---
 lib/coderay/scanners/go.rb | 82 +++++++++++++++++++++++-----------------------
 1 file changed, 41 insertions(+), 41 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 4431ef2..d4a3598 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -1,12 +1,12 @@
 module CodeRay
 module Scanners
-
+  
   # Scanner for Go, copy from c
   class Go < Scanner
-
+    
     register_for :go
     file_extension 'go'
-
+    
     # http://golang.org/ref/spec#Keywords
     KEYWORDS = [
       'break', 'default', 'func', 'interface', 'select',
@@ -15,7 +15,7 @@ module Scanners
       'const', 'fallthrough', 'if', 'range', 'type',
       'continue', 'for', 'import', 'return', 'var',
     ]  # :nodoc:
-
+    
     # http://golang.org/ref/spec#Types
     PREDEFINED_TYPES = [
       'bool',
@@ -26,51 +26,51 @@ module Scanners
       'byte', 'rune',
       'uint', 'int', 'uintptr',
     ]  # :nodoc:
-
+    
     PREDEFINED_CONSTANTS = [
       'nil', 'iota',
       'true', 'false',
     ]  # :nodoc:
-
+    
     DIRECTIVES = [
       'go_no_directive',  # Seems no directive concept in Go?
     ]  # :nodoc:
-
+    
     IDENT_KIND = WordList.new(:ident).
       add(KEYWORDS, :keyword).
       add(PREDEFINED_TYPES, :predefined_type).
       add(DIRECTIVES, :directive).
       add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
-
+    
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
-
-  protected
-
+    
+    protected
+    
     def scan_tokens encoder, options
-
+      
       state = :initial
       label_expected = true
       case_expected = false
       label_expected_before_preproc_line = nil
       in_preproc_line = false
-
+      
       until eos?
-
+        
         case state
-
+        
         when :initial
-
+          
           if match = scan(/ \s+ | \\\n /x)
             if in_preproc_line && match != "\\\n" && match.index(?\n)
               in_preproc_line = false
               label_expected = label_expected_before_preproc_line
             end
             encoder.text_token match, :space
-
+          
           elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
             encoder.text_token match, :comment
-
+          
           elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
             label_expected = match =~ /[;\{\}]/
             if case_expected
@@ -78,7 +78,7 @@ module Scanners
               case_expected = false
             end
             encoder.text_token match, :operator
-
+          
           elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
             kind = IDENT_KIND[match]
             if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
@@ -94,7 +94,7 @@ module Scanners
               end
             end
             encoder.text_token match, kind
-
+          
           elsif match = scan(/L?"/)
             encoder.begin_group :string
             if match[0] == ?L
@@ -107,41 +107,41 @@ module Scanners
           elsif match = scan(/ \# \s* if \s* 0 /x)
             match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
             encoder.text_token match, :comment
-
+          
           elsif match = scan(/#[ \t]*(\w*)/)
             encoder.text_token match, :preprocessor
             in_preproc_line = true
             label_expected_before_preproc_line = label_expected
             state = :include_expected if self[1] == 'include'
-
+          
           elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
             label_expected = false
             encoder.text_token match, :char
-
+          
           elsif match = scan(/\$/)
             encoder.text_token match, :ident
-
+          
           elsif match = scan(/0[xX][0-9A-Fa-f]+/)
             label_expected = false
             encoder.text_token match, :hex
-
+          
           elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/)
             label_expected = false
             encoder.text_token match, :octal
-
+          
           elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
             label_expected = false
             encoder.text_token match, :integer
-
+          
           elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
             label_expected = false
             encoder.text_token match, :float
-
+          
           else
             encoder.text_token getch, :error
-
+          
           end
-
+        
         when :string
           if match = scan(/[^\\\n"]+/)
             encoder.text_token match, :content
@@ -160,36 +160,36 @@ module Scanners
           else
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
-
+        
         when :include_expected
           if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
             encoder.text_token match, :include
             state = :initial
-
+          
           elsif match = scan(/\s+/)
             encoder.text_token match, :space
             state = :initial if match.index ?\n
-
+          
           else
             state = :initial
-
+          
           end
-
+        
         else
           raise_inspect 'Unknown state', encoder
-
+        
         end
-
+        
       end
-
+      
       if state == :string
         encoder.end_group :string
       end
-
+      
       encoder
     end
-
+    
   end
-
+  
 end
 end
-- 
cgit v1.2.1


From afa2be73c03d033056024354787abd66f2ff7fa0 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 18:15:05 +0200
Subject: add string as predefined type

---
 lib/coderay/scanners/go.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index d4a3598..dbf9595 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -23,7 +23,7 @@ module Scanners
       'int8', 'int16', 'int32', 'int64',
       'float32', 'float64',
       'complex64', 'complex128',
-      'byte', 'rune',
+      'byte', 'rune', 'string',
       'uint', 'int', 'uintptr',
     ]  # :nodoc:
     
-- 
cgit v1.2.1


From 7ef6f7783d788bf656a3c59600cd45cd5b694376 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 18:15:22 +0200
Subject: add support for raw strings in Go

---
 lib/coderay/scanners/go.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index dbf9595..c0c602d 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -103,7 +103,14 @@ module Scanners
             end
             encoder.text_token match, :delimiter
             state = :string
-
+          
+          elsif match = scan(/ ` ([^`]+)? (`)? /x)
+            encoder.begin_group :shell
+            encoder.text_token '`', :delimiter
+            encoder.text_token self[1], :content if self[1]
+            encoder.text_token self[2], :delimiter if self[2]
+            encoder.end_group :shell
+          
           elsif match = scan(/ \# \s* if \s* 0 /x)
             match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
             encoder.text_token match, :comment
-- 
cgit v1.2.1


From a24c207a7e547408de0f6157d495b0dd2e8d5a40 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 18:15:36 +0200
Subject: fix empty token in Go scanner

---
 lib/coderay/scanners/go.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index c0c602d..76a0820 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -161,7 +161,7 @@ module Scanners
             encoder.text_token match, :char
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group :string
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
             label_expected = false
           else
-- 
cgit v1.2.1


From 4669b7086136647f2eece41a122fa204eb16e1e9 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 18:16:47 +0200
Subject: fix label_expected (test case?)

---
 lib/coderay/scanners/go.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 76a0820..23cbfe6 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -64,6 +64,7 @@ module Scanners
           if match = scan(/ \s+ | \\\n /x)
             if in_preproc_line && match != "\\\n" && match.index(?\n)
               in_preproc_line = false
+              case_expected = false
               label_expected = label_expected_before_preproc_line
             end
             encoder.text_token match, :space
@@ -72,7 +73,6 @@ module Scanners
             encoder.text_token match, :comment
           
           elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
-            label_expected = match =~ /[;\{\}]/
             if case_expected
               label_expected = true if match == ':'
               case_expected = false
@@ -83,6 +83,7 @@ module Scanners
             kind = IDENT_KIND[match]
             if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
               kind = :label
+              label_expected = false
               match << matched
             else
               label_expected = false
-- 
cgit v1.2.1


From 85275cf21e7d15459abc11fd76606bd7d38fb8b5 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Sun, 23 Jun 2013 12:11:49 -0600
Subject: Go doesn't have a "f" suffix for floats like C.

---
 lib/coderay/scanners/go.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 49d24c2..afcfca5 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -141,7 +141,7 @@ module Scanners
             label_expected = false
             encoder.text_token match, :integer
           
-          elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+          elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
             label_expected = false
             encoder.text_token match, :float
           
-- 
cgit v1.2.1


From cad9a00e28d61781d5a12a0556be7126eb790725 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Sun, 23 Jun 2013 12:35:26 -0600
Subject: add imaginary numbers to Go scanner

---
 lib/coderay/scanners/go.rb | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index afcfca5..04504ab 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -128,7 +128,11 @@ module Scanners
           
           elsif match = scan(/\$/)
             encoder.text_token match, :ident
-          
+
+          elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/)
+            label_expected = false
+            encoder.text_token match, :imaginary
+
           elsif match = scan(/0[xX][0-9A-Fa-f]+/)
             label_expected = false
             encoder.text_token match, :hex
@@ -137,13 +141,13 @@ module Scanners
             label_expected = false
             encoder.text_token match, :octal
           
-          elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
-            label_expected = false
-            encoder.text_token match, :integer
-          
           elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
             label_expected = false
             encoder.text_token match, :float
+
+          elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
+            label_expected = false
+            encoder.text_token match, :integer
           
           else
             encoder.text_token getch, :error
-- 
cgit v1.2.1


From 4c877bf0eb655cf2c04a0efb24b4e5f83b97e160 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Sun, 23 Jun 2013 12:43:50 -0600
Subject: predeclared identifiers

http://golang.org/ref/spec#Predeclared_identifiers
---
 lib/coderay/scanners/go.rb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 04504ab..ae50aa4 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -31,6 +31,11 @@ module Scanners
       'nil', 'iota',
       'true', 'false',
     ]  # :nodoc:
+
+    PREDEFINED_FUNCTIONS = %w[
+      append cap close complex copy delete imag len
+      make new panic print println real recover
+    ] # :nodoc:
     
     DIRECTIVES = [
       'go_no_directive',  # Seems no directive concept in Go?
@@ -40,7 +45,8 @@ module Scanners
       add(KEYWORDS, :keyword).
       add(PREDEFINED_TYPES, :predefined_type).
       add(DIRECTIVES, :directive).
-      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
+      add(PREDEFINED_FUNCTIONS, :predefined)  # :nodoc:
     
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
-- 
cgit v1.2.1


From dd9ec43a3ea4cfce8eb1e8dd38504a9d8f24df54 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Sun, 23 Jun 2013 12:46:15 -0600
Subject: yup, no C-style directives (auto extern static)

---
 lib/coderay/scanners/go.rb | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index ae50aa4..eb06fb0 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -37,14 +37,9 @@ module Scanners
       make new panic print println real recover
     ] # :nodoc:
     
-    DIRECTIVES = [
-      'go_no_directive',  # Seems no directive concept in Go?
-    ]  # :nodoc:
-    
     IDENT_KIND = WordList.new(:ident).
       add(KEYWORDS, :keyword).
       add(PREDEFINED_TYPES, :predefined_type).
-      add(DIRECTIVES, :directive).
       add(PREDEFINED_CONSTANTS, :predefined_constant).
       add(PREDEFINED_FUNCTIONS, :predefined)  # :nodoc:
     
-- 
cgit v1.2.1


From 94e4bb3366537def28ed257fcf2b70a634711c9f Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Fri, 12 Jul 2013 14:57:46 +0200
Subject: don't change value of objects you don't own

---
 lib/coderay/scanners/css.rb  | 2 +-
 lib/coderay/scanners/sass.rb | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 9ed4618..5977b9c 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -53,7 +53,7 @@ module Scanners
     end
     
     def scan_tokens encoder, options
-      states = Array(options[:state] || @state)
+      states = Array(options[:state] || @state).dup
       value_expected = @value_expected
       
       until eos?
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index e20bebe..7ba9bf5 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -19,7 +19,7 @@ module Scanners
     end
     
     def scan_tokens encoder, options
-      states = Array(options[:state] || @state)
+      states = Array(options[:state] || @state).dup
       string_delimiter = nil
       
       until eos?
@@ -119,7 +119,7 @@ module Scanners
           
           else
             #:nocov:
-            raise_inspect 'Unknown state', encoder
+            raise_inspect 'Unknown state: %p' % [states.last], encoder
             #:nocov:
             
           end
@@ -215,7 +215,7 @@ module Scanners
       end
       
       if options[:keep_state]
-        @state = states
+        @state = states.dup
       end
       
       while state = states.pop
-- 
cgit v1.2.1


From 990ed25fd1bf350dabae4bed031e07ee485beb79 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 13 Jul 2013 16:20:59 +0200
Subject: split '" string states in Sass scanner (edge case)

---
 lib/coderay/scanners/sass.rb | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 7ba9bf5..85b4711 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -7,11 +7,6 @@ module Scanners
     register_for :sass
     file_extension 'sass'
     
-    STRING_CONTENT_PATTERN = {
-      "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
-      '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
-    }
-    
   protected
     
     def setup
@@ -20,7 +15,8 @@ module Scanners
     
     def scan_tokens encoder, options
       states = Array(options[:state] || @state).dup
-      string_delimiter = nil
+      
+      encoder.begin_group :string if states.last == :sqstring || states.last == :dqstring
       
       until eos?
         
@@ -91,24 +87,23 @@ module Scanners
               next
             end
             
-          when :string
-            if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
+          when :sqstring, :dqstring
+            if match = scan(states.last == :sqstring ? /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o : /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o)
               encoder.text_token match, :content
             elsif match = scan(/['"]/)
               encoder.text_token match, :delimiter
               encoder.end_group :string
-              string_delimiter = nil
               states.pop
             elsif match = scan(/#\{/)
               encoder.begin_group :inline
               encoder.text_token match, :inline_delimiter
               states.push :sass_inline
             elsif match = scan(/ \\ | $ /x)
-              encoder.end_group :string
+              encoder.end_group states.last
               encoder.text_token match, :error unless match.empty?
               states.pop
             else
-              raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
+              raise_inspect "else case #{states.last} reached; %p not handled." % peek(1), encoder
             end
           
           when :include
@@ -157,15 +152,15 @@ module Scanners
           
         elsif match = scan(/['"]/)
           encoder.begin_group :string
-          string_delimiter = match
           encoder.text_token match, :delimiter
           if states.include? :sass_inline
-            content = scan_until(/(?=#{string_delimiter}|\}|\z)/)
+            # no nesting, just scan the string until delimiter
+            content = scan_until(/(?=#{match}|\}|\z)/)
             encoder.text_token content, :content unless content.empty?
-            encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/)
+            encoder.text_token match, :delimiter if scan(/#{match}/)
             encoder.end_group :string
           else
-            states.push :string
+            states.push match == "'" ? :sqstring : :dqstring
           end
           
         elsif match = scan(/#{RE::Function}/o)
@@ -221,7 +216,7 @@ module Scanners
       while state = states.pop
         if state == :sass_inline
           encoder.end_group :inline
-        elsif state == :string
+        elsif state == :sqstring || state == :dqstring
           encoder.end_group :string
         end
       end
-- 
cgit v1.2.1


From def7e09db1963368e20bfd53c72532d6a631e0e8 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 13 Jul 2013 16:21:42 +0200
Subject: fix #139: don't scan for :include after eos

---
 lib/coderay/scanners/sass.rb | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 85b4711..1bbd534 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -209,6 +209,8 @@ module Scanners
         
       end
       
+      states.pop if states.last == :include
+      
       if options[:keep_state]
         @state = states.dup
       end
-- 
cgit v1.2.1


From 62a0be9509f8814902a4a97df4ad84913728d059 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 13 Jul 2013 16:34:17 +0200
Subject: fix #143 (Sass scanner key vs tag heuristic)

---
 lib/coderay/scanners/sass.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 1bbd534..e3296b9 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -44,7 +44,7 @@ module Scanners
         elsif case states.last
           when :initial, :media, :sass_inline
             if match = scan(/(?>#{RE::Ident})(?!\()/ox)
-              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag)
+              encoder.text_token match, value_expected ? :value : (check(/.*:(?![a-z])/) ? :key : :tag)
               next
             elsif !value_expected && (match = scan(/\*/))
               encoder.text_token match, :tag
-- 
cgit v1.2.1


From 70c9ba896e1bba5ac727fb6fdfc3ba94510e652d Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 13 Jul 2013 16:34:41 +0200
Subject: fix CSS scanner for things like "nth-child(2n)"

---
 lib/coderay/scanners/css.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 5977b9c..55d5239 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -25,7 +25,7 @@ module Scanners
       
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
       
-      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/
+      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
-- 
cgit v1.2.1


From bbe4d72ba785f1bd6fd703d63b096a907da1b09f Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 13 Jul 2013 20:31:34 +0200
Subject: tweak numeral tokens handling (#147)

---
 lib/coderay/scanners/go.rb | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index eb06fb0..938da9d 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -31,7 +31,7 @@ module Scanners
       'nil', 'iota',
       'true', 'false',
     ]  # :nodoc:
-
+    
     PREDEFINED_FUNCTIONS = %w[
       append cap close complex copy delete imag len
       make new panic print println real recover
@@ -73,7 +73,7 @@ module Scanners
           elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
             encoder.text_token match, :comment
           
-          elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+          elsif match = scan(/ <?- (?![\d.]) | [+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
             if case_expected
               label_expected = true if match == ':'
               case_expected = false
@@ -129,24 +129,24 @@ module Scanners
           
           elsif match = scan(/\$/)
             encoder.text_token match, :ident
-
-          elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/)
+          
+          elsif match = scan(/-?\d*(\.\d*)?([eE][+-]?\d+)?i/)
             label_expected = false
             encoder.text_token match, :imaginary
-
-          elsif match = scan(/0[xX][0-9A-Fa-f]+/)
+          
+          elsif match = scan(/-?0[xX][0-9A-Fa-f]+/)
             label_expected = false
             encoder.text_token match, :hex
           
-          elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/)
+          elsif match = scan(/-?(?:0[0-7]+)(?![89.eEfF])/)
             label_expected = false
             encoder.text_token match, :octal
           
-          elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+          elsif match = scan(/-?(?:\d*\.\d+|\d+\.)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
             label_expected = false
             encoder.text_token match, :float
-
-          elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
+          
+          elsif match = scan(/-?(?:\d+)(?![.eEfF])L?L?/)
             label_expected = false
             encoder.text_token match, :integer
           
-- 
cgit v1.2.1


From 6dd14ef018ee4417771504117819121bd4b8520d Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 13 Jul 2013 20:34:10 +0200
Subject: be a bit more graceful with buggy Go strings

---
 lib/coderay/scanners/go.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 938da9d..5034adc 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -165,9 +165,10 @@ module Scanners
             label_expected = false
           elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
             encoder.text_token match, :char
-          elsif match = scan(/ \\ | $ /x)
+          elsif match = scan(/ \\ /x)
+            encoder.text_token match, :error
+          elsif match = scan(/$/)
             encoder.end_group :string
-            encoder.text_token match, :error unless match.empty?
             state = :initial
             label_expected = false
           else
-- 
cgit v1.2.1


From 82864efabda9dc17fa6a28b0740ffc8f58706126 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 13 Jul 2013 20:36:27 +0200
Subject: allow unicode characters in char literals

---
 lib/coderay/scanners/go.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 5034adc..59473f6 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -44,7 +44,7 @@ module Scanners
       add(PREDEFINED_FUNCTIONS, :predefined)  # :nodoc:
     
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
     
     protected
     
@@ -123,7 +123,7 @@ module Scanners
             label_expected_before_preproc_line = label_expected
             state = :include_expected if self[1] == 'include'
           
-          elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+          elsif match = scan(/ L?' (?: [^\'\n\\] | \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) )? '? /ox)
             label_expected = false
             encoder.text_token match, :char
           
-- 
cgit v1.2.1


From 59ca07b0d1a1710ab729636ea00de4b638f56110 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 20 Jul 2013 11:17:30 +0200
Subject: =?UTF-8?q?add=20Ruby=202=20syntax:=20%i(=E2=80=A6)=20and=20%I(?=
 =?UTF-8?q?=E2=80=A6)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 lib/coderay/scanners/ruby/patterns.rb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb
index ed071d2..0b36e13 100644
--- a/lib/coderay/scanners/ruby/patterns.rb
+++ b/lib/coderay/scanners/ruby/patterns.rb
@@ -157,13 +157,16 @@ module Scanners
       yield
     ])
     
-    FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
+    FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
     FANCY_STRING_KIND = Hash.new(:string).merge({
+      'i' => :symbol,
+      'I' => :symbol,
       'r' => :regexp,
       's' => :symbol,
       'x' => :shell,
     })
     FANCY_STRING_INTERPRETED = Hash.new(true).merge({
+      'i' => false,
       'q' => false,
       's' => false,
       'w' => false,
-- 
cgit v1.2.1


From 5c23a731ca55729fc65630eca3b37a5b1a71e5b1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 20 Jul 2013 18:12:04 +0200
Subject: mark possibly problematic spots with FIXME

---
 lib/coderay/scanners/debug.rb             | 2 ++
 lib/coderay/scanners/diff.rb              | 1 +
 lib/coderay/scanners/python.rb            | 3 +++
 lib/coderay/scanners/raydebug.rb          | 2 ++
 lib/coderay/scanners/ruby/string_state.rb | 1 +
 5 files changed, 9 insertions(+)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index 566bfa7..9d10864 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -21,6 +21,7 @@ module Scanners
           encoder.text_token match, :space
           
         elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
+          # FIXME: cache attack
           kind = self[1].to_sym
           match = self[2].gsub(/\\(.)/m, '\1')
           unless TokenKinds.has_key? kind
@@ -30,6 +31,7 @@ module Scanners
           encoder.text_token match, kind
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
+          # FIXME: cache attack
           kind = self[1].to_sym
           opened_tokens << kind
           case self[2]
diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index fd1aed6..836fa41 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -21,6 +21,7 @@ module Scanners
       line_kind = nil
       state = :initial
       deleted_lines_count = 0
+      # FIXME: cache attack
       scanners = Hash.new do |h, lang|
         h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true
       end
diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index 09c8b6e..23630f9 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -75,10 +75,12 @@ module Scanners
       <<=? | >>=? | [<>=]=? | !=  # comparison and assignment
     /x  # :nodoc:
     
+    # FIXME: cache attack
     STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter|
       h[delimiter] = Regexp.union delimiter  # :nodoc:
     }
     
+    # FIXME: cache attack
     STRING_CONTENT_REGEXP = Hash.new { |h, delimiter|
       h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x  # :nodoc:
     }
@@ -183,6 +185,7 @@ module Scanners
               kind = :ident
             elsif kind == :keyword
               state = DEF_NEW_STATE[match]
+              # FIXME: cache attack
               from_import_state << match.to_sym if state == :include_expected
             end
             encoder.text_token match, kind
diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index d39d962..ca35de0 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -26,6 +26,7 @@ module Scanners
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
+          # FIXME: cache attack
           encoder.text_token match, kind.to_sym unless match.empty?
           encoder.text_token match, :operator if match = scan(/\)/)
           
@@ -39,6 +40,7 @@ module Scanners
           else
             raise 'CodeRay bug: This case should not be reached.'
           end
+          # FIXME: cache attack
           kind = kind.to_sym
           opened_tokens << kind
           encoder.begin_group kind
diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb
index 2f398d1..fe37d07 100644
--- a/lib/coderay/scanners/ruby/string_state.rb
+++ b/lib/coderay/scanners/ruby/string_state.rb
@@ -14,6 +14,7 @@ module Scanners
         { }
       ] ].each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
       
+      # FIXME: cache attack
       STRING_PATTERN = Hash.new do |h, k|
         delim, interpreted = *k
         # delim = delim.dup  # workaround for old Ruby
-- 
cgit v1.2.1


From e2546068d0f16fcba15268e740bbb6d9f4f223e9 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 18:28:54 +0200
Subject: prevent Symbol attack in Raydebug scanner

---
 lib/coderay/scanners/raydebug.rb | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index ca35de0..6c1c10f 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -1,3 +1,5 @@
+require 'set'
+
 module CodeRay
 module Scanners
   
@@ -12,6 +14,11 @@ module Scanners
     
   protected
     
+    def setup
+      super
+      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
+    end
+    
     def scan_tokens encoder, options
       
       opened_tokens = []
@@ -26,8 +33,13 @@ module Scanners
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
-          # FIXME: cache attack
-          encoder.text_token match, kind.to_sym unless match.empty?
+          unless match.empty?
+            if @known_token_kinds.include? kind
+              encoder.text_token match, kind.to_sym
+            else
+              encoder.text_token match, :plain
+            end
+          end
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-- 
cgit v1.2.1


From 2ab42c7b5e674453fac0320fe0c4a40daf6197e1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 18:53:41 +0200
Subject: prevent Symbol attack in Debug scanner

---
 lib/coderay/scanners/debug.rb | 39 ++++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index 9d10864..ac12c16 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -1,3 +1,5 @@
+require 'set'
+
 module CodeRay
 module Scanners
   
@@ -11,6 +13,11 @@ module Scanners
     
   protected
     
+    def setup
+      super
+      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
+    end
+    
     def scan_tokens encoder, options
       
       opened_tokens = []
@@ -21,26 +28,24 @@ module Scanners
           encoder.text_token match, :space
           
         elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
-          # FIXME: cache attack
-          kind = self[1].to_sym
-          match = self[2].gsub(/\\(.)/m, '\1')
-          unless TokenKinds.has_key? kind
-            kind = :error
-            match = matched
+          if @known_token_kinds.include? self[1]
+            encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
+          else
+            encoder.text_token matched, :error
           end
-          encoder.text_token match, kind
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          # FIXME: cache attack
-          kind = self[1].to_sym
-          opened_tokens << kind
-          case self[2]
-          when '<'
-            encoder.begin_group kind
-          when '['
-            encoder.begin_line kind
-          else
-            raise 'CodeRay bug: This case should not be reached.'
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
+            opened_tokens << kind
+            case self[2]
+            when '<'
+              encoder.begin_group kind
+            when '['
+              encoder.begin_line kind
+            else
+              raise 'CodeRay bug: This case should not be reached.'
+            end
           end
           
         elsif !opened_tokens.empty? && match = scan(/ > /x)
-- 
cgit v1.2.1


From 5d6bee7f5caced1383e6aac427fb356a4788794b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 20:14:21 +0200
Subject: tweak Debug scanners again, introduce :unknown token kind

---
 lib/coderay/scanners/debug.rb    | 25 ++++++++++++++-----------
 lib/coderay/scanners/raydebug.rb | 17 ++++++-----------
 2 files changed, 20 insertions(+), 22 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index ac12c16..83ede9a 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -5,7 +5,7 @@ module Scanners
   
   # = Debug Scanner
   # 
-  # Interprets the output of the Encoders::Debug encoder.
+  # Interprets the output of the Encoders::Debug encoder (basically the inverse function).
   class Debug < Scanner
     
     register_for :debug
@@ -31,21 +31,24 @@ module Scanners
           if @known_token_kinds.include? self[1]
             encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
           else
-            encoder.text_token matched, :error
+            encoder.text_token matched, :unknown
           end
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
           if @known_token_kinds.include? self[1]
             kind = self[1].to_sym
-            opened_tokens << kind
-            case self[2]
-            when '<'
-              encoder.begin_group kind
-            when '['
-              encoder.begin_line kind
-            else
-              raise 'CodeRay bug: This case should not be reached.'
-            end
+          else
+            kind = :unknown
+          end
+          
+          opened_tokens << kind
+          case self[2]
+          when '<'
+            encoder.begin_group kind
+          when '['
+            encoder.begin_line kind
+          else
+            raise 'CodeRay bug: This case should not be reached.'
           end
           
         elsif !opened_tokens.empty? && match = scan(/ > /x)
diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index 6c1c10f..1effdc8 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -3,9 +3,9 @@ require 'set'
 module CodeRay
 module Scanners
   
-  # = Debug Scanner
+  # = Raydebug Scanner
   # 
-  # Parses the output of the Encoders::Debug encoder.
+  # Highlights the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
     
     register_for :raydebug
@@ -43,17 +43,12 @@ module Scanners
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          kind = self[1]
-          case self[2]
-          when '<'
-            encoder.text_token kind, :class
-          when '['
-            encoder.text_token kind, :class
+          encoder.text_token self[1], :class
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
           else
-            raise 'CodeRay bug: This case should not be reached.'
+            kind = :unknown
           end
-          # FIXME: cache attack
-          kind = kind.to_sym
           opened_tokens << kind
           encoder.begin_group kind
           encoder.text_token self[2], :operator
-- 
cgit v1.2.1


From 368e053880819edc74fdcef38f38b5fd4806a3f4 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 20:18:49 +0200
Subject: FileType should guard against attacks here

---
 lib/coderay/scanners/diff.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index 836fa41..fd1aed6 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -21,7 +21,6 @@ module Scanners
       line_kind = nil
       state = :initial
       deleted_lines_count = 0
-      # FIXME: cache attack
       scanners = Hash.new do |h, lang|
         h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true
       end
-- 
cgit v1.2.1


From af04107b8b370452a17fa54e8ea0e8adc8b376b0 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 20:19:57 +0200
Subject: no attack vector, there are only 4 cases

---
 lib/coderay/scanners/python.rb | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index 23630f9..05e1f5f 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -75,12 +75,10 @@ module Scanners
       <<=? | >>=? | [<>=]=? | !=  # comparison and assignment
     /x  # :nodoc:
     
-    # FIXME: cache attack
     STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter|
       h[delimiter] = Regexp.union delimiter  # :nodoc:
     }
     
-    # FIXME: cache attack
     STRING_CONTENT_REGEXP = Hash.new { |h, delimiter|
       h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x  # :nodoc:
     }
-- 
cgit v1.2.1


From e9140073f4dcba5c022a2ad40a1b935a07a6b4c3 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 20:21:45 +0200
Subject: no attack vector, there are only 2 cases

---
 lib/coderay/scanners/python.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index 05e1f5f..09c8b6e 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -183,7 +183,6 @@ module Scanners
               kind = :ident
             elsif kind == :keyword
               state = DEF_NEW_STATE[match]
-              # FIXME: cache attack
               from_import_state << match.to_sym if state == :include_expected
             end
             encoder.text_token match, kind
-- 
cgit v1.2.1


From c3c70e0b3497939dbfb1958a0764f4fd18c05a48 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 20:31:35 +0200
Subject: cleanup

---
 lib/coderay/scanners/ruby/string_state.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb
index fe37d07..bcc0507 100644
--- a/lib/coderay/scanners/ruby/string_state.rb
+++ b/lib/coderay/scanners/ruby/string_state.rb
@@ -17,7 +17,6 @@ module Scanners
       # FIXME: cache attack
       STRING_PATTERN = Hash.new do |h, k|
         delim, interpreted = *k
-        # delim = delim.dup  # workaround for old Ruby
         delim_pattern = Regexp.escape(delim)
         if closing_paren = CLOSING_PAREN[delim]
           delim_pattern << Regexp.escape(closing_paren)
-- 
cgit v1.2.1


From 65983f38eaed758a9901adf9e4e8c4be3e3a6123 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 21 Jul 2013 20:41:55 +0200
Subject: avoid cache attack in Ruby scanner (eg. using Unicode-delimited Fancy
 Strings)

---
 lib/coderay/scanners/ruby/string_state.rb | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb
index bcc0507..28ddd6c 100644
--- a/lib/coderay/scanners/ruby/string_state.rb
+++ b/lib/coderay/scanners/ruby/string_state.rb
@@ -14,7 +14,6 @@ module Scanners
         { }
       ] ].each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
       
-      # FIXME: cache attack
       STRING_PATTERN = Hash.new do |h, k|
         delim, interpreted = *k
         delim_pattern = Regexp.escape(delim)
@@ -29,12 +28,13 @@ module Scanners
         #     '| [|?*+(){}\[\].^$]'
         #   end
         
-        h[k] =
-          if interpreted && delim != '#'
-            / (?= [#{delim_pattern}] | \# [{$@] ) /mx
-          else
-            / (?= [#{delim_pattern}] ) /mx
-          end
+        if interpreted && delim != '#'
+          / (?= [#{delim_pattern}] | \# [{$@] ) /mx
+        else
+          / (?= [#{delim_pattern}] ) /mx
+        end.tap do |pattern|
+          h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256
+        end
       end
       
       def initialize kind, interpreted, delim, heredoc = false
-- 
cgit v1.2.1


From a48037b85a12228431b32103786456f36beb355f Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 1 Sep 2013 01:01:35 +0200
Subject: final cleanup

---
 lib/coderay/scanners/go.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/coderay/scanners')

diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
index 59473f6..99fdd63 100644
--- a/lib/coderay/scanners/go.rb
+++ b/lib/coderay/scanners/go.rb
@@ -1,7 +1,6 @@
 module CodeRay
 module Scanners
   
-  # Scanner for Go, copy from c
   class Go < Scanner
     
     register_for :go
-- 
cgit v1.2.1