From 501df7613c67bcd743eaa00071f0dbb179607394 Mon Sep 17 00:00:00 2001
From: Quintus <sutniuq@gmx.net>
Date: Sun, 22 Apr 2012 22:10:42 +0200
Subject: Lua scanner for CodeRay. Meta-commit. This commit is a super-commit
 containing all the subcommits for implementing the Lua scanner.

---
 lib/coderay/scanners/lua.rb | 267 ++++++++++++++++++++++++++++++++++++++++++++
 lib/coderay/styles/alpha.rb |   3 +
 lib/coderay/token_kinds.rb  |   1 +
 3 files changed, 271 insertions(+)
 create mode 100644 lib/coderay/scanners/lua.rb

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
new file mode 100644
index 0000000..e7706fc
--- /dev/null
+++ b/lib/coderay/scanners/lua.rb
@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+
+# Scanner for the Lua[http://lua.org] programming lanuage.
+#
+# The language’s complete syntax is defined in
+# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
+# which is what this scanner tries to conform to.
+class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+
+  register_for :lua
+  file_extension "lua"
+  title "Lua"
+
+  # Keywords used in Lua.
+  KEYWORDS = %w[and break do else elseif end
+  for function goto if in
+  local not or repeat return
+  then until while
+  ]
+
+  # Constants set by the Lua core.
+  PREDEFINED_CONSTANTS = %w[false true nil]
+
+  # The expressions contained in this array are parts of Lua’s `basic'
+  # library. Although it’s not entirely necessary to load that library,
+  # it is highly recommended and one would have to provide own implementations
+  # of some of these expressions if one does not do so. They however aren’t
+  # keywords, neither are they constants, but nearly predefined, so they
+  # get tagged as `predefined' rather than anything else.
+  #
+  # This list excludes values of form `_UPPERCASE' because the Lua manual
+  # requires such identifiers to be reserved by Lua anyway and they are
+  # highlighted directly accordingly, without the need for specific
+  # identifiers to be listed here.
+  PREDEFINED_EXPRESSIONS = %w[
+  assert collectgarbage dofile error getmetatable
+  ipairs load loadfile next pairs pcall print
+  rawequal rawget rawlen rawset select setmetatable
+  tonumber tostring type xpcall
+  ]
+
+  # Automatic token kind selection for normal words.
+  IDENT_KIND = CodeRay::WordList.new(:ident)
+    .add(KEYWORDS, :keyword)
+    .add(PREDEFINED_CONSTANTS, :predefined_constant)
+    .add(PREDEFINED_EXPRESSIONS, :predefined)
+
+  protected
+
+  # Scanner initialization.
+  def setup
+    @state       = :initial
+    @brace_depth = 0
+  end
+
+  # CodeRay entry hook. Starts parsing.
+  def scan_tokens(encoder, options)
+    @encoder = encoder
+    @options = options
+
+    send(:"handle_state_#@state") until eos?
+
+    @encoder
+  end
+
+  def handle_state_initial
+    if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
+      @num_equals = match.count("=") # Number must match for comment end
+      @encoder.begin_group(:comment)
+      @encoder.text_token(match, :delimiter)
+      @state = :long_comment
+
+    elsif match = scan(/--.*?$/) # --Lua comment
+      @encoder.text_token(match, :comment)
+
+    elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
+      @num_equals = match.count("=") # Number must match for comment end
+      @encoder.begin_group(:string)
+      @encoder.text_token(match, :delimiter)
+      @state = :long_string
+
+    elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label::
+      @encoder.text_token(match, :label)
+
+    elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua
+      @encoder.text_token(match, :predefined)
+
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits)
+      kind = IDENT_KIND[match]
+
+      # Extra highlighting for entities following certain keywords
+      if kind == :keyword and match == "function"
+        @state = :function_expected
+      elsif kind == :keyword and match == "goto"
+        @state = :goto_label_expected
+      elsif kind == :keyword and match == "local"
+        @state = :local_var_expected
+      end
+
+      @encoder.text_token(match, kind)
+
+    elsif match = scan(/{/) # Opening table brace {
+      @encoder.begin_group(:table)
+      @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
+      @brace_depth += 1
+      @state        = :table
+
+    elsif match = scan(/}/) # Closing table brace }
+      if @brace_depth == 1
+        @brace_depth = 0
+        @encoder.text_token(match, :delimiter)
+      elsif @brace_depth == 0 # Mismatched brace
+        @encoder.text_token(match, :error)
+      else
+        @brace_depth -= 1
+        @encoder.text_token(match, :inline_delimiter)
+        @state = :table
+      end
+      @encoder.end_group(:table)
+
+    elsif match = scan(/["']/) # String delimiters " and '
+      @encoder.begin_group(:string)
+      @encoder.text_token(match, :delimiter)
+      @start_delim = match
+      @state       = :string
+
+                      # ↓Prefix                hex number ←|→ decimal number
+    elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+      @encoder.text_token(match, :float)
+
+                      # ↓Prefix         hex number ←|→ decimal number
+    elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+      @encoder.text_token(match, :integer)
+
+    elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators
+      @encoder.text_token(match, :operator)
+
+    elsif match = scan(/\s+/) # Space
+      @encoder.text_token(match, :space)
+
+    else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors.
+      @encoder.text_token(getch, :error)
+    end
+
+    # It may be that we’re scanning a full-blown subexpression of a table
+    # (tables can contain full expressions in parts).
+    # If this is the case, return to :table scanning state.
+    @state = :table if @state == :initial && @brace_depth >= 1
+  end
+
+  def handle_state_function_expected
+    if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
+      @encoder.text_token(match, :operator)
+      @state = :initial
+    elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator
+      @encoder.text_token(match, :ident)
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo()
+      @encoder.text_token(match, :function)
+      @state = :initial
+    elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace
+      @encoder.text_token(match, :space)
+    else
+      @encoder.text_token(getch, :error)
+      @state = :initial
+    end
+  end
+
+  def handle_state_goto_label_expected
+    if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+      @encoder.text_token(match, :label)
+      @state = :initial
+    elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace
+      @encoder.text_token(match, :space)
+    else
+      @encoder.text_token(getch, :error)
+    end
+  end
+
+  def handle_state_local_var_expected
+    if match = scan(/function/) # local function ...
+      @encoder.text_token(match, :keyword)
+      @state = :function_expected
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+      @encoder.text_token(match, :local_variable)
+    elsif match = scan(/,/)
+      @encoder.text_token(match, :operator)
+    elsif match = scan(/=/)
+      @encoder.text_token(match, :operator)
+      # After encountering the equal sign, arbitrary expressions are
+      # allowed again, so just return to the main state for further
+      # parsing.
+      @state = :initial
+    elsif match = scan(/\n/)
+      @encoder.text_token(match, :space)
+      @state = :initial
+    elsif match = scan(/\s+/)
+      @encoder.text_token(match, :space)
+    else
+      @encoder.text_token(getch, :error)
+    end
+  end
+
+  def handle_state_long_comment
+    if match = scan(/.*?(?=\]={#@num_equals}\])/m)
+      @encoder.text_token(match, :content)
+
+      delim = scan(/\]={#@num_equals}\]/)
+      @encoder.text_token(delim, :delimiter)
+    else # No terminator found till EOF
+      @encoder.text_token(rest, :error)
+      terminate
+    end
+    @encoder.end_group(:comment)
+    @state = :initial
+  end
+
+  def handle_state_long_string
+    if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
+      @encoder.text_token(match, :content)
+
+      delim = scan(/\]={#@num_equals}\]/)
+      @encoder.text_token(delim, :delimiter)
+    else # No terminator found till EOF
+      @encoder.text_token(rest, :error)
+      terminate
+    end
+    @encoder.end_group(:string)
+    @state = :initial
+  end
+
+  def handle_state_string
+    if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+      @encoder.text_token(match, :content)
+    elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
+      @encoder.text_token(match, :char)
+    elsif match = scan(Regexp.compile(@start_delim))
+      @encoder.text_token(match, :delimiter)
+      @encoder.end_group(:string)
+      @state = :initial
+    elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings
+      @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings
+      @encoder.end_group(:string)
+      @state = :initial
+    else
+      @encoder.text_token(getch, :error)
+    end
+  end
+
+  def handle_state_table
+    if match = scan(/[,;]/)
+      @encoder.text_token(match, :operator)
+    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
+      @encoder.text_token(match, :key)
+      @encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
+      @encoder.text_token(scan(/=/), :operator)
+      @state = :initial
+    elsif match = scan(/\s+/m)
+      @encoder.text_token(match, :space)
+    else
+      # Note this clause doesn’t advance the scan pointer, it’s a kind of
+      # "retry with other options" (the :initial state then of course
+      # advances the pointer).
+      @state = :initial
+    end
+  end
+
+end
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 8506d10..257083e 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -116,6 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .symbol .content { color:#A60 }
 .symbol .delimiter { color:#630 }
 .symbol { color:#A60 }
+.table .content { color:#808 }
+.table .delimiter { color:#40A}
+.table { background-color:hsla(200,100%,50%,0.06); }
 .tag { color:#070 }
 .type { color:#339; font-weight:bold }
 .value { color: #088; }
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 3b8d07e..e245623 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -63,6 +63,7 @@ module CodeRay
     :shell               => 'shell',
     :string              => 'string',
     :symbol              => 'symbol',
+    :table               => 'table',
     :tag                 => 'tag',
     :type                => 'type',
     :value               => 'value',
-- 
cgit v1.2.1


From 8979cc621431248fded86c341e2102a67c5344bb Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 19 Jun 2012 17:45:49 +0200
Subject: use case+when instead of send and methods

---
 lib/coderay/scanners/lua.rb | 64 ++++++++++++++++++++++-----------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index e7706fc..e712061 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
   ]
 
   # Automatic token kind selection for normal words.
-  IDENT_KIND = CodeRay::WordList.new(:ident)
-    .add(KEYWORDS, :keyword)
-    .add(PREDEFINED_CONSTANTS, :predefined_constant)
-    .add(PREDEFINED_EXPRESSIONS, :predefined)
+  IDENT_KIND = CodeRay::WordList.new(:ident).
+    add(KEYWORDS, :keyword).
+    add(PREDEFINED_CONSTANTS, :predefined_constant).
+    add(PREDEFINED_EXPRESSIONS, :predefined)
 
   protected
 
@@ -57,13 +57,11 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
   def scan_tokens(encoder, options)
     @encoder = encoder
     @options = options
-
-    send(:"handle_state_#@state") until eos?
-
-    @encoder
-  end
-
-  def handle_state_initial
+    
+    until eos?
+  case state
+        
+  when :initial
     if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
       @num_equals = match.count("=") # Number must match for comment end
       @encoder.begin_group(:comment)
@@ -99,13 +97,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
 
       @encoder.text_token(match, kind)
 
-    elsif match = scan(/{/) # Opening table brace {
+    elsif match = scan(/\{/) # Opening table brace {
       @encoder.begin_group(:table)
       @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
       @brace_depth += 1
       @state        = :table
 
-    elsif match = scan(/}/) # Closing table brace }
+    elsif match = scan(/\}/) # Closing table brace }
       if @brace_depth == 1
         @brace_depth = 0
         @encoder.text_token(match, :delimiter)
@@ -146,9 +144,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     # (tables can contain full expressions in parts).
     # If this is the case, return to :table scanning state.
     @state = :table if @state == :initial && @brace_depth >= 1
-  end
-
-  def handle_state_function_expected
+    
+  when :function_expected
     if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
       @encoder.text_token(match, :operator)
       @state = :initial
@@ -163,9 +160,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(getch, :error)
       @state = :initial
     end
-  end
 
-  def handle_state_goto_label_expected
+  when :goto_label_expected
     if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
       @encoder.text_token(match, :label)
       @state = :initial
@@ -174,9 +170,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  end
-
-  def handle_state_local_var_expected
+  
+  when :local_var_expected
     if match = scan(/function/) # local function ...
       @encoder.text_token(match, :keyword)
       @state = :function_expected
@@ -198,9 +193,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  end
-
-  def handle_state_long_comment
+    
+  when :long_comment
     if match = scan(/.*?(?=\]={#@num_equals}\])/m)
       @encoder.text_token(match, :content)
 
@@ -212,9 +206,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:comment)
     @state = :initial
-  end
-
-  def handle_state_long_string
+    
+  when :long_string
     if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
       @encoder.text_token(match, :content)
 
@@ -226,9 +219,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:string)
     @state = :initial
-  end
-
-  def handle_state_string
+    
+  when :string
     if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
       @encoder.text_token(match, :content)
     elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
@@ -244,9 +236,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  end
-
-  def handle_state_table
+  
+  when :table
     if match = scan(/[,;]/)
       @encoder.text_token(match, :operator)
     elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
@@ -262,6 +253,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       # advances the pointer).
       @state = :initial
     end
+  else
+    raise
+  end
+
+    end
+
+    @encoder
   end
 
 end
-- 
cgit v1.2.1


From 2b16d115f7d6caf21864934df763556e126d6357 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 19 Jun 2012 18:05:04 +0200
Subject: fix for Ruby 1.8, escape { and } in regexps

---
 lib/coderay/scanners/lua.rb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index e7706fc..8464ef7 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
   ]
 
   # Automatic token kind selection for normal words.
-  IDENT_KIND = CodeRay::WordList.new(:ident)
-    .add(KEYWORDS, :keyword)
-    .add(PREDEFINED_CONSTANTS, :predefined_constant)
-    .add(PREDEFINED_EXPRESSIONS, :predefined)
+  IDENT_KIND = CodeRay::WordList.new(:ident).
+    add(KEYWORDS, :keyword).
+    add(PREDEFINED_CONSTANTS, :predefined_constant).
+    add(PREDEFINED_EXPRESSIONS, :predefined)
 
   protected
 
@@ -99,13 +99,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
 
       @encoder.text_token(match, kind)
 
-    elsif match = scan(/{/) # Opening table brace {
+    elsif match = scan(/\{/) # Opening table brace {
       @encoder.begin_group(:table)
       @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
       @brace_depth += 1
       @state        = :table
 
-    elsif match = scan(/}/) # Closing table brace }
+    elsif match = scan(/\}/) # Closing table brace }
       if @brace_depth == 1
         @brace_depth = 0
         @encoder.text_token(match, :delimiter)
-- 
cgit v1.2.1


From b9cf7f67d869eead647e237d963ba2db2fdfbc4f Mon Sep 17 00:00:00 2001
From: Joel Holdbrooks <cjholdbrooks@gmail.com>
Date: Tue, 3 Jul 2012 18:17:56 -0700
Subject: add "id" token kind

---
 lib/coderay/token_kinds.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 3b8d07e..41a89e4 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -39,6 +39,7 @@ module CodeRay
     :function            => 'function',
     :global_variable     => 'global-variable',
     :hex                 => 'hex',
+    :id                  => 'id',
     :imaginary           => 'imaginary',
     :important           => 'important',
     :include             => 'include',
-- 
cgit v1.2.1


From e796a247e5bf2395ec72b0ae98d68bcb648f5368 Mon Sep 17 00:00:00 2001
From: Joel Holdbrooks <cjholdbrooks@gmail.com>
Date: Tue, 3 Jul 2012 18:19:05 -0700
Subject: use "id" instead of "constant" for css ids

---
 lib/coderay/scanners/css.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 7b731ef..c473574 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -8,7 +8,7 @@ module Scanners
     KINDS_NOT_LOC = [
       :comment,
       :class, :pseudo_class, :type,
-      :constant, :directive,
+      :id, :directive,
       :key, :value, :operator, :color, :float, :string,
       :error, :important,
     ]  # :nodoc:
@@ -73,7 +73,7 @@ module Scanners
               encoder.text_token match, :class
               next
             elsif match = scan(RE::Id)
-              encoder.text_token match, :constant
+              encoder.text_token match, :id
               next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
-- 
cgit v1.2.1


From 4eb4324c6fb20b9b5e7f73f17bc21e4a17fd3073 Mon Sep 17 00:00:00 2001
From: Joel Holdbrooks <cjholdbrooks@gmail.com>
Date: Tue, 24 Jul 2012 10:59:50 -0700
Subject: use :tag instead of :type for identifiers

---
 lib/coderay/scanners/css.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index c473574..503e14e 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -7,7 +7,7 @@ module Scanners
     
     KINDS_NOT_LOC = [
       :comment,
-      :class, :pseudo_class, :type,
+      :class, :pseudo_class, :tag,
       :id, :directive,
       :key, :value, :operator, :color, :float, :string,
       :error, :important,
@@ -67,7 +67,7 @@ module Scanners
         elsif case states.last
           when :initial, :media
             if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
@@ -102,7 +102,7 @@ module Scanners
             
           when :media_before_name
             if match = scan(RE::Ident)
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               states[-1] = :media_after_name
               next
             end
-- 
cgit v1.2.1


From b081c1f3ab670301199005c04ecbc11614593285 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Tue, 11 Sep 2012 15:04:45 -0600
Subject: escape = in regex (my editor gets confused:-)

---
 lib/coderay/scanners/lua.rb | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index e712061..e640397 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -57,10 +57,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
   def scan_tokens(encoder, options)
     @encoder = encoder
     @options = options
-    
+
     until eos?
   case state
-        
+
   when :initial
     if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
       @num_equals = match.count("=") # Number must match for comment end
@@ -144,7 +144,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     # (tables can contain full expressions in parts).
     # If this is the case, return to :table scanning state.
     @state = :table if @state == :initial && @brace_depth >= 1
-    
+
   when :function_expected
     if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
       @encoder.text_token(match, :operator)
@@ -170,7 +170,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  
+
   when :local_var_expected
     if match = scan(/function/) # local function ...
       @encoder.text_token(match, :keyword)
@@ -179,7 +179,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(match, :local_variable)
     elsif match = scan(/,/)
       @encoder.text_token(match, :operator)
-    elsif match = scan(/=/)
+    elsif match = scan(/\=/)
       @encoder.text_token(match, :operator)
       # After encountering the equal sign, arbitrary expressions are
       # allowed again, so just return to the main state for further
@@ -193,7 +193,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-    
+
   when :long_comment
     if match = scan(/.*?(?=\]={#@num_equals}\])/m)
       @encoder.text_token(match, :content)
@@ -206,7 +206,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:comment)
     @state = :initial
-    
+
   when :long_string
     if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
       @encoder.text_token(match, :content)
@@ -219,7 +219,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     end
     @encoder.end_group(:string)
     @state = :initial
-    
+
   when :string
     if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
       @encoder.text_token(match, :content)
@@ -236,14 +236,14 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
     else
       @encoder.text_token(getch, :error)
     end
-  
+
   when :table
     if match = scan(/[,;]/)
       @encoder.text_token(match, :operator)
     elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
       @encoder.text_token(match, :key)
       @encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
-      @encoder.text_token(scan(/=/), :operator)
+      @encoder.text_token(scan(/\=/), :operator)
       @state = :initial
     elsif match = scan(/\s+/m)
       @encoder.text_token(match, :space)
-- 
cgit v1.2.1


From aeac6672fbe70702288b259605cb863493ad8ac2 Mon Sep 17 00:00:00 2001
From: SHIMOMURA Sho <graueule@gmail.com>
Date: Sun, 21 Oct 2012 01:23:56 +0900
Subject: TaskPaper scanner for CodeRay

---
 lib/coderay/scanners/taskpaper.rb | 34 ++++++++++++++++++++++++++++++++++
 lib/coderay/styles/alpha.rb       |  3 +++
 lib/coderay/token_kinds.rb        |  3 +++
 3 files changed, 40 insertions(+)
 create mode 100644 lib/coderay/scanners/taskpaper.rb

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/taskpaper.rb b/lib/coderay/scanners/taskpaper.rb
new file mode 100644
index 0000000..c6eb3b3
--- /dev/null
+++ b/lib/coderay/scanners/taskpaper.rb
@@ -0,0 +1,34 @@
+module CodeRay
+module Scanners
+  
+  class Taskpaper < Scanner
+    
+    register_for :taskpaper
+    file_extension 'taskpaper'
+    
+  protected
+    
+    def scan_tokens encoder, options
+      until eos?
+        if match = scan(/^\S.*:.*$/)                  # project
+          encoder.text_token(match, :project)
+        elsif match = scan(/^-.+@done(?:\(.*)?.*$/)   # completed task
+          encoder.text_token(match, :complete)
+        elsif match = scan(/^-.+$/)                   # task
+          encoder.text_token(match, :plain)
+        elsif match = scan(/^.+$/)                    # comment
+          encoder.text_token(match, :comment)
+        elsif match = scan(/\s+/)                     # space
+          encoder.text_token(match, :space)
+        else                                          # other
+          encoder.text_token getch, :error
+        end
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 8506d10..259e458 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -134,6 +134,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .delete .delete { color: #c00; background:transparent; font-weight:bold }
 .change .change { color: #88f }
 .head .head { color: #f4f }
+
+.project  { color: #707; font-weight: bold }
+.complete { text-decoration: line-through; color: gray }
     TOKENS
 
   end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 3b8d07e..a36d877 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -74,6 +74,9 @@ module CodeRay
     :insert              => 'insert',
     
     :eyecatcher          => 'eyecatcher',
+
+    :project             => 'project',
+    :complete            => 'complete',
     
     :ident               => false,
     :operator            => false,
-- 
cgit v1.2.1


From 279348d3c2159df4ce6ac5949ada0177aa7c7159 Mon Sep 17 00:00:00 2001
From: Nathan Youngman <git@nathany.com>
Date: Sat, 27 Oct 2012 23:41:52 -0600
Subject: :map token kind

Use :map instead of :table. It's more generic, and won't be confused with the :table rendering style.
---
 lib/coderay/scanners/lua.rb | 14 +++++++-------
 lib/coderay/styles/alpha.rb |  8 ++++----
 lib/coderay/token_kinds.rb  | 20 ++++++++++----------
 3 files changed, 21 insertions(+), 21 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index e640397..64763dc 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -98,10 +98,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(match, kind)
 
     elsif match = scan(/\{/) # Opening table brace {
-      @encoder.begin_group(:table)
+      @encoder.begin_group(:map)
       @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
       @brace_depth += 1
-      @state        = :table
+      @state        = :map
 
     elsif match = scan(/\}/) # Closing table brace }
       if @brace_depth == 1
@@ -112,9 +112,9 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       else
         @brace_depth -= 1
         @encoder.text_token(match, :inline_delimiter)
-        @state = :table
+        @state = :map
       end
-      @encoder.end_group(:table)
+      @encoder.end_group(:map)
 
     elsif match = scan(/["']/) # String delimiters " and '
       @encoder.begin_group(:string)
@@ -142,8 +142,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
 
     # It may be that we’re scanning a full-blown subexpression of a table
     # (tables can contain full expressions in parts).
-    # If this is the case, return to :table scanning state.
-    @state = :table if @state == :initial && @brace_depth >= 1
+    # If this is the case, return to :map scanning state.
+    @state = :map if @state == :initial && @brace_depth >= 1
 
   when :function_expected
     if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
@@ -237,7 +237,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
       @encoder.text_token(getch, :error)
     end
 
-  when :table
+  when :map
     if match = scan(/[,;]/)
       @encoder.text_token(match, :operator)
     elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 257083e..a60725b 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -1,6 +1,6 @@
 module CodeRay
 module Styles
-  
+
   # A colorful theme using CSS 3 colors (with alpha channel).
   class Alpha < Style
 
@@ -116,9 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .symbol .content { color:#A60 }
 .symbol .delimiter { color:#630 }
 .symbol { color:#A60 }
-.table .content { color:#808 }
-.table .delimiter { color:#40A}
-.table { background-color:hsla(200,100%,50%,0.06); }
+.map .content { color:#808 }
+.map .delimiter { color:#40A}
+.map { background-color:hsla(200,100%,50%,0.06); }
 .tag { color:#070 }
 .type { color:#339; font-weight:bold }
 .value { color: #088; }
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index e245623..a0ad5e4 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -1,14 +1,14 @@
 module CodeRay
-  
+
   # A Hash of all known token kinds and their associated CSS classes.
   TokenKinds = Hash.new do |h, k|
     warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
     false
   end
-  
+
   # speedup
   TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
-  
+
   TokenKinds.update(  # :nodoc:
     :annotation          => 'annotation',
     :attribute_name      => 'attribute-name',
@@ -50,6 +50,7 @@ module CodeRay
     :keyword             => 'keyword',
     :label               => 'label',
     :local_variable      => 'local-variable',
+    :map                 => 'map',
     :modifier            => 'modifier',
     :namespace           => 'namespace',
     :octal               => 'octal',
@@ -63,29 +64,28 @@ module CodeRay
     :shell               => 'shell',
     :string              => 'string',
     :symbol              => 'symbol',
-    :table               => 'table',
     :tag                 => 'tag',
     :type                => 'type',
     :value               => 'value',
     :variable            => 'variable',
-    
+
     :change              => 'change',
     :delete              => 'delete',
     :head                => 'head',
     :insert              => 'insert',
-    
+
     :eyecatcher          => 'eyecatcher',
-    
+
     :ident               => false,
     :operator            => false,
-    
+
     :space               => false,
     :plain               => false
   )
-  
+
   TokenKinds[:method]    = TokenKinds[:function]
   TokenKinds[:escape]    = TokenKinds[:delimiter]
   TokenKinds[:docstring] = TokenKinds[:comment]
-  
+
   TokenKinds.freeze
 end
-- 
cgit v1.2.1


From 51526173dd7ee640171ee597d3c380c5dda42762 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 10 Mar 2013 22:40:10 +0100
Subject: support multi-line comments in JavaScript diff

---
 lib/coderay/scanners/java_script.rb | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb
index 43ecb18..75881a0 100644
--- a/lib/coderay/scanners/java_script.rb
+++ b/lib/coderay/scanners/java_script.rb
@@ -54,10 +54,17 @@ module Scanners
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     def scan_tokens encoder, options
       
-      state = :initial
-      string_delimiter = nil
+      state, string_delimiter = options[:state] || @state
+      if string_delimiter
+        encoder.begin_group state
+      end
+      
       value_expected = true
       key_expected = false
       function_expected = false
@@ -72,9 +79,10 @@ module Scanners
             value_expected = true if !value_expected && match.index(?\n)
             encoder.text_token match, :space
             
-          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .*() ) !mx)
             value_expected = true
             encoder.text_token match, :comment
+            state = :open_multi_line_comment if self[1]
             
           elsif check(/\.?\d/)
             key_expected = value_expected = false
@@ -182,13 +190,28 @@ module Scanners
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
           
+        when :open_multi_line_comment
+          if match = scan(%r! .*? \*/ !mx)
+            state = :initial
+          else
+            match = scan(%r! .+ !mx)
+          end
+          value_expected = true
+          encoder.text_token match, :comment if match
+          
         else
-          raise_inspect 'Unknown state', encoder
+          #:nocov:
+          raise_inspect 'Unknown state: %p' % [state], encoder
+          #:nocov:
           
         end
         
       end
       
+      if options[:keep_state]
+        @state = state, string_delimiter
+      end
+      
       if [:string, :regexp].include? state
         encoder.end_group state
       end
-- 
cgit v1.2.1


From 469dba8c6378fb62ebf12c4a2f32b6be433bb16e Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 11 Mar 2013 02:23:22 +0100
Subject: added Sass scanner (work in progress)

---
 lib/coderay/helpers/file_type.rb |   1 +
 lib/coderay/scanners/css.rb      |  31 +++---
 lib/coderay/scanners/sass.rb     | 203 +++++++++++++++++++++++++++++++++++++++
 lib/coderay/styles/alpha.rb      |   2 +
 4 files changed, 220 insertions(+), 17 deletions(-)
 create mode 100644 lib/coderay/scanners/sass.rb

(limited to 'lib/coderay')

diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 637001b..b06b9bc 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -116,6 +116,7 @@ module CodeRay
       'rpdf'     => :ruby,
       'ru'       => :ruby,
       'rxml'     => :ruby,
+      'sass'     => :sass,
       # 'sch'      => :scheme,
       'sql'      => :sql,
       # 'ss'       => :scheme,
diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 7b731ef..003eed6 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -15,19 +15,17 @@ module Scanners
     
     module RE  # :nodoc:
       Hex = /[0-9a-fA-F]/
-      Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
-      Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
-      NMChar = /[-_a-zA-Z0-9]|#{Escape}/
-      NMStart = /[_a-zA-Z]|#{Escape}/
-      NL = /\r\n|\r|\n|\f/
-      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # TODO: buggy regexp
-      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # TODO: buggy regexp
+      Unicode = /\\#{Hex}{1,6}\b/ # differs from standard because it allows uppercase hex too
+      Escape = /#{Unicode}|\\[^\n0-9a-fA-F]/
+      NMChar = /[-_a-zA-Z0-9]/
+      NMStart = /[_a-zA-Z]/
+      String1 = /"(?:[^\n\\"]+|\\\n|#{Escape})*"?/  # TODO: buggy regexp
+      String2 = /'(?:[^\n\\']+|\\\n|#{Escape})*'?/  # TODO: buggy regexp
       String = /#{String1}|#{String2}/
       
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
-      Color = /#{HexColor}/
       
-      Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
+      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
@@ -35,16 +33,15 @@ module Scanners
       
       reldimensions = %w[em ex px]
       absdimensions = %w[in cm mm pt pc]
-      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s]))
+      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s dpi dppx deg]))
       
       Dimension = /#{Num}#{Unit}/
       
-      Comment = %r! /\* (?: .*? \*/ | .* ) !mx
-      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/
+      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n]|\\\))*\)?/
       
-      Id = /##{Name}/
+      Id = /(?!#{HexColor}\b(?!-))##{Name}/
       Class = /\.#{Name}/
-      PseudoClass = /:#{Name}/
+      PseudoClass = /::?#{Ident}/
       AttributeSelector = /\[[^\]]*\]?/
     end
     
@@ -52,7 +49,7 @@ module Scanners
     
     def setup
       @state = :initial
-      @value_expected = nil
+      @value_expected = false
     end
     
     def scan_tokens encoder, options
@@ -158,7 +155,7 @@ module Scanners
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
           encoder.text_token match, :float
           
-        elsif match = scan(/#{RE::Color}/o)
+        elsif match = scan(/#{RE::HexColor}/o)
           encoder.text_token match, :color
           
         elsif match = scan(/! *important/)
@@ -170,7 +167,7 @@ module Scanners
         elsif match = scan(RE::AtKeyword)
           encoder.text_token match, :directive
           
-        elsif match = scan(/ [+>:;,.=()\/] /x)
+        elsif match = scan(/ [+>~:;,.=()\/] /x)
           if match == ':'
             value_expected = true
           elsif match == ';'
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
new file mode 100644
index 0000000..89d7737
--- /dev/null
+++ b/lib/coderay/scanners/sass.rb
@@ -0,0 +1,203 @@
+module CodeRay
+module Scanners
+  
+  # A scanner for Sass.
+  class Sass < CSS
+    
+    register_for :sass
+    file_extension 'sass'
+    
+    SASS_FUNCTION = /(?:inline-image|linear-gradient|color-stops|mix|lighten|darken|rotate|image-url|image-width|image-height|sprite|sprite-url|sprite-path|sprite-file|sprite-map|sprite-position|unquote|join|round|ceil|floor|nth)/
+    
+    STRING_CONTENT_PATTERN = {
+      "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
+      '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
+    }
+    
+  protected
+    
+    def setup
+      @state = :initial
+      @value_expected = false
+    end
+    
+    def scan_tokens encoder, options
+      states = Array(options[:state] || @state)
+      string_delimiter = nil
+      value_expected = @value_expected
+      
+      until eos?
+        
+        if match = scan(/\s+/)
+          encoder.text_token match, :space
+          value_expected = false if match.index(/\n/)
+          
+        elsif states.last == :sass_inline && (match = scan(/\}/))
+          encoder.text_token match, :inline_delimiter
+          encoder.end_group :inline
+          states.pop
+        
+        elsif case states.last
+          when :initial, :media, :sass_inline
+            if match = scan(/(?>#{RE::Ident})(?!\()/ox)
+              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :type)
+              next
+            elsif !value_expected && (match = scan(/\*/))
+              encoder.text_token match, :type
+              next
+            elsif match = scan(RE::Class)
+              encoder.text_token match, :class
+              next
+            elsif match = scan(RE::Id)
+              encoder.text_token match, :constant
+              next
+            elsif match = scan(RE::PseudoClass)
+              encoder.text_token match, :pseudo_class
+              next
+            elsif match = scan(RE::AttributeSelector)
+              # TODO: Improve highlighting inside of attribute selectors.
+              encoder.text_token match[0,1], :operator
+              encoder.text_token match[1..-2], :attribute_name if match.size > 2
+              encoder.text_token match[-1,1], :operator if match[-1] == ?]
+              next
+            elsif match = scan(/(\=|@mixin +)#{RE::Ident}/o)
+              encoder.text_token match, :function
+              next
+            elsif match = scan(/@media/)
+              encoder.text_token match, :directive
+              # states.push :media_before_name
+              next
+            end
+          
+          when :block
+            if match = scan(/(?>#{RE::Ident})(?!\()/ox)
+              if value_expected
+                encoder.text_token match, :value
+              else
+                encoder.text_token match, :key
+              end
+              next
+            end
+            
+          when :string
+            if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
+              encoder.text_token match, :content
+            elsif match = scan(/['"]/)
+              encoder.text_token match, :delimiter
+              encoder.end_group :string
+              string_delimiter = nil
+              states.pop
+            elsif match = scan(/#\{/)
+              encoder.begin_group :inline
+              encoder.text_token match, :inline_delimiter
+              states.push :sass_inline
+            elsif match = scan(/ \\ | $ /x)
+              encoder.end_group state
+              encoder.text_token match, :error unless match.empty?
+              states.pop
+            else
+              raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
+            end
+          
+          else
+            #:nocov:
+            raise_inspect 'Unknown state', encoder
+            #:nocov:
+            
+          end
+          
+        elsif match = scan(/\$#{RE::Ident}/o)
+          encoder.text_token match, :variable
+          next
+        
+        elsif match = scan(/&/)
+          encoder.text_token match, :local_variable
+          
+        elsif match = scan(/\+#{RE::Ident}/o)
+          encoder.text_token match, :include
+          value_expected = true
+          
+        elsif match = scan(/\/\*(?:.*?\*\/|.*)|\/\/.*/)
+          encoder.text_token match, :comment
+          
+        elsif match = scan(/#\{/)
+          encoder.begin_group :inline
+          encoder.text_token match, :inline_delimiter
+          states.push :sass_inline
+          
+        elsif match = scan(/\{/)
+          value_expected = false
+          encoder.text_token match, :operator
+          states.push :block
+          
+        elsif match = scan(/\}/)
+          value_expected = false
+          encoder.text_token match, :operator
+          if states.last == :block || states.last == :media
+            states.pop
+          end
+          
+        elsif match = scan(/['"]/)
+          encoder.begin_group :string
+          string_delimiter = match
+          encoder.text_token match, :delimiter
+          states.push :string
+          
+        elsif match = scan(/#{SASS_FUNCTION}/o)
+          encoder.text_token match, :predefined
+          
+        elsif match = scan(/#{RE::Function}/o)
+          encoder.begin_group :function
+          start = match[/^[-\w]+\(/]
+          encoder.text_token start, :delimiter
+          if match[-1] == ?)
+            encoder.text_token match[start.size..-2], :content
+            encoder.text_token ')', :delimiter
+          else
+            encoder.text_token match[start.size..-1], :content
+          end
+          encoder.end_group :function
+          
+        elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
+          encoder.text_token match, :float
+          
+        elsif match = scan(/#{RE::HexColor}/o)
+          encoder.text_token match, :color
+          
+        elsif match = scan(/! *(?:important|optional)/)
+          encoder.text_token match, :important
+          
+        elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
+          encoder.text_token match, :color
+          
+        elsif match = scan(/@else if\b|#{RE::AtKeyword}/)
+          encoder.text_token match, :directive
+          value_expected = true
+          
+        elsif match = scan(/ == | != | [-+*\/>~:;,.=()] /x)
+          if match == ':'
+            value_expected = true
+          elsif match == ';'
+            value_expected = false
+          end
+          encoder.text_token match, :operator
+          
+        else
+          encoder.text_token getch, :error
+          
+        end
+        
+      end
+      
+      if options[:keep_state]
+        @state = states
+        @value_expected = value_expected
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 8506d10..1f073b6 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -78,6 +78,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .exception { color:#C00; font-weight:bold }
 .float { color:#60E }
 .function { color:#06B; font-weight:bold }
+.function .delimiter { color:#024; font-weight:bold }
 .global-variable { color:#d70 }
 .hex { color:#02b }
 .imaginary { color:#f00 }
@@ -86,6 +87,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .inline-delimiter { font-weight: bold; color: #666 }
 .instance-variable { color:#33B }
 .integer  { color:#00D }
+.important { color:#D00 }
 .key .char { color: #60f }
 .key .delimiter { color: #404 }
 .key { color: #606 }
-- 
cgit v1.2.1


From 2708fd32f45a6de34249ffbd5331c6df9c985889 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Thu, 11 Apr 2013 21:02:31 +0200
Subject: fix order of SASS_FUNCTIONs

---
 lib/coderay/scanners/sass.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 89d7737..b1b1cba 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -7,7 +7,7 @@ module Scanners
     register_for :sass
     file_extension 'sass'
     
-    SASS_FUNCTION = /(?:inline-image|linear-gradient|color-stops|mix|lighten|darken|rotate|image-url|image-width|image-height|sprite|sprite-url|sprite-path|sprite-file|sprite-map|sprite-position|unquote|join|round|ceil|floor|nth)/
+    SASS_FUNCTION = /(?:inline-image|linear-gradient|color-stops|mix|lighten|darken|rotate|image-url|image-width|image-height|sprite-url|sprite-path|sprite-file|sprite-map|sprite-position|sprite|unquote|join|round|ceil|floor|nth)/
     
     STRING_CONTENT_PATTERN = {
       "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
-- 
cgit v1.2.1


From 57ffbc753f8e7fd676a13bf5960043b23ee1d5ed Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 04:30:23 +0200
Subject: unfreeze TokenKinds hash, allow changes

---
 lib/coderay/token_kinds.rb | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 3b8d07e..8db8530 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -85,6 +85,4 @@ module CodeRay
   TokenKinds[:method]    = TokenKinds[:function]
   TokenKinds[:escape]    = TokenKinds[:delimiter]
   TokenKinds[:docstring] = TokenKinds[:comment]
-  
-  TokenKinds.freeze
 end
-- 
cgit v1.2.1


From 1952ef4c4dd1105665a3a4f9a0266598d6acf892 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 06:22:39 +0200
Subject: two fixes for CSS scanner

---
 lib/coderay/scanners/css.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 7b731ef..dd5106e 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -27,7 +27,7 @@ module Scanners
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
       Color = /#{HexColor}/
       
-      Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
+      Num = /-?(?:[0-9]+(?!\.\d)|[0-9]*\.[0-9]+)/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
@@ -44,7 +44,7 @@ module Scanners
       
       Id = /##{Name}/
       Class = /\.#{Name}/
-      PseudoClass = /:#{Name}/
+      PseudoClass = /::?#{Name}/
       AttributeSelector = /\[[^\]]*\]?/
     end
     
-- 
cgit v1.2.1


From 2347f995a70d7607fc0e417c5c7323eacfdcc341 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 06:23:48 +0200
Subject: treat empty token as error in Debug encoder

---
 lib/coderay/encoders/debug.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 62f9f0a..c03d3fb 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -24,6 +24,7 @@ module Encoders
     end
     
     def text_token text, kind
+      raise 'empty token' if $CODERAY_DEBUG && text.empty?
       if kind == :space
         @out << text
       else
-- 
cgit v1.2.1


From 667d4262e42b5866859fb9eb91a0617c1722e7a4 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 06:24:29 +0200
Subject: avoid empty tokens in Diff output, fix split_into_parts

---
 lib/coderay/scanners/diff.rb | 2 +-
 lib/coderay/tokens.rb        | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index 38efaf4..8e3bf3c 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -45,7 +45,7 @@ module Scanners
           if match = scan(/--- |\+\+\+ |=+|_+/)
             encoder.begin_line line_kind = :head
             encoder.text_token match, :head
-            if match = scan(/.*?(?=$|[\t\n\x00]|  \(revision)/)
+            if match = scan(/.+?(?=$|[\t\n\x00]|  \(revision)/)
               encoder.text_token match, :filename
               if options[:highlight_code] && match != '/dev/null'
                 file_type = CodeRay::FileType.fetch(match, :text)
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index c747017..6957d69 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -93,6 +93,7 @@ module CodeRay
     # This method is used by @Scanner#tokenize@ when called with an Array
     # of source strings. The Diff encoder uses it for inline highlighting.
     def split_into_parts *sizes
+      return Array.new(sizes.size) { Tokens.new } if size == 2 && first == ''
       parts = []
       opened = []
       content = nil
-- 
cgit v1.2.1


From 4323bb7ad9b5c5ddc770691d66e5727553ca33a3 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 06:27:14 +0200
Subject: bump version to 1.1.0

---
 lib/coderay/version.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb
index bfb5f24..4b4f085 100644
--- a/lib/coderay/version.rb
+++ b/lib/coderay/version.rb
@@ -1,3 +1,3 @@
 module CodeRay
-  VERSION = '1.0.9'
+  VERSION = '1.1.0'
 end
-- 
cgit v1.2.1


From 1ad75645ad880ede3c0590fd6a700b1847cc3fe4 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 07:14:12 +0200
Subject: don't allow \x00 in diff filenames

---
 lib/coderay/scanners/diff.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index 8e3bf3c..af0f755 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -45,7 +45,7 @@ module Scanners
           if match = scan(/--- |\+\+\+ |=+|_+/)
             encoder.begin_line line_kind = :head
             encoder.text_token match, :head
-            if match = scan(/.+?(?=$|[\t\n\x00]|  \(revision)/)
+            if match = scan(/[^\x00\n]+?(?=$|[\t\n]|  \(revision)/)
               encoder.text_token match, :filename
               if options[:highlight_code] && match != '/dev/null'
                 file_type = CodeRay::FileType.fetch(match, :text)
-- 
cgit v1.2.1


From 737fb3608c92506a573a79f611b94ff5d5da18e6 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 07:25:15 +0200
Subject: do not warn about plugin fallback

---
 lib/coderay/helpers/plugin.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
index 137c1ab..dd4e830 100644
--- a/lib/coderay/helpers/plugin.rb
+++ b/lib/coderay/helpers/plugin.rb
@@ -180,7 +180,6 @@ module CodeRay
         rescue LoadError => boom
           if @plugin_map_loaded
             if h.has_key?(:default)
-              warn '%p could not load plugin %p; falling back to %p' % [self, id, h[:default]]
               h[:default]
             else
               raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
-- 
cgit v1.2.1


From 8563acc7e29e3d5bf5281aedbc0b7088abe44877 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 19:30:49 +0200
Subject: fix Sass highlighting in diff

---
 lib/coderay/scanners/sass.rb | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index b1b1cba..17514c0 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -18,13 +18,11 @@ module Scanners
     
     def setup
       @state = :initial
-      @value_expected = false
     end
     
     def scan_tokens encoder, options
       states = Array(options[:state] || @state)
       string_delimiter = nil
-      value_expected = @value_expected
       
       until eos?
         
@@ -191,7 +189,6 @@ module Scanners
       
       if options[:keep_state]
         @state = states
-        @value_expected = value_expected
       end
       
       encoder
-- 
cgit v1.2.1


From 5bea8e3041de85266c3596627224ced102f195e0 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 19:31:10 +0200
Subject: don't nest :string recursively in :sass_inline

---
 lib/coderay/scanners/sass.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 17514c0..0eb2caa 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -139,7 +139,14 @@ module Scanners
           encoder.begin_group :string
           string_delimiter = match
           encoder.text_token match, :delimiter
-          states.push :string
+          if states.include? :sass_inline
+            content = scan_until(/(?=#{string_delimiter}|\}|\z)/)
+            encoder.text_token content, :content unless content.empty?
+            encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/)
+            encoder.end_group :string
+          else
+            states.push :string
+          end
           
         elsif match = scan(/#{SASS_FUNCTION}/o)
           encoder.text_token match, :predefined
-- 
cgit v1.2.1


From 8a95c0a59eab55caaa4aff142f6448ede91db4d7 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 21:11:29 +0200
Subject: whitespace

---
 lib/coderay/encoders/html.rb | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 635a4d8..0fd1317 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -286,8 +286,7 @@ module Encoders
     
     def end_group kind
       if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a token (%p) ' \
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+        warn 'Malformed token stream: Trying to close a token group (%p) that is not open. Open are: %p.' % [kind, @opened[1..-1]]
       end
       if @opened.pop
         @out << '</span>'
@@ -312,8 +311,7 @@ module Encoders
     
     def end_line kind
       if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a line (%p) ' \
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+        warn 'Malformed token stream: Trying to close a line (%p) that is not open. Open are: %p.' % [kind, @opened[1..-1]]
       end
       if @opened.pop
         @out << '</span>'
-- 
cgit v1.2.1


From 60d03a22df232528d7cc1b6a11e0ecc68171873a Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 9 Jun 2013 21:17:31 +0200
Subject: add .xaml file type (use XML scanner)

---
 lib/coderay/helpers/file_type.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 12556d0..a5d83ff 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -118,10 +118,9 @@ module CodeRay
       'ru'       => :ruby,
       'rxml'     => :ruby,
       'sass'     => :sass,
-      # 'sch'      => :scheme,
       'sql'      => :sql,
-      # 'ss'       => :scheme,
       'tmproj'   => :xml,
+      'xaml'     => :xml,
       'xhtml'    => :html,
       'xml'      => :xml,
       'yaml'     => :yaml,
-- 
cgit v1.2.1


From 5a833b265c5cb918f906fb1047c0dc65e539260f Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 06:09:59 +0200
Subject: always load the plugin map

---
 lib/coderay/helpers/plugin.rb | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
index dd4e830..d14c5a9 100644
--- a/lib/coderay/helpers/plugin.rb
+++ b/lib/coderay/helpers/plugin.rb
@@ -131,7 +131,7 @@ module CodeRay
     
     # A Hash of plugion_id => Plugin pairs.
     def plugin_hash
-      @plugin_hash ||= make_plugin_hash
+      @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map }
     end
     
     # Returns an array of all .rb files in the plugin path.
@@ -158,7 +158,6 @@ module CodeRay
     # This is done automatically when plugin_path is called.
     def load_plugin_map
       mapfile = path_to '_map'
-      @plugin_map_loaded = true
       if File.exist? mapfile
         require mapfile
         true
@@ -171,22 +170,16 @@ module CodeRay
     
     # Return a plugin hash that automatically loads plugins.
     def make_plugin_hash
-      @plugin_map_loaded ||= false
       Hash.new do |h, plugin_id|
         id = validate_id(plugin_id)
         path = path_to id
         begin
           require path
         rescue LoadError => boom
-          if @plugin_map_loaded
-            if h.has_key?(:default)
-              h[:default]
-            else
-              raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
-            end
+          if h.has_key?(:default)
+            h[:default]
           else
-            load_plugin_map
-            h[plugin_id]
+            raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
           end
         else
           # Plugin should have registered by now
@@ -270,7 +263,6 @@ module CodeRay
     end
     
     def aliases
-      plugin_host.load_plugin_map
       plugin_host.plugin_hash.inject [] do |aliases, (key, _)|
         aliases << key if plugin_host[key] == self
         aliases
-- 
cgit v1.2.1


From e2acec3ef141725d2fc264e56d1aa18e838c6acf Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 15:23:52 +0200
Subject: massively improve Terminal encoder speed

---
 lib/coderay/encoders/terminal.rb | 141 +++++++++++++++++++--------------------
 1 file changed, 67 insertions(+), 74 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb
index a0ceb3c..500e5d8 100644
--- a/lib/coderay/encoders/terminal.rb
+++ b/lib/coderay/encoders/terminal.rb
@@ -19,73 +19,73 @@ module CodeRay
       register_for :terminal
       
       TOKEN_COLORS = {
-        :annotation => '35',
-        :attribute_name => '33',
-        :attribute_value => '31',
-        :binary => '1;35',
+        :annotation => "\e[35m",
+        :attribute_name => "\e[33m",
+        :attribute_value => "\e[31m",
+        :binary => "\e[1;35m",
         :char => {
-          :self => '36', :delimiter => '1;34'
+          :self => "\e[36m", :delimiter => "\e[1;34m"
         },
-        :class => '1;35',
-        :class_variable => '36',
-        :color => '32',
-        :comment => '37',
-        :complex => '1;34',
-        :constant => ['1;34', '4'],
-        :decoration => '35',
-        :definition => '1;32',
-        :directive => ['32', '4'],
-        :doc => '46',
-        :doctype => '1;30',
-        :doc_string => ['31', '4'],
-        :entity => '33',
-        :error => ['1;33', '41'],
-        :exception => '1;31',
-        :float => '1;35',
-        :function => '1;34',
-        :global_variable => '42',
-        :hex => '1;36',
-        :include => '33',
-        :integer => '1;34',
-        :key => '35',
-        :label => '1;15',
-        :local_variable => '33',
-        :octal => '1;35',
-        :operator_name => '1;29',
-        :predefined_constant => '1;36',
-        :predefined_type => '1;30',
-        :predefined => ['4', '1;34'],
-        :preprocessor => '36',
-        :pseudo_class => '1;34',
+        :class => "\e[1;35m",
+        :class_variable => "\e[36m",
+        :color => "\e[32m",
+        :comment => "\e[37m",
+        :complex => "\e[1;34m",
+        :constant => "\e[1;34m\e[4m",
+        :decoration => "\e[35m",
+        :definition => "\e[1;32m",
+        :directive => "\e[32m\e[4m",
+        :doc => "\e[46m",
+        :doctype => "\e[1;30m",
+        :doc_string => "\e[31m\e[4m",
+        :entity => "\e[33m",
+        :error => "\e[1;33m\e[41m",
+        :exception => "\e[1;31m",
+        :float => "\e[1;35m",
+        :function => "\e[1;34m",
+        :global_variable => "\e[42m",
+        :hex => "\e[1;36m",
+        :include => "\e[33m",
+        :integer => "\e[1;34m",
+        :key => "\e[35m",
+        :label => "\e[1;15m",
+        :local_variable => "\e[33m",
+        :octal => "\e[1;35m",
+        :operator_name => "\e[1;29m",
+        :predefined_constant => "\e[1;36m",
+        :predefined_type => "\e[1;30m",
+        :predefined => "\e[4m\e[1;34m",
+        :preprocessor => "\e[36m",
+        :pseudo_class => "\e[1;34m",
         :regexp => {
-          :self => '31',
-          :content => '31',
-          :delimiter => '1;29',
-          :modifier => '35',
+          :self => "\e[31m",
+          :content => "\e[31m",
+          :delimiter => "\e[1;29m",
+          :modifier => "\e[35m",
         },
-        :reserved => '1;31',
+        :reserved => "\e[1;31m",
         :shell => {
-          :self => '42',
-          :content => '1;29',
-          :delimiter => '37',
+          :self => "\e[42m",
+          :content => "\e[1;29m",
+          :delimiter => "\e[37m",
         },
         :string => {
-          :self => '32',
-          :modifier => '1;32',
-          :escape => '1;36',
-          :delimiter => '1;32',
-          :char => '1;36',
+          :self => "\e[32m",
+          :modifier => "\e[1;32m",
+          :escape => "\e[1;36m",
+          :delimiter => "\e[1;32m",
+          :char => "\e[1;36m",
         },
-        :symbol => '1;32',
-        :tag => '1;34',
-        :type => '1;34',
-        :value => '36',
-        :variable => '1;34',
+        :symbol => "\e[1;32m",
+        :tag => "\e[1;34m",
+        :type => "\e[1;34m",
+        :value => "\e[36m",
+        :variable => "\e[1;34m",
         
-        :insert => '42',
-        :delete => '41',
-        :change => '44',
-        :head => '45'
+        :insert => "\e[42m",
+        :delete => "\e[41m",
+        :change => "\e[44m",
+        :head => "\e[45m"
       }
       TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
@@ -114,10 +114,10 @@ module CodeRay
             end
           end
           
-          @out << ansi_colorize(color)
-          @out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color))
-          @out << ansi_clear
-          @out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self]
+          @out << color
+          @out << text.gsub("\n", "\e[0m\n" + color)
+          @out << "\e[0m"
+          @out << @subcolors[:self] if @subcolors
         else
           @out << text
         end
@@ -134,7 +134,7 @@ module CodeRay
           # nothing to close
         else
           @opened.pop
-          @out << ansi_clear
+          @out << "\e[0m"
           @out << open_token(@opened.last)
         end
       end
@@ -146,7 +146,7 @@ module CodeRay
           @opened.pop
           # whole lines to be highlighted,
           # eg. added/modified/deleted lines in a diff
-          @out << "\t" * 100 + ansi_clear
+          @out << (@line_filler ||= "\t" * 100 + "\e[0m")
           @out << open_token(@opened.last)
         end
       end
@@ -157,23 +157,16 @@ module CodeRay
         if color = TOKEN_COLORS[kind]
           if Hash === color
             @subcolors = color
-            ansi_colorize(color[:self]) if color[:self]
+            color[:self]
           else
             @subcolors = {}
-            ansi_colorize(color)
+            color
           end
         else
           @subcolors = nil
           ''
         end
       end
-      
-      def ansi_colorize(color)
-        Array(color).map { |c| "\e[#{c}m" }.join
-      end
-      def ansi_clear
-        ansi_colorize(0)
-      end
     end
   end
-end
\ No newline at end of file
+end
-- 
cgit v1.2.1


From 9d41587459ec6de14993e62c41666b0cc2692e9f Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 16:06:08 +0200
Subject: cleanup HTML encoder (#135)

---
 lib/coderay/encoders/html.rb | 66 ++++++++++++++++++++++----------------------
 1 file changed, 33 insertions(+), 33 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 0fd1317..2c66aab 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -126,22 +126,21 @@ module Encoders
     
   protected
     
-    HTML_ESCAPE = {  #:nodoc:
-      '&' => '&amp;',
-      '"' => '&quot;',
-      '>' => '&gt;',
-      '<' => '&lt;',
-    }
+    def self.make_html_escape_hash
+      {
+        '&' => '&amp;',
+        '"' => '&quot;',
+        '>' => '&gt;',
+        '<' => '&lt;',
+        # "\t" => will be set to ' ' * options[:tab_width] during setup
+      }.tap do |hash|
+        # Escape ASCII control codes except \x9 == \t and \xA == \n.
+        (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' }
+      end
+    end
     
-    # This was to prevent illegal HTML.
-    # Strange chars should still be avoided in codes.
-    evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
-    evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
-    #ansi_chars = Array(0x7f..0xff)
-    #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
-    # \x9 (\t) and \xA (\n) not included
-    #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
-    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
+    HTML_ESCAPE = make_html_escape_hash
+    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/
     
     TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
       h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
@@ -255,20 +254,10 @@ module Encoders
   public
     
     def text_token text, kind
-      if text =~ /#{HTML_ESCAPE_PATTERN}/o
-        text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
-      end
-      
       style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
       
-      if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0
-        close = '</span>' * c
-        reopen = ''
-        @opened.each_with_index do |k, index|
-          reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '<span>')
-        end
-        text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}")
-      end
+      text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
+      text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
       
       if style
         @out << style << text << '</span>'
@@ -285,9 +274,7 @@ module Encoders
     end
     
     def end_group kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a token group (%p) that is not open. Open are: %p.' % [kind, @opened[1..-1]]
-      end
+      check_group_nesting 'token group', kind if $CODERAY_DEBUG
       if @opened.pop
         @out << '</span>'
         @last_opened = @opened.last if @last_opened
@@ -310,15 +297,28 @@ module Encoders
     end
     
     def end_line kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a line (%p) that is not open. Open are: %p.' % [kind, @opened[1..-1]]
-      end
+      check_group_nesting 'line', kind if $CODERAY_DEBUG
       if @opened.pop
         @out << '</span>'
         @last_opened = @opened.last if @last_opened
       end
     end
     
+  protected
+    
+    def check_group_nesting name, kind
+      if @opened.empty? || @opened.last != kind
+        warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
+      end
+    end
+    
+    def break_lines text, style
+      reopen = ''
+      @opened.each_with_index do |k, index|
+        reopen << (@span_for_kind[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
+      end
+      text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
+    end
   end
   
 end
-- 
cgit v1.2.1


From c39168e8a8f8313a4e3b6a9f070f14b2cf5e18a7 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 17:17:07 +0200
Subject: cleanup HTML encoder #setup (#135)

---
 lib/coderay/encoders/html.rb | 70 ++++++++++++++++++++------------------------
 1 file changed, 32 insertions(+), 38 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 2c66aab..f0f5d62 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -171,13 +171,14 @@ module Encoders
     def setup options
       super
       
+      check_options options
+      
       if options[:wrap] || options[:line_numbers]
         @real_out = @out
         @out = ''
       end
       
       options[:break_lines] = true if options[:line_numbers] == :inline
-      
       @break_lines = (options[:break_lines] == true)
       
       @HTML_ESCAPE = HTML_ESCAPE.dup
@@ -187,43 +188,7 @@ module Encoders
       @last_opened = nil
       @css = CSS.new options[:style]
       
-      hint = options[:hint]
-      if hint && ![:debug, :info, :info_long].include?(hint)
-        raise ArgumentError, "Unknown value %p for :hint; \
-          expected :info, :info_long, :debug, false, or nil." % hint
-      end
-      
-      css_classes = TokenKinds
-      case options[:css]
-      when :class
-        @span_for_kind = Hash.new do |h, k|
-          if k.is_a? ::Symbol
-            kind = k_dup = k
-          else
-            kind = k.first
-            k_dup = k.dup
-          end
-          if kind != :space && (hint || css_class = css_classes[kind])
-            title = HTML.token_path_to_hint hint, k if hint
-            css_class ||= css_classes[kind]
-            h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
-          else
-            h[k_dup] = nil
-          end
-        end
-      when :style
-        @span_for_kind = Hash.new do |h, k|
-          kind = k.is_a?(Symbol) ? k : k.first
-          h[k.is_a?(Symbol) ? k : k.dup] =
-            if kind != :space && (hint || css_classes[kind])
-              title = HTML.token_path_to_hint hint, k if hint
-              style = @css.get_style Array(k).map { |c| css_classes[c] }
-              "<span#{title}#{" style=\"#{style}\"" if style}>"
-            end
-        end
-      else
-        raise ArgumentError, "Unknown value %p for :css." % options[:css]
-      end
+      @span_for_kind = make_span_for_kind(options[:css], options[:hint], @css)
       
       @set_last_opened = options[:hint] || options[:css] == :style
     end
@@ -306,6 +271,35 @@ module Encoders
     
   protected
     
+    def check_options options
+      unless [false, nil, :debug, :info, :info_long].include? options[:hint]
+        raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
+      end
+      
+      unless [:class, :style].include? options[:css]
+        raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
+      end
+    end
+    
+    def make_span_for_kind method, hint, css
+      css_classes = TokenKinds
+      
+      Hash.new do |h, k|
+        kind = k.is_a?(Symbol) ? k : k.first
+        
+        h[k.is_a?(Symbol) ? k : k.dup] =
+          if kind != :space && ((css_class = css_classes[kind]) || hint)
+            title = HTML.token_path_to_hint hint, k if hint
+            if method == :class
+              "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
+            else
+              style = css.get_style k.is_a?(Array) ? k.map { |c| css_classes[c] } : [css_class]
+              "<span#{title}#{" style=\"#{style}\"" if style}>"
+            end
+          end
+      end
+    end
+    
     def check_group_nesting name, kind
       if @opened.empty? || @opened.last != kind
         warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
-- 
cgit v1.2.1


From a8a17fc78987bfd4236c929fd54c58f8014fe88d Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 17:35:17 +0200
Subject: simplify more HTML encoder methods (#135)

---
 lib/coderay/encoders/html.rb | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index f0f5d62..a6c0a8e 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -171,18 +171,16 @@ module Encoders
     def setup options
       super
       
-      check_options options
+      check_options! options
       
       if options[:wrap] || options[:line_numbers]
         @real_out = @out
         @out = ''
       end
       
-      options[:break_lines] = true if options[:line_numbers] == :inline
       @break_lines = (options[:break_lines] == true)
       
-      @HTML_ESCAPE = HTML_ESCAPE.dup
-      @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
+      @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
       
       @opened = []
       @last_opened = nil
@@ -271,7 +269,7 @@ module Encoders
     
   protected
     
-    def check_options options
+    def check_options! options
       unless [false, nil, :debug, :info, :info_long].include? options[:hint]
         raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
       end
@@ -279,24 +277,23 @@ module Encoders
       unless [:class, :style].include? options[:css]
         raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
       end
+      
+      options[:break_lines] = true if options[:line_numbers] == :inline
     end
     
     def make_span_for_kind method, hint, css
-      css_classes = TokenKinds
-      
-      Hash.new do |h, k|
-        kind = k.is_a?(Symbol) ? k : k.first
-        
-        h[k.is_a?(Symbol) ? k : k.dup] =
-          if kind != :space && ((css_class = css_classes[kind]) || hint)
-            title = HTML.token_path_to_hint hint, k if hint
-            if method == :class
-              "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
-            else
-              style = css.get_style k.is_a?(Array) ? k.map { |c| css_classes[c] } : [css_class]
-              "<span#{title}#{" style=\"#{style}\"" if style}>"
-            end
-          end
+      Hash.new do |h, kinds|
+        h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
+          css_class = TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
+          title     = HTML.token_path_to_hint hint, kinds if hint
+          
+          if method == :style
+            style = css.get_style(kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]])
+            "<span#{title}#{" style=\"#{style}\"" if style}>"
+          else
+            "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
+          end if css_class || title
+        end
       end
     end
     
-- 
cgit v1.2.1


From cd7433c5a4d9114e8a398d4fd3eeeadecf290260 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 17:52:33 +0200
Subject: cleanup HTML encoder CSS styles/classes algorithm (#135)

---
 lib/coderay/encoders/html.rb     | 54 ++++++++++++++++++++++++----------------
 lib/coderay/encoders/html/css.rb | 14 +++++------
 2 files changed, 40 insertions(+), 28 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index a6c0a8e..b897f5e 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -186,7 +186,7 @@ module Encoders
       @last_opened = nil
       @css = CSS.new options[:style]
       
-      @span_for_kind = make_span_for_kind(options[:css], options[:hint], @css)
+      @span_for_kinds = make_span_for_kinds(options[:css], options[:hint])
       
       @set_last_opened = options[:hint] || options[:css] == :style
     end
@@ -217,7 +217,7 @@ module Encoders
   public
     
     def text_token text, kind
-      style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
+      style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
       
       text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
       text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
@@ -231,22 +231,19 @@ module Encoders
     
     # token groups, eg. strings
     def begin_group kind
-      @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
+      @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>')
       @opened << kind
       @last_opened = kind if @set_last_opened
     end
     
     def end_group kind
       check_group_nesting 'token group', kind if $CODERAY_DEBUG
-      if @opened.pop
-        @out << '</span>'
-        @last_opened = @opened.last if @last_opened
-      end
+      close_span
     end
     
     # whole lines to be highlighted, eg. a deleted line in a diff
     def begin_line kind
-      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
+      if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
         if style['class="']
           @out << style.sub('class="', 'class="line ')
         else
@@ -261,10 +258,7 @@ module Encoders
     
     def end_line kind
       check_group_nesting 'line', kind if $CODERAY_DEBUG
-      if @opened.pop
-        @out << '</span>'
-        @last_opened = @opened.last if @last_opened
-      end
+      close_span
     end
     
   protected
@@ -281,18 +275,29 @@ module Encoders
       options[:break_lines] = true if options[:line_numbers] == :inline
     end
     
-    def make_span_for_kind method, hint, css
+    def css_class_for_kinds kinds
+      TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
+    end
+    
+    def style_for_kinds kinds
+      css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]
+      @css.get_style_for_css_classes css_classes
+    end
+    
+    def make_span_for_kinds method, hint
       Hash.new do |h, kinds|
         h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
-          css_class = TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
+          css_class = css_class_for_kinds(kinds)
           title     = HTML.token_path_to_hint hint, kinds if hint
           
-          if method == :style
-            style = css.get_style(kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]])
-            "<span#{title}#{" style=\"#{style}\"" if style}>"
-          else
-            "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
-          end if css_class || title
+          if css_class || title
+            if method == :style
+              style = style_for_kinds(kinds)
+              "<span#{title}#{" style=\"#{style}\"" if style}>"
+            else
+              "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
+            end
+          end
         end
       end
     end
@@ -306,10 +311,17 @@ module Encoders
     def break_lines text, style
       reopen = ''
       @opened.each_with_index do |k, index|
-        reopen << (@span_for_kind[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
+        reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
       end
       text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
     end
+    
+    def close_span
+      if @opened.pop
+        @out << '</span>'
+        @last_opened = @opened.last if @last_opened
+      end
+    end
   end
   
 end
diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb
index 6de4b46..164d7f8 100644
--- a/lib/coderay/encoders/html/css.rb
+++ b/lib/coderay/encoders/html/css.rb
@@ -11,7 +11,7 @@ module Encoders
       end
 
       def initialize style = :default
-        @classes = Hash.new
+        @styles = Hash.new
         style = CSS.load_stylesheet style
         @stylesheet = [
           style::CSS_MAIN_STYLES,
@@ -20,12 +20,12 @@ module Encoders
         parse style::TOKEN_COLORS
       end
 
-      def get_style styles
-        cl = @classes[styles.first]
+      def get_style_for_css_classes css_classes
+        cl = @styles[css_classes.first]
         return '' unless cl
         style = ''
-        1.upto styles.size do |offset|
-          break if style = cl[styles[offset .. -1]]
+        1.upto css_classes.size do |offset|
+          break if style = cl[css_classes[offset .. -1]]
         end
         # warn 'Style not found: %p' % [styles] if style.empty?
         return style
@@ -52,8 +52,8 @@ module Encoders
           for selector in selectors.split(',')
             classes = selector.scan(/[-\w]+/)
             cl = classes.pop
-            @classes[cl] ||= Hash.new
-            @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
+            @styles[cl] ||= Hash.new
+            @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
           end
         end
       end
-- 
cgit v1.2.1


From 2fb89ea3abfd8ee6c4eed8defb805a1abd7187d9 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 18:03:18 +0200
Subject: refactor Scanner a bit (#135)

---
 lib/coderay/scanner.rb | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 907cf00..80ccc5d 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -184,14 +184,8 @@ module CodeRay
         options = @options.merge(options)
         @tokens = options[:tokens] || @tokens || Tokens.new
         @tokens.scanner = self if @tokens.respond_to? :scanner=
-        case source
-        when Array
-          self.string = self.class.normalize(source.join)
-        when nil
-          reset
-        else
-          self.string = self.class.normalize(source)
-        end
+        
+        set_string_from_source source
         
         begin
           scan_tokens @tokens, options
@@ -261,6 +255,17 @@ module CodeRay
       def setup  # :doc:
       end
       
+      def set_string_from_source source
+        case source
+        when Array
+          self.string = self.class.normalize(source.join)
+        when nil
+          reset
+        else
+          self.string = self.class.normalize(source)
+        end
+      end
+      
       # This is the central method, and commonly the only one a
       # subclass implements.
       #
@@ -277,9 +282,7 @@ module CodeRay
         @binary_string = nil if defined? @binary_string
       end
       
-      # Scanner error with additional status information
-      def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller
-        raise ScanError, <<-EOE % [
+      SCAN_ERROR_MESSAGE = <<-MESSAGE
 
 
 ***ERROR in %s: %s (after %d tokens)
@@ -297,7 +300,11 @@ surrounding code:
 
 ***ERROR***
 
-        EOE
+      MESSAGE
+      
+      # Scanner error with additional status information
+      def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller
+        raise ScanError, SCAN_ERROR_MESSAGE % [
           File.basename(caller[0]),
           msg,
           tokens.respond_to?(:size) ? tokens.size : 0,
-- 
cgit v1.2.1


From c386e044cff195ae922a0b7625dfa498954bf772 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 18:15:17 +0200
Subject: refactor Scanner a bit more (#135)

---
 lib/coderay/scanner.rb | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 80ccc5d..9a5be58 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -182,9 +182,8 @@ module CodeRay
       # Scan the code and returns all tokens in a Tokens object.
       def tokenize source = nil, options = {}
         options = @options.merge(options)
-        @tokens = options[:tokens] || @tokens || Tokens.new
-        @tokens.scanner = self if @tokens.respond_to? :scanner=
         
+        set_tokens_from_options options
         set_string_from_source source
         
         begin
@@ -266,6 +265,11 @@ module CodeRay
         end
       end
       
+      def set_tokens_from_options options
+        @tokens = options[:tokens] || @tokens || Tokens.new
+        @tokens.scanner = self if @tokens.respond_to? :scanner=
+      end
+      
       # This is the central method, and commonly the only one a
       # subclass implements.
       #
@@ -292,7 +296,7 @@ tokens:
 
 current line: %d  column: %d  pos: %d
 matched: %p  state: %p
-bol? = %p,  eos? = %p
+bol?: %p,  eos?: %p
 
 surrounding code:
 %p  ~~  %p
@@ -303,14 +307,15 @@ surrounding code:
       MESSAGE
       
       # Scanner error with additional status information
-      def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller
+      def raise_inspect msg, tokens, state = self.state, ambit = 30, backtrace = caller
         raise ScanError, SCAN_ERROR_MESSAGE % [
           File.basename(caller[0]),
           msg,
-          tokens.respond_to?(:size) ? tokens.size : 0,
-          tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '',
+          tokens.respond_to?(:size) ? tokens.size : '[tokens.size undefined]',
+          tokens.respond_to?(:last) ? tokens.last(10).map(&:inspect).join("\n") : '[tokens.last undefined]',
           line, column, pos,
-          matched, state, bol?, eos?,
+          matched, state || 'No state given!',
+          bol?, eos?,
           binary_string[pos - ambit, ambit],
           binary_string[pos, ambit],
         ], backtrace
-- 
cgit v1.2.1


From 65a4b4dfe93e9be0d9f753f063b8f77f8be5af77 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 18:23:17 +0200
Subject: trying to reduce "complexity" of #raise_inspect (#135)

---
 lib/coderay/scanner.rb | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 9a5be58..68451ae 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -289,7 +289,7 @@ module CodeRay
       SCAN_ERROR_MESSAGE = <<-MESSAGE
 
 
-***ERROR in %s: %s (after %d tokens)
+***ERROR in %s: %s (after %s tokens)
 
 tokens:
 %s
@@ -311,8 +311,8 @@ surrounding code:
         raise ScanError, SCAN_ERROR_MESSAGE % [
           File.basename(caller[0]),
           msg,
-          tokens.respond_to?(:size) ? tokens.size : '[tokens.size undefined]',
-          tokens.respond_to?(:last) ? tokens.last(10).map(&:inspect).join("\n") : '[tokens.last undefined]',
+          tokens_size,
+          tokens_last(10).map(&:inspect).join("\n"),
           line, column, pos,
           matched, state || 'No state given!',
           bol?, eos?,
@@ -321,6 +321,14 @@ surrounding code:
         ], backtrace
       end
       
+      def tokens_size
+        tokens.size if tokens.respond_to?(:size)
+      end
+      
+      def tokens_last n
+        tokens.respond_to?(:last) ? tokens.last(n) : []
+      end
+      
       # Shorthand for scan_until(/\z/).
       # This method also avoids a JRuby 1.9 mode bug.
       def scan_rest
-- 
cgit v1.2.1


From 65b8ab0fda2ac06a2d6109ea96485ab148bab1a4 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 18:32:16 +0200
Subject: trying to reduce "complexity" of #raise_inspect (#135)

---
 lib/coderay/scanner.rb | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 68451ae..26bcbf3 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -307,25 +307,27 @@ surrounding code:
       MESSAGE
       
       # Scanner error with additional status information
-      def raise_inspect msg, tokens, state = self.state, ambit = 30, backtrace = caller
-        raise ScanError, SCAN_ERROR_MESSAGE % [
-          File.basename(caller[0]),
-          msg,
-          tokens_size,
-          tokens_last(10).map(&:inspect).join("\n"),
+      def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller
+        raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace
+      end
+      
+      def raise_inspect_arguments message, tokens, state, ambit
+        return File.basename(caller[0]),
+          message,
+          tokens_size(tokens),
+          tokens_last(tokens, 10).map(&:inspect).join("\n"),
           line, column, pos,
           matched, state || 'No state given!',
           bol?, eos?,
           binary_string[pos - ambit, ambit],
-          binary_string[pos, ambit],
-        ], backtrace
+          binary_string[pos, ambit]
       end
       
-      def tokens_size
+      def tokens_size tokens
         tokens.size if tokens.respond_to?(:size)
       end
       
-      def tokens_last n
+      def tokens_last tokens, n
         tokens.respond_to?(:last) ? tokens.last(n) : []
       end
       
-- 
cgit v1.2.1


From cbf002e7c5e78cee9c2ef7f565b064d9f666bc2b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 18:39:19 +0200
Subject: trying to reduce "complexity" of #raise_inspect (#135)

---
 lib/coderay/scanner.rb | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 26bcbf3..b3f7e17 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -294,9 +294,7 @@ module CodeRay
 tokens:
 %s
 
-current line: %d  column: %d  pos: %d
-matched: %p  state: %p
-bol?: %p,  eos?: %p
+%s
 
 surrounding code:
 %p  ~~  %p
@@ -306,21 +304,33 @@ surrounding code:
 
       MESSAGE
       
-      # Scanner error with additional status information
-      def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller
-        raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace
-      end
-      
       def raise_inspect_arguments message, tokens, state, ambit
         return File.basename(caller[0]),
           message,
           tokens_size(tokens),
           tokens_last(tokens, 10).map(&:inspect).join("\n"),
+          scanner_state_info(state),
+          binary_string[pos - ambit, ambit],
+          binary_string[pos, ambit]
+      end
+      
+      SCANNER_STATE_INFO = <<-INFO
+current line: %d  column: %d  pos: %d
+matched: %p  state: %p
+bol?: %p,  eos?: %p
+      INFO
+      
+      def scanner_state_info state
+        SCANNER_STATE_INFO % [
           line, column, pos,
           matched, state || 'No state given!',
           bol?, eos?,
-          binary_string[pos - ambit, ambit],
-          binary_string[pos, ambit]
+        ]
+      end
+      
+      # Scanner error with additional status information
+      def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller
+        raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace
       end
       
       def tokens_size tokens
-- 
cgit v1.2.1


From a69d5d43b79a4f2cf5aaccc50b2275bf0dab4aba Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 23:29:14 +0200
Subject: accept all idents as Ruby 1.9 hash keys (#126)

---
 lib/coderay/scanners/ruby.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index c5cf1e2..c282f31 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -96,7 +96,7 @@ module Scanners
                                       /#{patterns::METHOD_NAME}/o)
               
               kind = patterns::IDENT_KIND[match]
-              if kind == :ident && value_expected != :colon_expected && scan(/:(?!:)/)
+              if value_expected != :colon_expected && scan(/:(?!:)/)
                 value_expected = true
                 encoder.text_token match, :key
                 encoder.text_token ':',   :operator
-- 
cgit v1.2.1


From 33f1a738ea85ea40a5fe760ac2591065f3e29033 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Mon, 10 Jun 2013 23:59:47 +0200
Subject: reset string_delimiter when detecting open string

---
 lib/coderay/scanners/java_script.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb
index 8587b06..9eb0a0a 100644
--- a/lib/coderay/scanners/java_script.rb
+++ b/lib/coderay/scanners/java_script.rb
@@ -184,6 +184,7 @@ module Scanners
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group state
             encoder.text_token match, :error unless match.empty?
+            string_delimiter = nil
             key_expected = value_expected = false
             state = :initial
           else
-- 
cgit v1.2.1


From f46a737548583a6cf12cf188bd24f89caa54fde7 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 11 Jun 2013 01:09:48 +0200
Subject: set line numbers to word-break: normal to fix #102

---
 lib/coderay/styles/alpha.rb | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 1f073b6..81bbe3b 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -39,6 +39,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
   color: gray !important;
   text-decoration: none !important;
 }
+.CodeRay .line-numbers pre {
+  word-break: normal;
+}
 .CodeRay .line-numbers a:target { color: blue !important; }
 .CodeRay .line-numbers .highlighted { color: red !important; }
 .CodeRay .line-numbers .highlighted a { color: red !important; }
-- 
cgit v1.2.1


From b159057934d2c84c0a25c83f6cbe59010345d0a7 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 11 Jun 2013 03:29:54 +0200
Subject: tweak CSS tag/id token kind patch from noprompt

---
 lib/coderay/scanners/css.rb  | 4 ++--
 lib/coderay/scanners/sass.rb | 6 +++---
 lib/coderay/styles/alpha.rb  | 1 +
 3 files changed, 6 insertions(+), 5 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 22c6bfc..732f9c5 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -10,7 +10,7 @@ module Scanners
       :class, :pseudo_class, :tag,
       :id, :directive,
       :key, :value, :operator, :color, :float, :string,
-      :error, :important,
+      :error, :important, :type,
     ]  # :nodoc:
     
     module RE  # :nodoc:
@@ -99,7 +99,7 @@ module Scanners
             
           when :media_before_name
             if match = scan(RE::Ident)
-              encoder.text_token match, :tag
+              encoder.text_token match, :type
               states[-1] = :media_after_name
               next
             end
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 0eb2caa..218ee57 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -38,16 +38,16 @@ module Scanners
         elsif case states.last
           when :initial, :media, :sass_inline
             if match = scan(/(?>#{RE::Ident})(?!\()/ox)
-              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :type)
+              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag)
               next
             elsif !value_expected && (match = scan(/\*/))
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
               next
             elsif match = scan(RE::Id)
-              encoder.text_token match, :constant
+              encoder.text_token match, :id
               next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 81bbe3b..a21fbf1 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -84,6 +84,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .function .delimiter { color:#024; font-weight:bold }
 .global-variable { color:#d70 }
 .hex { color:#02b }
+.id  { color:#33D; font-weight:bold }
 .imaginary { color:#f00 }
 .include { color:#B44; font-weight:bold }
 .inline { background-color: hsla(0,0%,0%,0.07); color: black }
-- 
cgit v1.2.1


From 2e4e83bf84282d6ce7fbc9eeff2cbe1c79788a9b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Tue, 11 Jun 2013 16:30:26 +0200
Subject: tweak Sass scanner: comments, includes, functions

---
 lib/coderay/scanners/sass.rb | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 218ee57..167051d 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -7,8 +7,6 @@ module Scanners
     register_for :sass
     file_extension 'sass'
     
-    SASS_FUNCTION = /(?:inline-image|linear-gradient|color-stops|mix|lighten|darken|rotate|image-url|image-width|image-height|sprite-url|sprite-path|sprite-file|sprite-map|sprite-position|sprite|unquote|join|round|ceil|floor|nth)/
-    
     STRING_CONTENT_PATTERN = {
       "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
       '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
@@ -26,10 +24,22 @@ module Scanners
       
       until eos?
         
-        if match = scan(/\s+/)
+        if bol? && (match = scan(/(?>( +)?(\/[\*\/])(.+)?)(?=\n)/))
+          encoder.text_token self[1], :space if self[1]
+          encoder.begin_group :comment
+          encoder.text_token self[2], :delimiter
+          encoder.text_token self[3], :content if self[3]
+          if match = scan(/(?:\n+#{self[1]} .*)+/)
+            encoder.text_token match, :content
+          end
+          encoder.end_group :comment
+        elsif match = scan(/\n|[^\n\S]+\n?/)
           encoder.text_token match, :space
-          value_expected = false if match.index(/\n/)
-          
+          if match.index(/\n/)
+            value_expected = false
+            states.pop if states.last == :include
+          end
+        
         elsif states.last == :sass_inline && (match = scan(/\}/))
           encoder.text_token match, :inline_delimiter
           encoder.end_group :inline
@@ -61,7 +71,11 @@ module Scanners
             elsif match = scan(/(\=|@mixin +)#{RE::Ident}/o)
               encoder.text_token match, :function
               next
-            elsif match = scan(/@media/)
+            elsif match = scan(/@import\b/)
+              encoder.text_token match, :directive
+              states << :include
+              next
+            elsif match = scan(/@media\b/)
               encoder.text_token match, :directive
               # states.push :media_before_name
               next
@@ -90,13 +104,19 @@ module Scanners
               encoder.text_token match, :inline_delimiter
               states.push :sass_inline
             elsif match = scan(/ \\ | $ /x)
-              encoder.end_group state
+              encoder.end_group :string
               encoder.text_token match, :error unless match.empty?
               states.pop
             else
               raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
             end
           
+          when :include
+            if match = scan(/[^\s'",]+/)
+              encoder.text_token match, :include
+              next
+            end
+          
           else
             #:nocov:
             raise_inspect 'Unknown state', encoder
@@ -148,9 +168,6 @@ module Scanners
             states.push :string
           end
           
-        elsif match = scan(/#{SASS_FUNCTION}/o)
-          encoder.text_token match, :predefined
-          
         elsif match = scan(/#{RE::Function}/o)
           encoder.begin_group :function
           start = match[/^[-\w]+\(/]
@@ -163,6 +180,9 @@ module Scanners
           end
           encoder.end_group :function
           
+        elsif match = scan(/[a-z][-a-z_]*(?=\()/o)
+          encoder.text_token match, :predefined
+          
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
           encoder.text_token match, :float
           
-- 
cgit v1.2.1


From abdc3e4a18b664e9d6e20e06913a0aa342f8ca73 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Wed, 12 Jun 2013 15:49:44 +0200
Subject: tweak Taskpaper scanner, cleanup doc_string kind

---
 lib/coderay/encoders/terminal.rb  |  2 +-
 lib/coderay/scanners/taskpaper.rb | 18 ++++++++++--------
 lib/coderay/styles/alpha.rb       |  3 +--
 lib/coderay/token_kinds.rb        |  8 ++------
 4 files changed, 14 insertions(+), 17 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb
index 500e5d8..9894b91 100644
--- a/lib/coderay/encoders/terminal.rb
+++ b/lib/coderay/encoders/terminal.rb
@@ -37,7 +37,7 @@ module CodeRay
         :directive => "\e[32m\e[4m",
         :doc => "\e[46m",
         :doctype => "\e[1;30m",
-        :doc_string => "\e[31m\e[4m",
+        :docstring => "\e[31m\e[4m",
         :entity => "\e[33m",
         :error => "\e[1;33m\e[41m",
         :exception => "\e[1;31m",
diff --git a/lib/coderay/scanners/taskpaper.rb b/lib/coderay/scanners/taskpaper.rb
index c6eb3b3..42670bc 100644
--- a/lib/coderay/scanners/taskpaper.rb
+++ b/lib/coderay/scanners/taskpaper.rb
@@ -10,17 +10,19 @@ module Scanners
     
     def scan_tokens encoder, options
       until eos?
-        if match = scan(/^\S.*:.*$/)                  # project
-          encoder.text_token(match, :project)
-        elsif match = scan(/^-.+@done(?:\(.*)?.*$/)   # completed task
-          encoder.text_token(match, :complete)
-        elsif match = scan(/^-.+$/)                   # task
+        if match = scan(/\S.*:.*$/)                  # project
+          encoder.text_token(match, :namespace)
+        elsif match = scan(/-.+@done.*/)             # completed task
+          encoder.text_token(match, :done)
+        elsif match = scan(/-(?:[^@\n]+|@(?!due))*/) # task
           encoder.text_token(match, :plain)
-        elsif match = scan(/^.+$/)                    # comment
+        elsif match = scan(/@due.*/)                 # comment
+          encoder.text_token(match, :important)
+        elsif match = scan(/.+/)                     # comment
           encoder.text_token(match, :comment)
-        elsif match = scan(/\s+/)                     # space
+        elsif match = scan(/\s+/)                    # space
           encoder.text_token(match, :space)
-        else                                          # other
+        else                                         # other
           encoder.text_token getch, :error
         end
       end
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 6829686..f57e4a1 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -141,8 +141,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .change .change { color: #88f }
 .head .head { color: #f4f }
 
-.project  { color: #707; font-weight: bold }
-.complete { text-decoration: line-through; color: gray }
+.done { text-decoration: line-through; color: gray }
     TOKENS
 
   end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index f7f19bb..de3a0d0 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -29,7 +29,8 @@ module CodeRay
     :directive           => 'directive',
     :doc                 => 'doc',
     :doctype             => 'doctype',
-    :doc_string          => 'doc-string',
+    :docstring           => 'doc-string',
+    :done                => 'done',
     :entity              => 'entity',
     :error               => 'error',
     :escape              => 'escape',
@@ -75,9 +76,6 @@ module CodeRay
     :insert              => 'insert',
     
     :eyecatcher          => 'eyecatcher',
-
-    :project             => 'project',
-    :complete            => 'complete',
     
     :ident               => false,
     :operator            => false,
@@ -87,6 +85,4 @@ module CodeRay
   )
   
   TokenKinds[:method]    = TokenKinds[:function]
-  TokenKinds[:escape]    = TokenKinds[:delimiter]
-  TokenKinds[:docstring] = TokenKinds[:comment]
 end
-- 
cgit v1.2.1


From 65eeaca6c18bea379b966559084e47d6fb45c75c Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Wed, 12 Jun 2013 16:19:41 +0200
Subject: add python binary string

---
 lib/coderay/scanners/python.rb | 4 ++--
 lib/coderay/styles/alpha.rb    | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index a9492ab..09c8b6e 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -157,12 +157,12 @@ module Scanners
             encoder.text_token match, :operator
           
           elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
+            modifiers = self[1]
             string_delimiter = self[2]
-            string_type = docstring_coming ? :docstring : :string
+            string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string)
             docstring_coming = false if docstring_coming
             encoder.begin_group string_type
             string_raw = false
-            modifiers = self[1]
             unless modifiers.empty?
               string_raw = !!modifiers.index(?r)
               encoder.text_token modifiers, :modifier
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index f57e4a1..2ad088a 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -56,7 +56,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .annotation { color:#007 }
 .attribute-name { color:#b48 }
 .attribute-value { color:#700 }
-.binary { color:#509 }
+.binary { color:#549 }
+.binary .delimiter { color:#325 }
+.binary .char { color:#325 }
 .char .content { color:#D20 }
 .char .delimiter { color:#710 }
 .char { color:#D20 }
-- 
cgit v1.2.1


From 3cc51316b3836c030b7da079f5177a79405700a7 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Thu, 13 Jun 2013 04:05:06 +0200
Subject: add taskpaper file extension

---
 lib/coderay/helpers/file_type.rb | 97 ++++++++++++++++++++--------------------
 1 file changed, 49 insertions(+), 48 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index a5d83ff..6144e8c 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -77,54 +77,55 @@ module CodeRay
     end
     
     TypeFromExt = {
-      'c'        => :c,
-      'cfc'      => :xml,
-      'cfm'      => :xml,
-      'clj'      => :clojure,
-      'css'      => :css,
-      'diff'     => :diff,
-      'dpr'      => :delphi,
-      'erb'      => :erb,
-      'gemspec'  => :ruby,
-      'groovy'   => :groovy,
-      'gvy'      => :groovy,
-      'h'        => :c,
-      'haml'     => :haml,
-      'htm'      => :html,
-      'html'     => :html,
-      'html.erb' => :erb,
-      'java'     => :java,
-      'js'       => :java_script,
-      'json'     => :json,
-      'mab'      => :ruby,
-      'pas'      => :delphi,
-      'patch'    => :diff,
-      'phtml'    => :php,
-      'php'      => :php,
-      'php3'     => :php,
-      'php4'     => :php,
-      'php5'     => :php,
-      'prawn'    => :ruby,
-      'py'       => :python,
-      'py3'      => :python,
-      'pyw'      => :python,
-      'rake'     => :ruby,
-      'raydebug' => :raydebug,
-      'rb'       => :ruby,
-      'rbw'      => :ruby,
-      'rhtml'    => :erb,
-      'rjs'      => :ruby,
-      'rpdf'     => :ruby,
-      'ru'       => :ruby,
-      'rxml'     => :ruby,
-      'sass'     => :sass,
-      'sql'      => :sql,
-      'tmproj'   => :xml,
-      'xaml'     => :xml,
-      'xhtml'    => :html,
-      'xml'      => :xml,
-      'yaml'     => :yaml,
-      'yml'      => :yaml,
+      'c'         => :c,
+      'cfc'       => :xml,
+      'cfm'       => :xml,
+      'clj'       => :clojure,
+      'css'       => :css,
+      'diff'      => :diff,
+      'dpr'       => :delphi,
+      'erb'       => :erb,
+      'gemspec'   => :ruby,
+      'groovy'    => :groovy,
+      'gvy'       => :groovy,
+      'h'         => :c,
+      'haml'      => :haml,
+      'htm'       => :html,
+      'html'      => :html,
+      'html.erb'  => :erb,
+      'java'      => :java,
+      'js'        => :java_script,
+      'json'      => :json,
+      'mab'       => :ruby,
+      'pas'       => :delphi,
+      'patch'     => :diff,
+      'phtml'     => :php,
+      'php'       => :php,
+      'php3'      => :php,
+      'php4'      => :php,
+      'php5'      => :php,
+      'prawn'     => :ruby,
+      'py'        => :python,
+      'py3'       => :python,
+      'pyw'       => :python,
+      'rake'      => :ruby,
+      'raydebug'  => :raydebug,
+      'rb'        => :ruby,
+      'rbw'       => :ruby,
+      'rhtml'     => :erb,
+      'rjs'       => :ruby,
+      'rpdf'      => :ruby,
+      'ru'        => :ruby,
+      'rxml'      => :ruby,
+      'sass'      => :sass,
+      'sql'       => :sql,
+      'taskpaper' => :taskpaper,
+      'tmproj'    => :xml,
+      'xaml'      => :xml,
+      'xhtml'     => :html,
+      'xml'       => :xml,
+      'yaml'      => :yaml,
+      'yml'       => :yaml,
     }
     for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
       TypeFromExt[cpp_alias] = :cpp
-- 
cgit v1.2.1


From 4a85580f76421bb0afb92501119c4dfcfa6540ef Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Thu, 13 Jun 2013 04:12:14 +0200
Subject: tweak Terminal encoder, cleanup token kinds

---
 lib/coderay/encoders/terminal.rb | 179 ++++++++++++++++++++++-----------------
 lib/coderay/styles/alpha.rb      |  29 +++----
 lib/coderay/token_kinds.rb       | 136 +++++++++++++++--------------
 3 files changed, 181 insertions(+), 163 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb
index 9894b91..c7ae014 100644
--- a/lib/coderay/encoders/terminal.rb
+++ b/lib/coderay/encoders/terminal.rb
@@ -19,105 +19,135 @@ module CodeRay
       register_for :terminal
       
       TOKEN_COLORS = {
-        :annotation => "\e[35m",
-        :attribute_name => "\e[33m",
+        :debug => "\e[1;37;44m",
+        
+        :annotation => "\e[34m",
+        :attribute_name => "\e[35m",
         :attribute_value => "\e[31m",
-        :binary => "\e[1;35m",
+        :binary => {
+          :self => "\e[31m",
+          :char => "\e[1;31m",
+          :delimiter => "\e[1;31m",
+        },
         :char => {
-          :self => "\e[36m", :delimiter => "\e[1;34m"
+          :self => "\e[35m",
+          :delimiter => "\e[1;35m"
         },
-        :class => "\e[1;35m",
+        :class => "\e[1;35;4m",
         :class_variable => "\e[36m",
         :color => "\e[32m",
-        :comment => "\e[37m",
-        :complex => "\e[1;34m",
-        :constant => "\e[1;34m\e[4m",
-        :decoration => "\e[35m",
-        :definition => "\e[1;32m",
-        :directive => "\e[32m\e[4m",
-        :doc => "\e[46m",
-        :doctype => "\e[1;30m",
-        :docstring => "\e[31m\e[4m",
-        :entity => "\e[33m",
-        :error => "\e[1;33m\e[41m",
+        :comment => {
+          :self => "\e[1;30m",
+          :char => "\e[37m",
+          :delimiter => "\e[37m",
+        },
+        :constant => "\e[1;34;4m",
+        :decorator => "\e[35m",
+        :definition => "\e[1;33m",
+        :directive => "\e[33m",
+        :docstring => "\e[31m",
+        :doctype => "\e[1;34m",
+        :done => "\e[1;30;2m",
+        :entity => "\e[31m",
+        :error => "\e[1;37;41m",
         :exception => "\e[1;31m",
         :float => "\e[1;35m",
         :function => "\e[1;34m",
-        :global_variable => "\e[42m",
+        :global_variable => "\e[1;32m",
         :hex => "\e[1;36m",
-        :include => "\e[33m",
+        :id => "\e[1;34m",
+        :include => "\e[31m",
         :integer => "\e[1;34m",
-        :key => "\e[35m",
-        :label => "\e[1;15m",
+        :imaginary => "\e[1;34m",
+        :important => "\e[1;31m",
+        :key => {
+          :self => "\e[35m",
+          :char => "\e[1;35m",
+          :delimiter => "\e[1;35m",
+        },
+        :keyword => "\e[32m",
+        :label => "\e[1;33m",
         :local_variable => "\e[33m",
-        :octal => "\e[1;35m",
-        :operator_name => "\e[1;29m",
+        :namespace => "\e[1;35m",
+        :octal => "\e[1;34m",
+        :predefined => "\e[36m",
         :predefined_constant => "\e[1;36m",
-        :predefined_type => "\e[1;30m",
-        :predefined => "\e[4m\e[1;34m",
-        :preprocessor => "\e[36m",
+        :predefined_type => "\e[1;32m",
+        :preprocessor => "\e[1;36m",
         :pseudo_class => "\e[1;34m",
         :regexp => {
-          :self => "\e[31m",
-          :content => "\e[31m",
-          :delimiter => "\e[1;29m",
+          :self => "\e[35m",
+          :delimiter => "\e[1;35m",
           :modifier => "\e[35m",
+          :char => "\e[1;35m",
         },
-        :reserved => "\e[1;31m",
+        :reserved => "\e[32m",
         :shell => {
-          :self => "\e[42m",
-          :content => "\e[1;29m",
-          :delimiter => "\e[37m",
+          :self => "\e[33m",
+          :char => "\e[1;33m",
+          :delimiter => "\e[1;33m",
+          :escape => "\e[1;33m",
         },
         :string => {
-          :self => "\e[32m",
-          :modifier => "\e[1;32m",
-          :escape => "\e[1;36m",
-          :delimiter => "\e[1;32m",
-          :char => "\e[1;36m",
+          :self => "\e[31m",
+          :modifier => "\e[1;31m",
+          :char => "\e[1;35m",
+          :delimiter => "\e[1;31m",
+          :escape => "\e[1;31m",
+        },
+        :symbol => {
+          :self => "\e[33m",
+          :delimiter => "\e[1;33m",
         },
-        :symbol => "\e[1;32m",
-        :tag => "\e[1;34m",
+        :tag => "\e[32m",
         :type => "\e[1;34m",
         :value => "\e[36m",
-        :variable => "\e[1;34m",
+        :variable => "\e[34m",
         
-        :insert => "\e[42m",
-        :delete => "\e[41m",
-        :change => "\e[44m",
-        :head => "\e[45m"
+        :insert => {
+          :self => "\e[42m",
+          :insert => "\e[1;32;42m",
+          :eyecatcher => "\e[102m",
+        },
+        :delete => {
+          :self => "\e[41m",
+          :delete => "\e[1;31;41m",
+          :eyecatcher => "\e[101m",
+        },
+        :change => {
+          :self => "\e[44m",
+          :change => "\e[37;44m",
+        },
+        :head => {
+          :self => "\e[45m",
+          :filename => "\e[37;45m"
+        },
       }
+      
       TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
-      TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
-      TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] =
-        TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
+      TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
       
     protected
       
       def setup(options)
         super
         @opened = []
-        @subcolors = nil
+        @color_scopes = [TOKEN_COLORS]
       end
       
     public
       
       def text_token text, kind
-        if color = (@subcolors || TOKEN_COLORS)[kind]
-          if Hash === color
-            if color[:self]
-              color = color[:self]
-            else
-              @out << text
-              return
-            end
-          end
+        if color = @color_scopes.last[kind]
+          color = color[:self] if color.is_a? Hash
           
           @out << color
-          @out << text.gsub("\n", "\e[0m\n" + color)
+          @out << (text.index("\n") ? text.gsub("\n", "\e[0m\n" + color) : text)
           @out << "\e[0m"
-          @out << @subcolors[:self] if @subcolors
+          if outer_color = @color_scopes.last[:self]
+            @out << outer_color
+          end
         else
           @out << text
         end
@@ -130,40 +160,33 @@ module CodeRay
       alias begin_line begin_group
       
       def end_group kind
-        if @opened.empty?
-          # nothing to close
-        else
-          @opened.pop
+        if @opened.pop
+          @color_scopes.pop
           @out << "\e[0m"
-          @out << open_token(@opened.last)
+          if outer_color = @color_scopes.last[:self]
+            @out << outer_color
+          end
         end
       end
       
       def end_line kind
-        if @opened.empty?
-          # nothing to close
-        else
-          @opened.pop
-          # whole lines to be highlighted,
-          # eg. added/modified/deleted lines in a diff
-          @out << (@line_filler ||= "\t" * 100 + "\e[0m")
-          @out << open_token(@opened.last)
-        end
+        @out << (@line_filler ||= "\t" * 100)
+        end_group kind
       end
       
     private
       
       def open_token kind
-        if color = TOKEN_COLORS[kind]
-          if Hash === color
-            @subcolors = color
+        if color = @color_scopes.last[kind]
+          if color.is_a? Hash
+            @color_scopes << color
             color[:self]
           else
-            @subcolors = {}
+            @color_scopes << @color_scopes.last
             color
           end
         else
-          @subcolors = nil
+          @color_scopes << @color_scopes.last
           ''
         end
       end
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 2ad088a..9a94091 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -57,26 +57,25 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .attribute-name { color:#b48 }
 .attribute-value { color:#700 }
 .binary { color:#549 }
-.binary .delimiter { color:#325 }
 .binary .char { color:#325 }
+.binary .delimiter { color:#325 }
+.char { color:#D20 }
 .char .content { color:#D20 }
 .char .delimiter { color:#710 }
-.char { color:#D20 }
 .class { color:#B06; font-weight:bold }
 .class-variable { color:#369 }
 .color { color:#0A0 }
 .comment { color:#777 }
 .comment .char { color:#444 }
 .comment .delimiter { color:#444 }
-.complex { color:#A08 }
 .constant { color:#036; font-weight:bold }
 .decorator { color:#B0B }
 .definition { color:#099; font-weight:bold }
 .delimiter { color:black }
 .directive { color:#088; font-weight:bold }
-.doc { color:#970 }
-.doc-string { color:#D42; font-weight:bold }
+.docstring { color:#D42; }
 .doctype { color:#34b }
+.done { text-decoration: line-through; color: gray }
 .entity { color:#800; font-weight:bold }
 .error { color:#F00; background-color:#FAA }
 .escape  { color:#666 }
@@ -87,16 +86,16 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .global-variable { color:#d70 }
 .hex { color:#02b }
 .id  { color:#33D; font-weight:bold }
-.imaginary { color:#f00 }
 .include { color:#B44; font-weight:bold }
 .inline { background-color: hsla(0,0%,0%,0.07); color: black }
 .inline-delimiter { font-weight: bold; color: #666 }
 .instance-variable { color:#33B }
 .integer  { color:#00D }
+.imaginary { color:#f00 }
 .important { color:#D00 }
+.key { color: #606 }
 .key .char { color: #60f }
 .key .delimiter { color: #404 }
-.key { color: #606 }
 .keyword { color:#080; font-weight:bold }
 .label { color:#970; font-weight:bold }
 .local-variable { color:#963 }
@@ -108,30 +107,30 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .predefined-type { color:#0a5; font-weight:bold }
 .preprocessor { color:#579 }
 .pseudo-class { color:#00C; font-weight:bold }
+.regexp { background-color:hsla(300,100%,50%,0.06); }
 .regexp .content { color:#808 }
 .regexp .delimiter { color:#404 }
 .regexp .modifier { color:#C2C }
-.regexp { background-color:hsla(300,100%,50%,0.06); }
 .reserved { color:#080; font-weight:bold }
+.shell { background-color:hsla(120,100%,50%,0.06); }
 .shell .content { color:#2B2 }
 .shell .delimiter { color:#161 }
-.shell { background-color:hsla(120,100%,50%,0.06); }
+.string { background-color:hsla(0,100%,50%,0.05); }
 .string .char { color: #b0b }
 .string .content { color: #D20 }
 .string .delimiter { color: #710 }
 .string .modifier { color: #E40 }
-.string { background-color:hsla(0,100%,50%,0.05); }
+.symbol { color:#A60 }
 .symbol .content { color:#A60 }
 .symbol .delimiter { color:#630 }
-.symbol { color:#A60 }
 .tag { color:#070 }
 .type { color:#339; font-weight:bold }
-.value { color: #088; }
-.variable  { color:#037 }
+.value { color: #088 }
+.variable { color:#037 }
 
 .insert { background: hsla(120,100%,50%,0.12) }
 .delete { background: hsla(0,100%,50%,0.12) }
-.change { color: #bbf; background: #007; }
+.change { color: #bbf; background: #007 }
 .head { color: #f8f; background: #505 }
 .head .filename { color: white; }
 
@@ -142,8 +141,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .delete .delete { color: #c00; background:transparent; font-weight:bold }
 .change .change { color: #88f }
 .head .head { color: #f4f }
-
-.done { text-decoration: line-through; color: gray }
     TOKENS
 
   end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index de3a0d0..42ea427 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -10,79 +10,77 @@ module CodeRay
   TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
   
   TokenKinds.update(  # :nodoc:
-    :annotation          => 'annotation',
-    :attribute_name      => 'attribute-name',
-    :attribute_value     => 'attribute-value',
-    :binary              => 'bin',
-    :char                => 'char',
-    :class               => 'class',
-    :class_variable      => 'class-variable',
-    :color               => 'color',
-    :comment             => 'comment',
-    :complex             => 'complex',
-    :constant            => 'constant',
-    :content             => 'content',
-    :debug               => 'debug',
-    :decorator           => 'decorator',
-    :definition          => 'definition',
-    :delimiter           => 'delimiter',
-    :directive           => 'directive',
-    :doc                 => 'doc',
-    :doctype             => 'doctype',
-    :docstring           => 'doc-string',
-    :done                => 'done',
-    :entity              => 'entity',
-    :error               => 'error',
-    :escape              => 'escape',
-    :exception           => 'exception',
-    :filename            => 'filename',
-    :float               => 'float',
-    :function            => 'function',
-    :global_variable     => 'global-variable',
-    :hex                 => 'hex',
-    :id                  => 'id',
-    :imaginary           => 'imaginary',
-    :important           => 'important',
-    :include             => 'include',
-    :inline              => 'inline',
-    :inline_delimiter    => 'inline-delimiter',
-    :instance_variable   => 'instance-variable',
-    :integer             => 'integer',
-    :key                 => 'key',
-    :keyword             => 'keyword',
-    :label               => 'label',
-    :local_variable      => 'local-variable',
-    :modifier            => 'modifier',
-    :namespace           => 'namespace',
-    :octal               => 'octal',
-    :predefined          => 'predefined',
-    :predefined_constant => 'predefined-constant',
-    :predefined_type     => 'predefined-type',
-    :preprocessor        => 'preprocessor',
-    :pseudo_class        => 'pseudo-class',
-    :regexp              => 'regexp',
-    :reserved            => 'reserved',
-    :shell               => 'shell',
-    :string              => 'string',
-    :symbol              => 'symbol',
-    :tag                 => 'tag',
-    :type                => 'type',
-    :value               => 'value',
-    :variable            => 'variable',
+    :debug               => 'debug',              # highlight for debugging (white on blue background)
     
-    :change              => 'change',
-    :delete              => 'delete',
-    :head                => 'head',
-    :insert              => 'insert',
+    :annotation          => 'annotation',         # Groovy, Java
+    :attribute_name      => 'attribute-name',     # HTML, CSS
+    :attribute_value     => 'attribute-value',    # HTML
+    :binary              => 'binary',             # Python, Ruby
+    :char                => 'char',               # most scanners, also inside of strings
+    :class               => 'class',              # lots of scanners, for different purposes also in CSS
+    :class_variable      => 'class-variable',     # Ruby, YAML
+    :color               => 'color',              # CSS
+    :comment             => 'comment',            # most scanners
+    :constant            => 'constant',           # PHP, Ruby
+    :content             => 'content',            # inside of strings, most scanners
+    :decorator           => 'decorator',          # Python
+    :definition          => 'definition',         # CSS
+    :delimiter           => 'delimiter',          # inside strings, comments and other types
+    :directive           => 'directive',          # lots of scanners
+    :doctype             => 'doctype',            # Goorvy, HTML, Ruby, YAML
+    :docstring           => 'docstring',          # Python
+    :done                => 'done',               # Taskpaper
+    :entity              => 'entity',             # HTML
+    :error               => 'error',              # invalid token, most scanners
+    :escape              => 'escape',             # Ruby (string inline variables like #$foo, #@bar)
+    :exception           => 'exception',          # Java, PHP, Python
+    :filename            => 'filename',           # Diff
+    :float               => 'float',              # most scanners
+    :function            => 'function',           # CSS, JavaScript, PHP
+    :global_variable     => 'global-variable',    # Ruby, YAML
+    :hex                 => 'hex',                # hexadecimal number; lots of scanners
+    :id                  => 'id',                 # CSS
+    :imaginary           => 'imaginary',          # Python
+    :important           => 'important',          # CSS, Taskpaper
+    :include             => 'include',            # C, Groovy, Java, Python, Sass
+    :inline              => 'inline',             # nested code, eg. inline string evaluation; lots of scanners
+    :inline_delimiter    => 'inline-delimiter',   # used instead of :inline > :delimiter FIXME: Why?
+    :instance_variable   => 'instance-variable',  # Ruby
+    :integer             => 'integer',            # most scanners
+    :key                 => 'key',                # lots of scanners, used together with :value
+    :keyword             => 'keyword',            # reserved word that's actually implemented; most scanners
+    :label               => 'label',              # C, PHP
+    :local_variable      => 'local-variable',     # local and magic variables; some scanners
+    :modifier            => 'modifier',           # used inside on strings; lots of scanners
+    :namespace           => 'namespace',          # Clojure, Java, Taskpaper
+    :octal               => 'octal',              # lots of scanners
+    :predefined          => 'predefined',         # predefined function: lots of scanners
+    :predefined_constant => 'predefined-constant',# lots of scanners
+    :predefined_type     => 'predefined-type',    # C, Java, PHP
+    :preprocessor        => 'preprocessor',       # C, Delphi, HTML
+    :pseudo_class        => 'pseudo-class',       # CSS
+    :regexp              => 'regexp',             # Groovy, JavaScript, Ruby
+    :reserved            => 'reserved',           # most scanners
+    :shell               => 'shell',              # Ruby
+    :string              => 'string',             # most scanners
+    :symbol              => 'symbol',             # Clojure, Ruby, YAML
+    :tag                 => 'tag',                # CSS, HTML
+    :type                => 'type',               # CSS, Java, SQL, YAML
+    :value               => 'value',              # used together with :key; CSS, JSON, YAML
+    :variable            => 'variable',           # Sass, SQL, YAML
     
-    :eyecatcher          => 'eyecatcher',
+    :change              => 'change',             # Diff
+    :delete              => 'delete',             # Diff
+    :head                => 'head',               # Diff, YAML
+    :insert              => 'insert',             # Diff
+    :eyecatcher          => 'eyecatcher',         # Diff
     
-    :ident               => false,
-    :operator            => false,
+    :ident               => false,                # almost all scanners
+    :operator            => false,                # almost all scanners
     
-    :space               => false,
-    :plain               => false
+    :space               => false,                # almost all scanners
+    :plain               => false                 # almost all scanners
   )
   
-  TokenKinds[:method]    = TokenKinds[:function]
+  TokenKinds[:method] = TokenKinds[:function]
 end
-- 
cgit v1.2.1


From df21150ed089f1e3d1d35badde23b2d905a421cd Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Thu, 13 Jun 2013 04:35:39 +0200
Subject: speedup/tweak debug encoder

---
 lib/coderay/encoders/debug.rb | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index c03d3fb..94d52af 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -25,34 +25,37 @@ module Encoders
     
     def text_token text, kind
       raise 'empty token' if $CODERAY_DEBUG && text.empty?
+      
       if kind == :space
         @out << text
       else
-        # TODO: Escape (
-        text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/)
-        @out << kind.to_s << '(' << text << ')'
+        text = text.gsub('\\', '\\\\\\\\') if text.index('\\')
+        text = text.gsub(')',  '\\\\)')    if text.index(')')
+        @out << "#{kind}(#{text})"
       end
     end
     
     def begin_group kind
-      @opened << kind
-      @out << kind.to_s << '<'
+      @opened << kind if $CODERAY_DEBUG
+      
+      @out << "#{kind}<"
     end
     
     def end_group kind
-      if @opened.last != kind
-        puts @out
-        raise "we are inside #{@opened.inspect}, not #{kind}"
-      end
-      @opened.pop
+      raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind
+      
       @out << '>'
     end
     
     def begin_line kind
-      @out << kind.to_s << '['
+      @opened << kind if $CODERAY_DEBUG
+      
+      @out << "#{kind}["
     end
     
     def end_line kind
+      raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind
+      
       @out << ']'
     end
     
-- 
cgit v1.2.1


From 88b812274a808f69675445a9265dab1d4e41cc44 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Thu, 13 Jun 2013 04:46:11 +0200
Subject: refactor check functionality to DebugLint encoder

---
 lib/coderay/encoders/debug.rb      | 15 -----------
 lib/coderay/encoders/debug_lint.rb | 55 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 15 deletions(-)
 create mode 100644 lib/coderay/encoders/debug_lint.rb

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 94d52af..61520a1 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -18,14 +18,7 @@ module Encoders
     
     FILE_EXTENSION = 'raydebug'
     
-    def initialize options = {}
-      super
-      @opened = []
-    end
-    
     def text_token text, kind
-      raise 'empty token' if $CODERAY_DEBUG && text.empty?
-      
       if kind == :space
         @out << text
       else
@@ -36,26 +29,18 @@ module Encoders
     end
     
     def begin_group kind
-      @opened << kind if $CODERAY_DEBUG
-      
       @out << "#{kind}<"
     end
     
     def end_group kind
-      raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind
-      
       @out << '>'
     end
     
     def begin_line kind
-      @opened << kind if $CODERAY_DEBUG
-      
       @out << "#{kind}["
     end
     
     def end_line kind
-      raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind
-      
       @out << ']'
     end
     
diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
new file mode 100644
index 0000000..eeb2a92
--- /dev/null
+++ b/lib/coderay/encoders/debug_lint.rb
@@ -0,0 +1,55 @@
+module CodeRay
+module Encoders
+  
+  # = Debug Lint Encoder
+  #
+  # Debug encoder with additional checks for:
+  # 
+  # - empty tokens
+  # - incorrect nesting
+  # 
+  # It will raise an InvalidTokenStream exception when any of the above occurs.
+  # 
+  # See also: Encoders::Debug
+  class DebugLint < Debug
+    
+    register_for :debug_lint
+    
+    InvalidTokenStream = Class.new StandardError
+    EmptyToken = Class.new InvalidTokenStream
+    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
+    
+    def initialize options = {}
+      super
+      @opened = []
+    end
+    
+    def text_token text, kind
+      raise EmptyToken, 'empty token' if text.empty?
+      super
+    end
+    
+    def begin_group kind
+      @opened << kind
+      super
+    end
+    
+    def end_group kind
+      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      super
+    end
+    
+    def begin_line kind
+      @opened << kind
+      super
+    end
+    
+    def end_line kind
+      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      super
+    end
+    
+  end
+  
+end
+end
-- 
cgit v1.2.1


From e423275749744fb2cf768087a07b4c839eaf4734 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Thu, 13 Jun 2013 05:33:14 +0200
Subject: remove dump/undump functionality

---
 lib/coderay/encoders/debug.rb |  1 -
 lib/coderay/helpers/gzip.rb   | 41 ------------------------------------
 lib/coderay/tokens.rb         | 48 +------------------------------------------
 3 files changed, 1 insertion(+), 89 deletions(-)
 delete mode 100644 lib/coderay/helpers/gzip.rb

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 61520a1..f4db330 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -9,7 +9,6 @@ module Encoders
   #
   # You cannot fully restore the tokens information from the
   # output, because consecutive :space tokens are merged.
-  # Use Tokens#dump for caching purposes.
   # 
   # See also: Scanners::Debug
   class Debug < Encoder
diff --git a/lib/coderay/helpers/gzip.rb b/lib/coderay/helpers/gzip.rb
deleted file mode 100644
index 245014a..0000000
--- a/lib/coderay/helpers/gzip.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-module CodeRay
-  
-  # A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
-  module GZip
-    
-    require 'zlib'
-    
-    # The default zipping level. 7 zips good and fast.
-    DEFAULT_GZIP_LEVEL = 7
-    
-    # Unzips the given string +s+.
-    #
-    # Example:
-    #   require 'gzip_simple'
-    #   print GZip.gunzip(File.read('adresses.gz'))
-    def GZip.gunzip s
-      Zlib::Inflate.inflate s
-    end
-    
-    # Zips the given string +s+.
-    #
-    # Example:
-    #   require 'gzip_simple'
-    #   File.open('adresses.gz', 'w') do |file
-    #     file.write GZip.gzip('Mum: 0123 456 789', 9)
-    #   end
-    #
-    # If you provide a +level+, you can control how strong
-    # the string is compressed:
-    # - 0: no compression, only convert to gzip format
-    # - 1: compress fast
-    # - 7: compress more, but still fast (default)
-    # - 8: compress more, slower
-    # - 9: compress best, very slow
-    def GZip.gzip s, level = DEFAULT_GZIP_LEVEL
-      Zlib::Deflate.new(level).deflate s, Zlib::FINISH
-    end
-    
-  end
-  
-end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 6957d69..9142b76 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -1,8 +1,5 @@
 module CodeRay
   
-  # GZip library for writing and reading token dumps.
-  autoload :GZip, coderay_path('helpers', 'gzip')
-  
   # = Tokens  TODO: Rewrite!
   #
   # The Tokens class represents a list of tokens returnd from
@@ -45,8 +42,7 @@ module CodeRay
   # See how small it is? ;)
   #
   # Tokens gives you the power to handle pre-scanned code very easily:
-  # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
-  # that you put in your DB.
+  # You can convert it to a webpage, a YAML file, or a .raydebug representation.
   # 
   # It also allows you to generate tokens directly (without using a scanner),
   # to load them from a file, and still use any Encoder that CodeRay provides.
@@ -157,53 +153,11 @@ module CodeRay
       parts
     end
     
-    # Dumps the object into a String that can be saved
-    # in files or databases.
-    #
-    # The dump is created with Marshal.dump;
-    # In addition, it is gzipped using GZip.gzip.
-    #
-    # The returned String object includes Undumping
-    # so it has an #undump method. See Tokens.load.
-    #
-    # You can configure the level of compression,
-    # but the default value 7 should be what you want
-    # in most cases as it is a good compromise between
-    # speed and compression rate.
-    #
-    # See GZip module.
-    def dump gzip_level = 7
-      dump = Marshal.dump self
-      dump = GZip.gzip dump, gzip_level
-      dump.extend Undumping
-    end
-    
     # Return the actual number of tokens.
     def count
       size / 2
     end
     
-    # Include this module to give an object an #undump
-    # method.
-    #
-    # The string returned by Tokens.dump includes Undumping.
-    module Undumping
-      # Calls Tokens.load with itself.
-      def undump
-        Tokens.load self
-      end
-    end
-    
-    # Undump the object using Marshal.load, then
-    # unzip it using GZip.gunzip.
-    #
-    # The result is commonly a Tokens object, but
-    # this is not guaranteed.
-    def Tokens.load dump
-      dump = GZip.gunzip dump
-      @dump = Marshal.load dump
-    end
-    
     alias text_token push
     def begin_group kind; push :begin_group, kind end
     def end_group kind; push :end_group, kind end
-- 
cgit v1.2.1


From cb41b00d5673312f4982e914883f9cea95f2ccae Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 16 Jun 2013 01:01:12 +0200
Subject: cleanup TODOs, FIXMEs

---
 lib/coderay/encoders/html/numbering.rb | 2 +-
 lib/coderay/encoders/statistic.rb      | 1 -
 lib/coderay/token_kinds.rb             | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb
index 332145b..5908bf0 100644
--- a/lib/coderay/encoders/html/numbering.rb
+++ b/lib/coderay/encoders/html/numbering.rb
@@ -75,7 +75,7 @@ module Encoders
           line_number = start
           output.gsub!(/^.*$\n?/) do |line|
             line_number_text = bolding.call line_number
-            indent = ' ' * (max_width - line_number.to_s.size)  # TODO: Optimize (10^x)
+            indent = ' ' * (max_width - line_number.to_s.size)
             line_number += 1
             "<span class=\"line-numbers\">#{indent}#{line_number_text}</span>#{line}"
           end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index 2315d9e..b2f8b83 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -67,7 +67,6 @@ Token Types (%d):
       @type_stats['TOTAL'].count += 1
     end
     
-    # TODO Hierarchy handling
     def begin_group kind
       block_token ':begin_group', kind
     end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 42ea427..9154658 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -44,7 +44,7 @@ module CodeRay
     :important           => 'important',          # CSS, Taskpaper
     :include             => 'include',            # C, Groovy, Java, Python, Sass
     :inline              => 'inline',             # nested code, eg. inline string evaluation; lots of scanners
-    :inline_delimiter    => 'inline-delimiter',   # used instead of :inline > :delimiter FIXME: Why?
+    :inline_delimiter    => 'inline-delimiter',   # used instead of :inline > :delimiter FIXME: Why use inline_delimiter?
     :instance_variable   => 'instance-variable',  # Ruby
     :integer             => 'integer',            # most scanners
     :key                 => 'key',                # lots of scanners, used together with :value
-- 
cgit v1.2.1


From 462302814dc56c33f26f43516d5ddfc345dcd2a3 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 16 Jun 2013 01:01:30 +0200
Subject: cleanup Tokens documentation

---
 lib/coderay/tokens.rb | 37 ++++++++++++++-----------------------
 1 file changed, 14 insertions(+), 23 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 6957d69..ad59b7f 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -3,25 +3,23 @@ module CodeRay
   # GZip library for writing and reading token dumps.
   autoload :GZip, coderay_path('helpers', 'gzip')
   
-  # = Tokens  TODO: Rewrite!
+  # The Tokens class represents a list of tokens returned from
+  # a Scanner. It's actually just an Array with a few helper methods.
   #
-  # The Tokens class represents a list of tokens returnd from
-  # a Scanner.
-  #
-  # A token is not a special object, just a two-element Array
+  # A token itself is not a special object, just a two-element Array
   # consisting of
   # * the _token_ _text_ (the original source of the token in a String) or
   #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
   #
-  # A token looks like this:
+  # It looks like this:
   #
   #   ['# It looks like this', :comment]
   #   ['3.1415926', :float]
   #   ['$^', :error]
   #
   # Some scanners also yield sub-tokens, represented by special
-  # token actions, namely begin_group and end_group.
+  # token actions, for example :begin_group and :end_group.
   #
   # The Ruby scanner, for example, splits "a string" into:
   #
@@ -33,23 +31,17 @@ module CodeRay
   #   [:end_group, :string]
   #  ]
   #
-  # Tokens is the interface between Scanners and Encoders:
-  # The input is split and saved into a Tokens object. The Encoder
-  # then builds the output from this object.
-  #
-  # Thus, the syntax below becomes clear:
-  #
-  #   CodeRay.scan('price = 2.59', :ruby).html
-  #   # the Tokens object is here -------^
+  # Tokens can be used to save the output of a Scanners in a simple
+  # Ruby object that can be send to an Encoder later:
   #
-  # See how small it is? ;)
+  #   tokens = CodeRay.scan('price = 2.59', :ruby).tokens
+  #   tokens.encode(:html)
+  #   tokens.html
+  #   CodeRay.encoder(:html).encode_tokens(tokens)
   #
   # Tokens gives you the power to handle pre-scanned code very easily:
-  # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
-  # that you put in your DB.
-  # 
-  # It also allows you to generate tokens directly (without using a scanner),
-  # to load them from a file, and still use any Encoder that CodeRay provides.
+  # You can serialize it to a JSON string and store it in a database, pass it
+  # around to encode it more than once, send it to other algorithms...
   class Tokens < Array
     
     # The Scanner instance that created the tokens.
@@ -58,8 +50,7 @@ module CodeRay
     # Encode the tokens using encoder.
     #
     # encoder can be
-    # * a symbol like :html oder :statistic
-    # * an Encoder class
+    # * a plugin name like :html oder 'statistic'
     # * an Encoder object
     #
     # options are passed to the encoder.
-- 
cgit v1.2.1


From dc9528456008216c265cfb6613cfa459c3c9ffad Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 16 Jun 2013 01:01:48 +0200
Subject: speedup HTML encoder numbering for Ruby 1.8.7+

---
 lib/coderay/encoders/html/numbering.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb
index 5908bf0..a1b9c04 100644
--- a/lib/coderay/encoders/html/numbering.rb
+++ b/lib/coderay/encoders/html/numbering.rb
@@ -26,7 +26,7 @@ module Encoders
               "<a href=\"##{anchor}\" name=\"#{anchor}\">#{line}</a>"
             end
           else
-            proc { |line| line.to_s }  # :to_s.to_proc in Ruby 1.8.7+
+            :to_s.to_proc
           end
         
         bold_every = options[:bold_every]
-- 
cgit v1.2.1


From fc21c91d7a89d98785e494ffd41e9f3adbdfed81 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Wed, 19 Jun 2013 11:50:04 +0200
Subject: use different key/value heuristic in JSON scanner

---
 lib/coderay/scanners/json.rb | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 4e0f462..3754a9b 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -14,15 +14,17 @@ module Scanners
     
     ESCAPE = / [bfnrt\\"\/] /x  # :nodoc:
     UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x  # :nodoc:
+    KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /x
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     # See http://json.org/ for a definition of the JSON lexic/grammar.
     def scan_tokens encoder, options
-      
-      state = :initial
-      stack = []
-      key_expected = false
+      state = options[:state] || @state
       
       until eos?
         
@@ -32,18 +34,11 @@ module Scanners
           if match = scan(/ \s+ /x)
             encoder.text_token match, :space
           elsif match = scan(/"/)
-            state = key_expected ? :key : :string
+            state = check(/#{KEY}/o) ? :key : :string
             encoder.begin_group state
             encoder.text_token match, :delimiter
           elsif match = scan(/ [:,\[{\]}] /x)
             encoder.text_token match, :operator
-            case match
-            when ':' then key_expected = false
-            when ',' then key_expected = true if stack.last == :object
-            when '{' then stack << :object; key_expected = true
-            when '[' then stack << :array
-            when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
-            end
           elsif match = scan(/ true | false | null /x)
             encoder.text_token match, :value
           elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x)
@@ -82,6 +77,10 @@ module Scanners
         end
       end
       
+      if options[:keep_state]
+        @state = state
+      end
+      
       if [:string, :key].include? state
         encoder.end_group state
       end
-- 
cgit v1.2.1


From 5d1812b9d0e1074eb9ec7b24d6ce0f4f7f80619a Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 22 Jun 2013 23:42:30 +0200
Subject: tags should be bold

---
 lib/coderay/styles/alpha.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 9a94091..f4d07f1 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -123,7 +123,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .symbol { color:#A60 }
 .symbol .content { color:#A60 }
 .symbol .delimiter { color:#630 }
-.tag { color:#070 }
+.tag { color:#070; font-weight:bold }
 .type { color:#339; font-weight:bold }
 .value { color: #088 }
 .variable { color:#037 }
-- 
cgit v1.2.1


From 2abfc49bdc9a9f4e86c90aa968c302ca76c20812 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 22 Jun 2013 23:43:06 +0200
Subject: use JSON to highlight .template files (AWS CF)

---
 lib/coderay/helpers/file_type.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 6144e8c..19f27ac 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -120,6 +120,7 @@ module CodeRay
       'sass'      => :sass,
       'sql'       => :sql,
       'taskpaper' => :taskpaper,
+      'template'  => :json,  # AWS CloudFormation template
       'tmproj'    => :xml,
       'xaml'      => :xml,
       'xhtml'     => :html,
-- 
cgit v1.2.1


From d0d1b129f09dbae8fc25c093c09f72c6396b122a Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sat, 22 Jun 2013 23:53:32 +0200
Subject: whitespace

---
 lib/coderay/styles/alpha.rb | 2 +-
 lib/coderay/token_kinds.rb  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index c05ffd5..c231423 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -1,6 +1,6 @@
 module CodeRay
 module Styles
-
+  
   # A colorful theme using CSS 3 colors (with alpha channel).
   class Alpha < Style
 
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index f1696df..9137a49 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -1,14 +1,14 @@
 module CodeRay
-
+  
   # A Hash of all known token kinds and their associated CSS classes.
   TokenKinds = Hash.new do |h, k|
     warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
     false
   end
-
+  
   # speedup
   TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
-
+  
   TokenKinds.update(  # :nodoc:
     :debug               => 'debug',              # highlight for debugging (white on blue background)
     
-- 
cgit v1.2.1


From 90adb394e8b4da3a733941fc764c308b34482293 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 00:15:59 +0200
Subject: add lua file type

---
 lib/coderay/helpers/file_type.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib/coderay')

diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 19f27ac..6d4fa92 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -96,6 +96,7 @@ module CodeRay
       'java'      => :java,
       'js'        => :java_script,
       'json'      => :json,
+      'lua'       => :lua,
       'mab'       => :ruby,
       'pas'       => :delphi,
       'patch'     => :diff,
-- 
cgit v1.2.1


From fbaf55d69c20897655c80e9723973e2f9490484d Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:04:53 +0200
Subject: more helpful error messages from DebugLint

---
 lib/coderay/encoders/debug_lint.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
index eeb2a92..0ac89ef 100644
--- a/lib/coderay/encoders/debug_lint.rb
+++ b/lib/coderay/encoders/debug_lint.rb
@@ -35,7 +35,7 @@ module Encoders
     end
     
     def end_group kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind
       super
     end
     
@@ -45,7 +45,7 @@ module Encoders
     end
     
     def end_line kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind
       super
     end
     
-- 
cgit v1.2.1


From c0dc9f2b82b84e6a7511ff9bfe3e80aaa1fef1d1 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:05:41 +0200
Subject: move .map styles before .string

---
 lib/coderay/styles/alpha.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index c231423..a869d9e 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -99,6 +99,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .keyword { color:#080; font-weight:bold }
 .label { color:#970; font-weight:bold }
 .local-variable { color:#963 }
+.map .content { color:#808 }
+.map .delimiter { color:#40A}
+.map { background-color:hsla(200,100%,50%,0.06); }
 .namespace { color:#707; font-weight:bold }
 .octal { color:#40E }
 .operator { }
@@ -123,9 +126,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .symbol { color:#A60 }
 .symbol .content { color:#A60 }
 .symbol .delimiter { color:#630 }
-.map .content { color:#808 }
-.map .delimiter { color:#40A}
-.map { background-color:hsla(200,100%,50%,0.06); }
 .tag { color:#070; font-weight:bold }
 .type { color:#339; font-weight:bold }
 .value { color: #088 }
-- 
cgit v1.2.1


From da1425058e9b9e04bd988ddf606331d9cd0c827b Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:06:33 +0200
Subject: cleanup Lua scanner, fix end_group(:map) issue - use local instead of
 instance variables

---
 lib/coderay/scanners/lua.rb | 524 ++++++++++++++++++++++----------------------
 1 file changed, 267 insertions(+), 257 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 64763dc..3bee275 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -1,265 +1,275 @@
-# -*- coding: utf-8 -*-
+# encoding: utf-8
 
-# Scanner for the Lua[http://lua.org] programming lanuage.
-#
-# The language’s complete syntax is defined in
-# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
-# which is what this scanner tries to conform to.
-class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+module CodeRay
+module Scanners
 
-  register_for :lua
-  file_extension "lua"
-  title "Lua"
-
-  # Keywords used in Lua.
-  KEYWORDS = %w[and break do else elseif end
-  for function goto if in
-  local not or repeat return
-  then until while
-  ]
-
-  # Constants set by the Lua core.
-  PREDEFINED_CONSTANTS = %w[false true nil]
-
-  # The expressions contained in this array are parts of Lua’s `basic'
-  # library. Although it’s not entirely necessary to load that library,
-  # it is highly recommended and one would have to provide own implementations
-  # of some of these expressions if one does not do so. They however aren’t
-  # keywords, neither are they constants, but nearly predefined, so they
-  # get tagged as `predefined' rather than anything else.
+  # Scanner for the Lua[http://lua.org] programming lanuage.
   #
-  # This list excludes values of form `_UPPERCASE' because the Lua manual
-  # requires such identifiers to be reserved by Lua anyway and they are
-  # highlighted directly accordingly, without the need for specific
-  # identifiers to be listed here.
-  PREDEFINED_EXPRESSIONS = %w[
-  assert collectgarbage dofile error getmetatable
-  ipairs load loadfile next pairs pcall print
-  rawequal rawget rawlen rawset select setmetatable
-  tonumber tostring type xpcall
-  ]
-
-  # Automatic token kind selection for normal words.
-  IDENT_KIND = CodeRay::WordList.new(:ident).
-    add(KEYWORDS, :keyword).
-    add(PREDEFINED_CONSTANTS, :predefined_constant).
-    add(PREDEFINED_EXPRESSIONS, :predefined)
-
-  protected
-
-  # Scanner initialization.
-  def setup
-    @state       = :initial
-    @brace_depth = 0
-  end
-
-  # CodeRay entry hook. Starts parsing.
-  def scan_tokens(encoder, options)
-    @encoder = encoder
-    @options = options
-
-    until eos?
-  case state
-
-  when :initial
-    if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
-      @num_equals = match.count("=") # Number must match for comment end
-      @encoder.begin_group(:comment)
-      @encoder.text_token(match, :delimiter)
-      @state = :long_comment
-
-    elsif match = scan(/--.*?$/) # --Lua comment
-      @encoder.text_token(match, :comment)
-
-    elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
-      @num_equals = match.count("=") # Number must match for comment end
-      @encoder.begin_group(:string)
-      @encoder.text_token(match, :delimiter)
-      @state = :long_string
-
-    elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label::
-      @encoder.text_token(match, :label)
-
-    elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua
-      @encoder.text_token(match, :predefined)
-
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits)
-      kind = IDENT_KIND[match]
-
-      # Extra highlighting for entities following certain keywords
-      if kind == :keyword and match == "function"
-        @state = :function_expected
-      elsif kind == :keyword and match == "goto"
-        @state = :goto_label_expected
-      elsif kind == :keyword and match == "local"
-        @state = :local_var_expected
+  # The language’s complete syntax is defined in
+  # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
+  # which is what this scanner tries to conform to.
+  class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+    
+    register_for :lua
+    file_extension "lua"
+    title "Lua"
+    
+    # Keywords used in Lua.
+    KEYWORDS = %w[and break do else elseif end
+    for function goto if in
+    local not or repeat return
+    then until while
+    ]
+    
+    # Constants set by the Lua core.
+    PREDEFINED_CONSTANTS = %w[false true nil]
+    
+    # The expressions contained in this array are parts of Lua’s `basic'
+    # library. Although it’s not entirely necessary to load that library,
+    # it is highly recommended and one would have to provide own implementations
+    # of some of these expressions if one does not do so. They however aren’t
+    # keywords, neither are they constants, but nearly predefined, so they
+    # get tagged as `predefined' rather than anything else.
+    #
+    # This list excludes values of form `_UPPERCASE' because the Lua manual
+    # requires such identifiers to be reserved by Lua anyway and they are
+    # highlighted directly accordingly, without the need for specific
+    # identifiers to be listed here.
+    PREDEFINED_EXPRESSIONS = %w[
+    assert collectgarbage dofile error getmetatable
+    ipairs load loadfile next pairs pcall print
+    rawequal rawget rawlen rawset select setmetatable
+    tonumber tostring type xpcall
+    ]
+    
+    # Automatic token kind selection for normal words.
+    IDENT_KIND = CodeRay::WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
+      add(PREDEFINED_EXPRESSIONS, :predefined)
+    
+    protected
+    
+    # Scanner initialization.
+    def setup
+      @state       = :initial
+      @brace_depth = 0
+    end
+    
+    # CodeRay entry hook. Starts parsing.
+    def scan_tokens(encoder, options)
+      state = options[:state] || @state
+      
+      until eos?
+        case state
+        
+        when :initial
+          if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
+            @num_equals = match.count("=") # Number must match for comment end
+            encoder.begin_group(:comment)
+            encoder.text_token(match, :delimiter)
+            state = :long_comment
+          
+          elsif match = scan(/--.*$/) # --Lua comment
+            encoder.text_token(match, :comment)
+          
+          elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
+            @num_equals = match.count("=") # Number must match for comment end
+            encoder.begin_group(:string)
+            encoder.text_token(match, :delimiter)
+            state = :long_string
+          
+          elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label::
+            encoder.text_token(match, :label)
+          
+          elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua
+            encoder.text_token(match, :predefined)
+          
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits)
+            kind = IDENT_KIND[match]
+            
+            # Extra highlighting for entities following certain keywords
+            if kind == :keyword and match == "function"
+              state = :function_expected
+            elsif kind == :keyword and match == "goto"
+              state = :goto_label_expected
+            elsif kind == :keyword and match == "local"
+              state = :local_var_expected
+            end
+            
+            encoder.text_token(match, kind)
+          
+          elsif match = scan(/\{/) # Opening table brace {
+            encoder.begin_group(:map)
+            encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
+            @brace_depth += 1
+            state        = :map
+          
+          elsif match = scan(/\}/) # Closing table brace }
+            if @brace_depth == 1
+              @brace_depth = 0
+              encoder.text_token(match, :delimiter)
+              encoder.end_group(:map)
+            elsif @brace_depth == 0 # Mismatched brace
+              encoder.text_token(match, :error)
+            else
+              @brace_depth -= 1
+              encoder.text_token(match, :inline_delimiter)
+              encoder.end_group(:map)
+              state = :map
+            end
+          
+          elsif match = scan(/["']/) # String delimiters " and '
+            encoder.begin_group(:string)
+            encoder.text_token(match, :delimiter)
+            @start_delim = match
+            state       = :string
+          
+                            # ↓Prefix                hex number ←|→ decimal number
+          elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+            encoder.text_token(match, :float)
+          
+                            # ↓Prefix         hex number ←|→ decimal number
+          elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+            encoder.text_token(match, :integer)
+          
+          elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators
+            encoder.text_token(match, :operator)
+          
+          elsif match = scan(/\s+/) # Space
+            encoder.text_token(match, :space)
+          
+          else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors.
+            encoder.text_token(getch, :error)
+          end
+          
+          # It may be that we’re scanning a full-blown subexpression of a table
+          # (tables can contain full expressions in parts).
+          # If this is the case, return to :map scanning state.
+          state = :map if state == :initial && @brace_depth >= 1
+        
+        when :function_expected
+          if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
+            encoder.text_token(match, :operator)
+            state = :initial
+          elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator
+            encoder.text_token(match, :ident)
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo()
+            encoder.text_token(match, :function)
+            state = :initial
+          elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+            state = :initial
+          end
+        
+        when :goto_label_expected
+          if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+            encoder.text_token(match, :label)
+            state = :initial
+          elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :local_var_expected
+          if match = scan(/function/) # local function ...
+            encoder.text_token(match, :keyword)
+            state = :function_expected
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+            encoder.text_token(match, :local_variable)
+          elsif match = scan(/,/)
+            encoder.text_token(match, :operator)
+          elsif match = scan(/\=/)
+            encoder.text_token(match, :operator)
+            # After encountering the equal sign, arbitrary expressions are
+            # allowed again, so just return to the main state for further
+            # parsing.
+            state = :initial
+          elsif match = scan(/\n/)
+            encoder.text_token(match, :space)
+            state = :initial
+          elsif match = scan(/\s+/)
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :long_comment
+          if match = scan(/.*?(?=\]={#@num_equals}\])/m)
+            encoder.text_token(match, :content)
+            
+            delim = scan(/\]={#@num_equals}\]/)
+            encoder.text_token(delim, :delimiter)
+          else # No terminator found till EOF
+            encoder.text_token(rest, :error)
+            terminate
+          end
+          encoder.end_group(:comment)
+          state = :initial
+        
+        when :long_string
+          if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
+            encoder.text_token(match, :content)
+            
+            delim = scan(/\]={#@num_equals}\]/)
+            encoder.text_token(delim, :delimiter)
+          else # No terminator found till EOF
+            encoder.text_token(rest, :error)
+            terminate
+          end
+          encoder.end_group(:string)
+          state = :initial
+        
+        when :string
+          if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+            encoder.text_token(match, :content)
+          elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
+            encoder.text_token(match, :char)
+          elsif match = scan(Regexp.compile(@start_delim))
+            encoder.text_token(match, :delimiter)
+            encoder.end_group(:string)
+            state = :initial
+          elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings
+            encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings
+            encoder.end_group(:string)
+            state = :initial
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :map
+          if match = scan(/[,;]/)
+            encoder.text_token(match, :operator)
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
+            encoder.text_token(match, :key)
+            encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
+            encoder.text_token(scan(/\=/), :operator)
+            state = :initial
+          elsif match = scan(/\s+/m)
+            encoder.text_token(match, :space)
+          else
+            # Note this clause doesn’t advance the scan pointer, it’s a kind of
+            # "retry with other options" (the :initial state then of course
+            # advances the pointer).
+            state = :initial
+          end
+        else
+          raise
+        end
+        
       end
-
-      @encoder.text_token(match, kind)
-
-    elsif match = scan(/\{/) # Opening table brace {
-      @encoder.begin_group(:map)
-      @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
-      @brace_depth += 1
-      @state        = :map
-
-    elsif match = scan(/\}/) # Closing table brace }
-      if @brace_depth == 1
-        @brace_depth = 0
-        @encoder.text_token(match, :delimiter)
-      elsif @brace_depth == 0 # Mismatched brace
-        @encoder.text_token(match, :error)
-      else
-        @brace_depth -= 1
-        @encoder.text_token(match, :inline_delimiter)
-        @state = :map
+      
+      if options[:keep_state]
+        @state = state
       end
-      @encoder.end_group(:map)
-
-    elsif match = scan(/["']/) # String delimiters " and '
-      @encoder.begin_group(:string)
-      @encoder.text_token(match, :delimiter)
-      @start_delim = match
-      @state       = :string
-
-                      # ↓Prefix                hex number ←|→ decimal number
-    elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
-      @encoder.text_token(match, :float)
-
-                      # ↓Prefix         hex number ←|→ decimal number
-    elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
-      @encoder.text_token(match, :integer)
-
-    elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators
-      @encoder.text_token(match, :operator)
-
-    elsif match = scan(/\s+/) # Space
-      @encoder.text_token(match, :space)
-
-    else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors.
-      @encoder.text_token(getch, :error)
-    end
-
-    # It may be that we’re scanning a full-blown subexpression of a table
-    # (tables can contain full expressions in parts).
-    # If this is the case, return to :map scanning state.
-    @state = :map if @state == :initial && @brace_depth >= 1
-
-  when :function_expected
-    if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
-      @encoder.text_token(match, :operator)
-      @state = :initial
-    elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator
-      @encoder.text_token(match, :ident)
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo()
-      @encoder.text_token(match, :function)
-      @state = :initial
-    elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace
-      @encoder.text_token(match, :space)
-    else
-      @encoder.text_token(getch, :error)
-      @state = :initial
-    end
-
-  when :goto_label_expected
-    if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
-      @encoder.text_token(match, :label)
-      @state = :initial
-    elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace
-      @encoder.text_token(match, :space)
-    else
-      @encoder.text_token(getch, :error)
-    end
-
-  when :local_var_expected
-    if match = scan(/function/) # local function ...
-      @encoder.text_token(match, :keyword)
-      @state = :function_expected
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
-      @encoder.text_token(match, :local_variable)
-    elsif match = scan(/,/)
-      @encoder.text_token(match, :operator)
-    elsif match = scan(/\=/)
-      @encoder.text_token(match, :operator)
-      # After encountering the equal sign, arbitrary expressions are
-      # allowed again, so just return to the main state for further
-      # parsing.
-      @state = :initial
-    elsif match = scan(/\n/)
-      @encoder.text_token(match, :space)
-      @state = :initial
-    elsif match = scan(/\s+/)
-      @encoder.text_token(match, :space)
-    else
-      @encoder.text_token(getch, :error)
-    end
-
-  when :long_comment
-    if match = scan(/.*?(?=\]={#@num_equals}\])/m)
-      @encoder.text_token(match, :content)
-
-      delim = scan(/\]={#@num_equals}\]/)
-      @encoder.text_token(delim, :delimiter)
-    else # No terminator found till EOF
-      @encoder.text_token(rest, :error)
-      terminate
-    end
-    @encoder.end_group(:comment)
-    @state = :initial
-
-  when :long_string
-    if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
-      @encoder.text_token(match, :content)
-
-      delim = scan(/\]={#@num_equals}\]/)
-      @encoder.text_token(delim, :delimiter)
-    else # No terminator found till EOF
-      @encoder.text_token(rest, :error)
-      terminate
-    end
-    @encoder.end_group(:string)
-    @state = :initial
-
-  when :string
-    if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
-      @encoder.text_token(match, :content)
-    elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
-      @encoder.text_token(match, :char)
-    elsif match = scan(Regexp.compile(@start_delim))
-      @encoder.text_token(match, :delimiter)
-      @encoder.end_group(:string)
-      @state = :initial
-    elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings
-      @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings
-      @encoder.end_group(:string)
-      @state = :initial
-    else
-      @encoder.text_token(getch, :error)
-    end
-
-  when :map
-    if match = scan(/[,;]/)
-      @encoder.text_token(match, :operator)
-    elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
-      @encoder.text_token(match, :key)
-      @encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
-      @encoder.text_token(scan(/\=/), :operator)
-      @state = :initial
-    elsif match = scan(/\s+/m)
-      @encoder.text_token(match, :space)
-    else
-      # Note this clause doesn’t advance the scan pointer, it’s a kind of
-      # "retry with other options" (the :initial state then of course
-      # advances the pointer).
-      @state = :initial
+      
+      encoder
     end
-  else
-    raise
+    
   end
-
-    end
-
-    @encoder
-  end
-
+  
+end
 end
-- 
cgit v1.2.1


From 546b489e4b3856d361a480de5f1f02cb55c7f002 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:09:14 +0200
Subject: fix Tokens documentation

---
 lib/coderay/tokens.rb | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 54358d4..e7bffce 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -3,17 +3,16 @@ module CodeRay
   # The Tokens class represents a list of tokens returned from
   # a Scanner. It's actually just an Array with a few helper methods.
   #
-  # A token itself is not a special object, just a two-element Array
-  # consisting of
+  # A token itself is not a special object, just two elements in an Array:
   # * the _token_ _text_ (the original source of the token in a String) or
   #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
   #
   # It looks like this:
   #
-  #   ['# It looks like this', :comment]
-  #   ['3.1415926', :float]
-  #   ['$^', :error]
+  #   ..., '# It looks like this', :comment, ...
+  #   ..., '3.1415926', :float, ...
+  #   ..., '$^', :error, ...
   #
   # Some scanners also yield sub-tokens, represented by special
   # token actions, for example :begin_group and :end_group.
@@ -21,11 +20,11 @@ module CodeRay
   # The Ruby scanner, for example, splits "a string" into:
   #
   #  [
-  #   [:begin_group, :string],
-  #   ['"', :delimiter],
-  #   ['a string', :content],
-  #   ['"', :delimiter],
-  #   [:end_group, :string]
+  #   :begin_group, :string,
+  #   '"',          :delimiter,
+  #   'a string',   :content,
+  #   '"',          :delimiter,
+  #   :end_group,   :string
   #  ]
   #
   # Tokens can be used to save the output of a Scanners in a simple
-- 
cgit v1.2.1


From 90c401c9001e0a670360a55d1189e9f814ae7592 Mon Sep 17 00:00:00 2001
From: Kornelius Kalnbach <murphy@rubychan.de>
Date: Sun, 23 Jun 2013 01:17:49 +0200
Subject: some more code cleanups before merge

---
 lib/coderay/scanners/lua.rb | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

(limited to 'lib/coderay')

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
index 3bee275..25bebbe 100644
--- a/lib/coderay/scanners/lua.rb
+++ b/lib/coderay/scanners/lua.rb
@@ -8,17 +8,17 @@ module Scanners
   # The language’s complete syntax is defined in
   # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
   # which is what this scanner tries to conform to.
-  class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner
+  class Lua < Scanner
     
     register_for :lua
-    file_extension "lua"
-    title "Lua"
+    file_extension 'lua'
+    title 'Lua'
     
     # Keywords used in Lua.
     KEYWORDS = %w[and break do else elseif end
-    for function goto if in
-    local not or repeat return
-    then until while
+      for function goto if in
+      local not or repeat return
+      then until while
     ]
     
     # Constants set by the Lua core.
@@ -36,10 +36,10 @@ module Scanners
     # highlighted directly accordingly, without the need for specific
     # identifiers to be listed here.
     PREDEFINED_EXPRESSIONS = %w[
-    assert collectgarbage dofile error getmetatable
-    ipairs load loadfile next pairs pcall print
-    rawequal rawget rawlen rawset select setmetatable
-    tonumber tostring type xpcall
+      assert collectgarbage dofile error getmetatable
+      ipairs load loadfile next pairs pcall print
+      rawequal rawget rawlen rawset select setmetatable
+      tonumber tostring type xpcall
     ]
     
     # Automatic token kind selection for normal words.
@@ -52,7 +52,7 @@ module Scanners
     
     # Scanner initialization.
     def setup
-      @state       = :initial
+      @state = :initial
       @brace_depth = 0
     end
     
-- 
cgit v1.2.1