New: *JSON* (closes #53). Version 0.8 is near!

* Simple and really nice scanner (I think), produces colorful output.
* Checked against strange examples from Ruby's JSON lib.

More changes:
* Changed version number. Finally!
* Added some token styles for :key token group.
* cYcnus style: chars inside of strings are highlighted purple instead of blue.
* murphy style needs work.

1 files changed, 106 insertions, 0 deletions

diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
new file mode 100644
index 0000000..ae941a0
--- /dev/null
+++ b/lib/coderay/scanners/json.rb
@@ -0,0 +1,106 @@
+module CodeRay
+module Scanners
+  
+  class JSON < Scanner
+    
+    include Streamable
+    
+    register_for :json
+    
+    CONSTANTS = %w( true false null )
+    IDENT_KIND = WordList.new(:key).add(CONSTANTS, :reserved)
+    
+    ESCAPE = / [bfnrt\\"\/] /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x
+    
+    def scan_tokens tokens, options
+      
+      state = :initial
+      stack = []
+      string_delimiter = nil
+      key_expected = false
+      
+      until eos?
+        
+        kind = nil
+        match = nil
+        
+        case state
+        
+        when :initial
+          if match = scan(/ \s+ | \\\n /x)
+            tokens << [match, :space]
+            next
+          elsif match = scan(/ [:,\[{\]}] /x)
+            kind = :operator
+            case match
+            when '{': stack << :object; key_expected = true
+            when '[': stack << :array
+            when ':': key_expected = false
+            when ',': key_expected = true if stack.last == :object
+            when '}', ']': stack.pop  # no error recovery, but works for valid JSON
+            end
+          elsif match = scan(/ true | false | null /x)
+            kind = IDENT_KIND[match]
+          elsif match = scan(/-?(?:0|[1-9]\d*)/)
+            kind = :integer
+            if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
+              match << matched
+              kind = :float
+            end
+          elsif match = scan(/"/)
+            state = key_expected ? :key : :string
+            tokens << [:open, state]
+            kind = :delimiter
+          else
+            getch
+            kind = :error
+          end
+          
+        when :string, :key
+          if scan(/[^\\"]+/)
+            kind = :content
+          elsif scan(/"/)
+            tokens << ['"', :delimiter]
+            tokens << [:close, state]
+            state = :initial
+            next
+          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          elsif scan(/\\./m)
+            kind = :content
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, :delimiter]
+            kind = :error
+            state = :initial
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+          
+        else
+          raise_inspect 'Unknown state', tokens
+          
+        end
+        
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+        
+      end
+      
+      if [:string, :key].include? state
+        tokens << [:close, state]
+      end
+      
+      tokens
+    end
+    
+  end
+  
+end
+end