Completed YAML Scanner (closes #34).

YAML Scanner:
* Added another example (multiline).
* Added multiline string recognition.
* This should be enough for most people.

Else:
* New method: Scanner#column (very useful and fast!)
* Added new token type :doctype, used also by HTML scanner.
* coderay_suite: minor output tweaks.

5 files changed, 38 insertions, 28 deletions

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 831fc39..371c858 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -180,6 +180,11 @@ module CodeRay
       def line
         string[0..pos].count("\n") + 1
       end
+      
+      def column pos = self.pos
+        return 0 if pos <= 0
+        pos - (string.rindex(?\n, pos) || 0)
+      end
 
     protected
 
@@ -216,7 +221,7 @@ module CodeRay
 tokens:
 %s
 
-current line: %d  pos = %d
+current line: %d  column: %d  pos: %d
 matched: %p  state: %p
 bol? = %p,  eos? = %p
 
@@ -231,7 +236,7 @@ surrounding code:
           msg,
           tokens.size,
           tokens.last(10).map { |t| t.inspect }.join("\n"),
-          line, pos,
+          line, column, pos,
           matched, state, bol?, eos?,
           string[pos - ambit, ambit],
           string[pos, ambit],
diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index bb7ad3a..1f4d8fe 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -63,7 +63,7 @@ module Scanners
             if scan(/<!--.*?-->/m)
               kind = :comment
             elsif scan(/<!DOCTYPE.*?>/m)
-              kind = :preprocessor
+              kind = :doctype
             elsif scan(/<\?xml.*?\?>/m)
               kind = :preprocessor
             elsif scan(/<\?.*?\?>|<%.*?%>/m)
diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb
index 3ecd7e1..d99a38f 100644
--- a/lib/coderay/scanners/yaml.rb
+++ b/lib/coderay/scanners/yaml.rb
@@ -13,7 +13,7 @@ module Scanners
       
       value_expected = nil
       state = :initial
-      indent = 0
+      key_indent = indent = 0
       
       until eos?
         
@@ -21,8 +21,11 @@ module Scanners
         match = nil
         
         if bol?
-          indent = matched.size if check(/ +/)
-          # tokens << [indent.to_s, :debug]
+          key_indent = nil
+          if $DEBUG
+            indent = check(/ +/) ? matched.size : 0
+            tokens << [indent.to_s, :debug]
+          end
         end
         
         if match = scan(/ +[\t ]*/)
@@ -41,6 +44,9 @@ module Scanners
             tokens << [match, :head]
             tokens << [:close, :head]
             next
+          when match = scan(/%.*/)
+            tokens << [match, :doctype]
+            next
           end
         
         elsif state == :value and case
@@ -51,8 +57,17 @@ module Scanners
             tokens << [matched, :delimiter] if scan(/"/)
             tokens << [:close, :string]
             next
-          when scan(/(?![!"*&]).+?(?=$|\s+#)/)
-            kind = :string
+          when match = scan(/[|>][-+]?/)
+            tokens << [:open, :string]
+            tokens << [match, :delimiter]
+            tokens << [matched, :content] if scan(/(?:\n+ {#{key_indent + 1}}.*)+/)
+            tokens << [:close, :string]
+            next
+          when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
+            tokens << [match, :string]
+            string_indent = key_indent || column(pos - match.size - 1)
+            tokens << [matched, :string] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
+            next
           end
           
         elsif case
@@ -62,15 +77,19 @@ module Scanners
             kind = :operator
           when match = scan(/[,{}\[\]]/)
             kind = :operator
-          when state == :initial && scan(/[\w.() ]*\S(?=: |:$)/)
+          when state == :initial && match = scan(/[\w.() ]*\S(?=: |:$)/)
             kind = :key
+            key_indent = column(pos - match.size - 1)
+            # tokens << [key_indent.inspect, :debug]
             state = :colon
-          when match = scan(/(?:"[^"]*"|'[^']*')(?=: |:$)/)
+          when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?=: |:$)/)
             tokens << [:open, :key]
             tokens << [match[0,1], :delimiter]
             tokens << [match[1..-2], :content]
             tokens << [match[-1,1], :delimiter]
             tokens << [:close, :key]
+            key_indent = column(pos - match.size - 1)
+            # tokens << [key_indent.inspect, :debug]
             state = :colon
             next
           when scan(/(![\w\/]+)(:([\w:]+))?/)
@@ -93,25 +112,9 @@ module Scanners
           when scan(/:\w+/)
             kind = :symbol
           when scan(/[^:\s]+(:(?! |$)[^:\s]*)* .*/)
-            kind = :string
+            kind = :error
           when scan(/[^:\s]+(:(?! |$)[^:\s]*)*/)
-            kind = :string
-          # when scan(/>-?/)
-          #   kind = :punct
-          #   kind = :normal, scan(/.*$/)
-          #   append getch until eos? || bol?
-          #   return if eos?
-          #   indent = check(/ */)
-          #   kind = :string
-          #   loop do
-          #     line = check_until(/[\n\r]|\Z/)
-          #     break if line.nil?
-          #     if line.chomp.length > 0
-          #       this_indent = line.chomp.match( /^\s*/ )[0]
-          #       break if this_indent.length < indent.length
-          #     end
-          #     append scan_until(/[\n\r]|\Z/)
-          #   end
+            kind = :error
           end
           
         else
diff --git a/lib/coderay/styles/cycnus.rb b/lib/coderay/styles/cycnus.rb
index 0a1c6f8..8d7c8cd 100644
--- a/lib/coderay/styles/cycnus.rb
+++ b/lib/coderay/styles/cycnus.rb
@@ -64,6 +64,7 @@ ol.CodeRay li { white-space: pre }
 .di { color:#088; font-weight:bold }
 .dl { color:black }
 .do { color:#970 }
+.dt { color:#34b }
 .ds { color:#D42; font-weight:bold }
 .e  { color:#666; font-weight:bold }
 .en { color:#800; font-weight:bold }
diff --git a/lib/coderay/token_classes.rb b/lib/coderay/token_classes.rb
index d85d1af..c71705b 100755
--- a/lib/coderay/token_classes.rb
+++ b/lib/coderay/token_classes.rb
@@ -21,6 +21,7 @@ module CodeRay
       :delimiter => 'dl',
       :directive => 'di',
       :doc => 'do',
+      :doctype => 'dt',
       :doc_string => 'ds',
       :entity => 'en',
       :error => 'er',