Fixing bug in Java and JSON scanners (unfinished strings where not closed correctly).

4 files changed, 35 insertions, 34 deletions

diff --git a/Changes.textile b/Changes.textile
index ac20c94..180371c 100644
--- a/Changes.textile
+++ b/Changes.textile
@@ -154,6 +154,10 @@ h3. @Scanners::JavaScript@
  
 * *FIXED*: Don't keep state of XML scanner between calls for E4X literals.
 
+h3. @Scanners::Java@, @Scanners::JSON@
+ 
+* *FIXED*: Close unfinished strings with the correct token kind.
+
 
 h2. Changes in 0.9.2
  
diff --git a/etc/coderay-lib.tmproj b/etc/coderay-lib.tmproj
index 8c3863a..b4d2d6a 100644
--- a/etc/coderay-lib.tmproj
+++ b/etc/coderay-lib.tmproj
@@ -3,10 +3,12 @@
 <plist version="1.0">
 <dict>
 	<key>currentDocument</key>
-	<string>../lib/coderay/scanners/diff.rb</string>
+	<string>speedup/direct-stream.rb</string>
 	<key>documents</key>
 	<array>
 		<dict>
+			<key>expanded</key>
+			<true/>
 			<key>name</key>
 			<string>lib</string>
 			<key>regexFolderFilter</key>
@@ -15,8 +17,6 @@
 			<string>../lib</string>
 		</dict>
 		<dict>
-			<key>expanded</key>
-			<true/>
 			<key>name</key>
 			<string>bin</string>
 			<key>regexFolderFilter</key>
@@ -28,15 +28,13 @@
 			<key>filename</key>
 			<string>../diff</string>
 			<key>lastUsed</key>
-			<date>2010-04-11T23:04:37Z</date>
+			<date>2010-04-15T00:18:50Z</date>
 		</dict>
 		<dict>
 			<key>filename</key>
 			<string>../Changes.textile</string>
 			<key>lastUsed</key>
-			<date>2010-04-11T23:14:08Z</date>
-			<key>selected</key>
-			<true/>
+			<date>2010-04-15T00:12:51Z</date>
 		</dict>
 		<dict>
 			<key>filename</key>
@@ -55,6 +53,8 @@
 			<string>../ftp.yaml</string>
 		</dict>
 		<dict>
+			<key>expanded</key>
+			<true/>
 			<key>name</key>
 			<string>etc</string>
 			<key>regexFolderFilter</key>
@@ -89,6 +89,8 @@
 			<string>../rake_helpers</string>
 		</dict>
 		<dict>
+			<key>expanded</key>
+			<true/>
 			<key>name</key>
 			<string>rake_tasks</string>
 			<key>regexFolderFilter</key>
@@ -114,59 +116,58 @@
 			<key>filename</key>
 			<string>../test/scanners/coderay_suite.rb</string>
 			<key>lastUsed</key>
-			<date>2010-04-11T21:08:06Z</date>
+			<date>2010-04-15T23:26:11Z</date>
 		</dict>
 		<dict>
 			<key>filename</key>
 			<string>../test/scanners/suite.rb</string>
 			<key>lastUsed</key>
-			<date>2010-03-26T05:02:45Z</date>
+			<date>2010-04-14T23:56:18Z</date>
 		</dict>
 		<dict>
 			<key>filename</key>
 			<string>../bench/bench.rb</string>
 			<key>lastUsed</key>
-			<date>2010-03-26T04:44:40Z</date>
+			<date>2010-04-15T01:40:12Z</date>
 		</dict>
 	</array>
 	<key>fileHierarchyDrawerWidth</key>
 	<integer>213</integer>
 	<key>metaData</key>
 	<dict>
-		<key>../lib/coderay/scanners/diff.rb</key>
+		<key>speedup/current.rb</key>
 		<dict>
 			<key>caret</key>
 			<dict>
 				<key>column</key>
-				<integer>34</integer>
-				<key>line</key>
 				<integer>38</integer>
+				<key>line</key>
+				<integer>115</integer>
 			</dict>
-			<key>columnSelection</key>
-			<false/>
 			<key>firstVisibleColumn</key>
 			<integer>0</integer>
 			<key>firstVisibleLine</key>
-			<integer>17</integer>
-			<key>selectFrom</key>
-			<dict>
-				<key>column</key>
-				<integer>23</integer>
-				<key>line</key>
-				<integer>38</integer>
-			</dict>
-			<key>selectTo</key>
+			<integer>90</integer>
+		</dict>
+		<key>speedup/direct-stream.rb</key>
+		<dict>
+			<key>caret</key>
 			<dict>
 				<key>column</key>
-				<integer>34</integer>
+				<integer>27</integer>
 				<key>line</key>
-				<integer>38</integer>
+				<integer>150</integer>
 			</dict>
+			<key>firstVisibleColumn</key>
+			<integer>0</integer>
+			<key>firstVisibleLine</key>
+			<integer>132</integer>
 		</dict>
 	</dict>
 	<key>openDocuments</key>
 	<array>
-		<string>../lib/coderay/scanners/diff.rb</string>
+		<string>speedup/current.rb</string>
+		<string>speedup/direct-stream.rb</string>
 	</array>
 	<key>showFileHierarchyDrawer</key>
 	<true/>
diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb
index 2dfb1b6..e4a7421 100644
--- a/lib/coderay/scanners/java.rb
+++ b/lib/coderay/scanners/java.rb
@@ -149,7 +149,7 @@ module Scanners
           elsif scan(/\\./m)
             kind = :content
           elsif scan(/ \\ | $ /x)
-            tokens << [:close, :delimiter]
+            tokens << [:close, state]
             kind = :error
             state = :initial
           else
diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 163ec46..ca74ff3 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -14,9 +14,6 @@ module Scanners
       :error, :integer, :operator, :value,
     ]  # :nodoc:
     
-    CONSTANTS = %w( true false null )  # :nodoc:
-    IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)  # :nodoc:
-    
     ESCAPE = / [bfnrt\\"\/] /x  # :nodoc:
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x  # :nodoc:
     
@@ -26,7 +23,6 @@ module Scanners
       
       state = :initial
       stack = []
-      string_delimiter = nil
       key_expected = false
       
       until eos?
@@ -50,7 +46,7 @@ module Scanners
             when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
             end
           elsif match = scan(/ true | false | null /x)
-            kind = IDENT_KIND[match]
+            kind = :value
           elsif match = scan(/-?(?:0|[1-9]\d*)/)
             kind = :integer
             if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
@@ -79,7 +75,7 @@ module Scanners
           elsif scan(/\\./m)
             kind = :content
           elsif scan(/ \\ | $ /x)
-            tokens << [:close, :delimiter]
+            tokens << [:close, state]
             kind = :error
             state = :initial
           else