From abb92f30b12e11781afa76f43a344627520b5b34 Mon Sep 17 00:00:00 2001 From: Eric Guo Date: Sun, 8 Jul 2012 14:32:28 +0800 Subject: New: *Go Encoder* Draft version, copy from c --- lib/coderay/scanners/go.rb | 195 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 lib/coderay/scanners/go.rb (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb new file mode 100644 index 0000000..4431ef2 --- /dev/null +++ b/lib/coderay/scanners/go.rb @@ -0,0 +1,195 @@ +module CodeRay +module Scanners + + # Scanner for Go, copy from c + class Go < Scanner + + register_for :go + file_extension 'go' + + # http://golang.org/ref/spec#Keywords + KEYWORDS = [ + 'break', 'default', 'func', 'interface', 'select', + 'case', 'defer', 'go', 'map', 'struct', + 'chan', 'else', 'goto', 'package', 'switch', + 'const', 'fallthrough', 'if', 'range', 'type', + 'continue', 'for', 'import', 'return', 'var', + ] # :nodoc: + + # http://golang.org/ref/spec#Types + PREDEFINED_TYPES = [ + 'bool', + 'uint8', 'uint16', 'uint32', 'uint64', + 'int8', 'int16', 'int32', 'int64', + 'float32', 'float64', + 'complex64', 'complex128', + 'byte', 'rune', + 'uint', 'int', 'uintptr', + ] # :nodoc: + + PREDEFINED_CONSTANTS = [ + 'nil', 'iota', + 'true', 'false', + ] # :nodoc: + + DIRECTIVES = [ + 'go_no_directive', # Seems no directive concept in Go? + ] # :nodoc: + + IDENT_KIND = WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_TYPES, :predefined_type). + add(DIRECTIVES, :directive). + add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + + protected + + def scan_tokens encoder, options + + state = :initial + label_expected = true + case_expected = false + label_expected_before_preproc_line = nil + in_preproc_line = false + + until eos? + + case state + + when :initial + + if match = scan(/ \s+ | \\\n /x) + if in_preproc_line && match != "\\\n" && match.index(?\n) + in_preproc_line = false + label_expected = label_expected_before_preproc_line + end + encoder.text_token match, :space + + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) + encoder.text_token match, :comment + + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + label_expected = match =~ /[;\{\}]/ + if case_expected + label_expected = true if match == ':' + case_expected = false + end + encoder.text_token match, :operator + + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) + kind = IDENT_KIND[match] + if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) + kind = :label + match << matched + else + label_expected = false + if kind == :keyword + case match + when 'case', 'default' + case_expected = true + end + end + end + encoder.text_token match, kind + + elsif match = scan(/L?"/) + encoder.begin_group :string + if match[0] == ?L + encoder.text_token 'L', :modifier + match = '"' + end + encoder.text_token match, :delimiter + state = :string + + elsif match = scan(/ \# \s* if \s* 0 /x) + match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? + encoder.text_token match, :comment + + elsif match = scan(/#[ \t]*(\w*)/) + encoder.text_token match, :preprocessor + in_preproc_line = true + label_expected_before_preproc_line = label_expected + state = :include_expected if self[1] == 'include' + + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + label_expected = false + encoder.text_token match, :char + + elsif match = scan(/\$/) + encoder.text_token match, :ident + + elsif match = scan(/0[xX][0-9A-Fa-f]+/) + label_expected = false + encoder.text_token match, :hex + + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + label_expected = false + encoder.text_token match, :octal + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer + + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + label_expected = false + encoder.text_token match, :float + + else + encoder.text_token getch, :error + + end + + when :string + if match = scan(/[^\\\n"]+/) + encoder.text_token match, :content + elsif match = scan(/"/) + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + label_expected = false + elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) + encoder.text_token match, :char + elsif match = scan(/ \\ | $ /x) + encoder.end_group :string + encoder.text_token match, :error + state = :initial + label_expected = false + else + raise_inspect "else case \" reached; %p not handled." % peek(1), encoder + end + + when :include_expected + if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) + encoder.text_token match, :include + state = :initial + + elsif match = scan(/\s+/) + encoder.text_token match, :space + state = :initial if match.index ?\n + + else + state = :initial + + end + + else + raise_inspect 'Unknown state', encoder + + end + + end + + if state == :string + encoder.end_group :string + end + + encoder + end + + end + +end +end -- cgit v1.2.1 From fd8c81f5c338e4f1448007c9100d15d912fadd27 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:43:54 -0600 Subject: additional types: string, error http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2..a66a5e3 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ module Scanners 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'error', 'uint', 'int', 'uintptr', ] # :nodoc: -- cgit v1.2.1 From 004d0c83222113ff732101a3e25785ce0625bfa2 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:14:29 +0200 Subject: whitespace --- lib/coderay/scanners/go.rb | 82 +++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 41 deletions(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2..d4a3598 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -1,12 +1,12 @@ module CodeRay module Scanners - + # Scanner for Go, copy from c class Go < Scanner - + register_for :go file_extension 'go' - + # http://golang.org/ref/spec#Keywords KEYWORDS = [ 'break', 'default', 'func', 'interface', 'select', @@ -15,7 +15,7 @@ module Scanners 'const', 'fallthrough', 'if', 'range', 'type', 'continue', 'for', 'import', 'return', 'var', ] # :nodoc: - + # http://golang.org/ref/spec#Types PREDEFINED_TYPES = [ 'bool', @@ -26,51 +26,51 @@ module Scanners 'byte', 'rune', 'uint', 'int', 'uintptr', ] # :nodoc: - + PREDEFINED_CONSTANTS = [ 'nil', 'iota', 'true', 'false', ] # :nodoc: - + DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? ] # :nodoc: - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: - + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - - protected - + + protected + def scan_tokens encoder, options - + state = :initial label_expected = true case_expected = false label_expected_before_preproc_line = nil in_preproc_line = false - + until eos? - + case state - + when :initial - + if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space - + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) label_expected = match =~ /[;\{\}]/ if case_expected @@ -78,7 +78,7 @@ module Scanners case_expected = false end encoder.text_token match, :operator - + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) @@ -94,7 +94,7 @@ module Scanners end end encoder.text_token match, kind - + elsif match = scan(/L?"/) encoder.begin_group :string if match[0] == ?L @@ -107,41 +107,41 @@ module Scanners elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment - + elsif match = scan(/#[ \t]*(\w*)/) encoder.text_token match, :preprocessor in_preproc_line = true label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) label_expected = false encoder.text_token match, :char - + elsif match = scan(/\$/) encoder.text_token match, :ident - + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer - + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) label_expected = false encoder.text_token match, :float - + else encoder.text_token getch, :error - + end - + when :string if match = scan(/[^\\\n"]+/) encoder.text_token match, :content @@ -160,36 +160,36 @@ module Scanners else raise_inspect "else case \" reached; %p not handled." % peek(1), encoder end - + when :include_expected if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) encoder.text_token match, :include state = :initial - + elsif match = scan(/\s+/) encoder.text_token match, :space state = :initial if match.index ?\n - + else state = :initial - + end - + else raise_inspect 'Unknown state', encoder - + end - + end - + if state == :string encoder.end_group :string end - + encoder end - + end - + end end -- cgit v1.2.1 From afa2be73c03d033056024354787abd66f2ff7fa0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:05 +0200 Subject: add string as predefined type --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index d4a3598..dbf9595 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ module Scanners 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'uint', 'int', 'uintptr', ] # :nodoc: -- cgit v1.2.1 From 7ef6f7783d788bf656a3c59600cd45cd5b694376 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:22 +0200 Subject: add support for raw strings in Go --- lib/coderay/scanners/go.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index dbf9595..c0c602d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -103,7 +103,14 @@ module Scanners end encoder.text_token match, :delimiter state = :string - + + elsif match = scan(/ ` ([^`]+)? (`)? /x) + encoder.begin_group :shell + encoder.text_token '`', :delimiter + encoder.text_token self[1], :content if self[1] + encoder.text_token self[2], :delimiter if self[2] + encoder.end_group :shell + elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment -- cgit v1.2.1 From a24c207a7e547408de0f6157d495b0dd2e8d5a40 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:36 +0200 Subject: fix empty token in Go scanner --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index c0c602d..76a0820 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -161,7 +161,7 @@ module Scanners encoder.text_token match, :char elsif match = scan(/ \\ | $ /x) encoder.end_group :string - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else -- cgit v1.2.1 From 4669b7086136647f2eece41a122fa204eb16e1e9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:16:47 +0200 Subject: fix label_expected (test case?) --- lib/coderay/scanners/go.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 76a0820..23cbfe6 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -64,6 +64,7 @@ module Scanners if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false + case_expected = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space @@ -72,7 +73,6 @@ module Scanners encoder.text_token match, :comment elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) - label_expected = match =~ /[;\{\}]/ if case_expected label_expected = true if match == ':' case_expected = false @@ -83,6 +83,7 @@ module Scanners kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) kind = :label + label_expected = false match << matched else label_expected = false -- cgit v1.2.1 From 85275cf21e7d15459abc11fd76606bd7d38fb8b5 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:11:49 -0600 Subject: Go doesn't have a "f" suffix for floats like C. --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 49d24c2..afcfca5 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -141,7 +141,7 @@ module Scanners label_expected = false encoder.text_token match, :integer - elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float -- cgit v1.2.1 From cad9a00e28d61781d5a12a0556be7126eb790725 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:35:26 -0600 Subject: add imaginary numbers to Go scanner --- lib/coderay/scanners/go.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index afcfca5..04504ab 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -128,7 +128,11 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :ident - + + elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + label_expected = false + encoder.text_token match, :imaginary + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex @@ -137,13 +141,13 @@ module Scanners label_expected = false encoder.text_token match, :octal - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) - label_expected = false - encoder.text_token match, :integer - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer else encoder.text_token getch, :error -- cgit v1.2.1 From 4c877bf0eb655cf2c04a0efb24b4e5f83b97e160 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:43:50 -0600 Subject: predeclared identifiers http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 04504ab..ae50aa4 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,6 +31,11 @@ module Scanners 'nil', 'iota', 'true', 'false', ] # :nodoc: + + PREDEFINED_FUNCTIONS = %w[ + append cap close complex copy delete imag len + make new panic print println real recover + ] # :nodoc: DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? @@ -40,7 +45,8 @@ module Scanners add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). - add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: -- cgit v1.2.1 From dd9ec43a3ea4cfce8eb1e8dd38504a9d8f24df54 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:46:15 -0600 Subject: yup, no C-style directives (auto extern static) --- lib/coderay/scanners/go.rb | 5 ----- 1 file changed, 5 deletions(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index ae50aa4..eb06fb0 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -37,14 +37,9 @@ module Scanners make new panic print println real recover ] # :nodoc: - DIRECTIVES = [ - 'go_no_directive', # Seems no directive concept in Go? - ] # :nodoc: - IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). - add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant). add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: -- cgit v1.2.1 From bbe4d72ba785f1bd6fd703d63b096a907da1b09f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:31:34 +0200 Subject: tweak numeral tokens handling (#147) --- lib/coderay/scanners/go.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index eb06fb0..938da9d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,7 +31,7 @@ module Scanners 'nil', 'iota', 'true', 'false', ] # :nodoc: - + PREDEFINED_FUNCTIONS = %w[ append cap close complex copy delete imag len make new panic print println real recover @@ -73,7 +73,7 @@ module Scanners elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + elsif match = scan(/ ?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) if case_expected label_expected = true if match == ':' case_expected = false @@ -129,24 +129,24 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :ident - - elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + + elsif match = scan(/-?\d*(\.\d*)?([eE][+-]?\d+)?i/) label_expected = false encoder.text_token match, :imaginary - - elsif match = scan(/0[xX][0-9A-Fa-f]+/) + + elsif match = scan(/-?0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + elsif match = scan(/-?(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) + elsif match = scan(/-?(?:\d*\.\d+|\d+\.)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float - - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + + elsif match = scan(/-?(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer -- cgit v1.2.1 From 6dd14ef018ee4417771504117819121bd4b8520d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:34:10 +0200 Subject: be a bit more graceful with buggy Go strings --- lib/coderay/scanners/go.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 938da9d..5034adc 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -165,9 +165,10 @@ module Scanners label_expected = false elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) encoder.text_token match, :char - elsif match = scan(/ \\ | $ /x) + elsif match = scan(/ \\ /x) + encoder.text_token match, :error + elsif match = scan(/$/) encoder.end_group :string - encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else -- cgit v1.2.1 From 82864efabda9dc17fa6a28b0740ffc8f58706126 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:36:27 +0200 Subject: allow unicode characters in char literals --- lib/coderay/scanners/go.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay/scanners') diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 5034adc..59473f6 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -44,7 +44,7 @@ module Scanners add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: protected @@ -123,7 +123,7 @@ module Scanners label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) )? '? /ox) label_expected = false encoder.text_token match, :char -- cgit v1.2.1