From 2fb89ea3abfd8ee6c4eed8defb805a1abd7187d9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:03:18 +0200 Subject: refactor Scanner a bit (#135) --- lib/coderay/scanner.rb | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'lib/coderay/scanner.rb') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 907cf00..80ccc5d 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -184,14 +184,8 @@ module CodeRay options = @options.merge(options) @tokens = options[:tokens] || @tokens || Tokens.new @tokens.scanner = self if @tokens.respond_to? :scanner= - case source - when Array - self.string = self.class.normalize(source.join) - when nil - reset - else - self.string = self.class.normalize(source) - end + + set_string_from_source source begin scan_tokens @tokens, options @@ -261,6 +255,17 @@ module CodeRay def setup # :doc: end + def set_string_from_source source + case source + when Array + self.string = self.class.normalize(source.join) + when nil + reset + else + self.string = self.class.normalize(source) + end + end + # This is the central method, and commonly the only one a # subclass implements. # @@ -277,9 +282,7 @@ module CodeRay @binary_string = nil if defined? @binary_string end - # Scanner error with additional status information - def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller - raise ScanError, <<-EOE % [ + SCAN_ERROR_MESSAGE = <<-MESSAGE ***ERROR in %s: %s (after %d tokens) @@ -297,7 +300,11 @@ surrounding code: ***ERROR*** - EOE + MESSAGE + + # Scanner error with additional status information + def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller + raise ScanError, SCAN_ERROR_MESSAGE % [ File.basename(caller[0]), msg, tokens.respond_to?(:size) ? tokens.size : 0, -- cgit v1.2.1 From c386e044cff195ae922a0b7625dfa498954bf772 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:15:17 +0200 Subject: refactor Scanner a bit more (#135) --- lib/coderay/scanner.rb | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'lib/coderay/scanner.rb') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 80ccc5d..9a5be58 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -182,9 +182,8 @@ module CodeRay # Scan the code and returns all tokens in a Tokens object. def tokenize source = nil, options = {} options = @options.merge(options) - @tokens = options[:tokens] || @tokens || Tokens.new - @tokens.scanner = self if @tokens.respond_to? :scanner= + set_tokens_from_options options set_string_from_source source begin @@ -266,6 +265,11 @@ module CodeRay end end + def set_tokens_from_options options + @tokens = options[:tokens] || @tokens || Tokens.new + @tokens.scanner = self if @tokens.respond_to? :scanner= + end + # This is the central method, and commonly the only one a # subclass implements. # @@ -292,7 +296,7 @@ tokens: current line: %d column: %d pos: %d matched: %p state: %p -bol? = %p, eos? = %p +bol?: %p, eos?: %p surrounding code: %p ~~ %p @@ -303,14 +307,15 @@ surrounding code: MESSAGE # Scanner error with additional status information - def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller + def raise_inspect msg, tokens, state = self.state, ambit = 30, backtrace = caller raise ScanError, SCAN_ERROR_MESSAGE % [ File.basename(caller[0]), msg, - tokens.respond_to?(:size) ? tokens.size : 0, - tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '', + tokens.respond_to?(:size) ? tokens.size : '[tokens.size undefined]', + tokens.respond_to?(:last) ? tokens.last(10).map(&:inspect).join("\n") : '[tokens.last undefined]', line, column, pos, - matched, state, bol?, eos?, + matched, state || 'No state given!', + bol?, eos?, binary_string[pos - ambit, ambit], binary_string[pos, ambit], ], backtrace -- cgit v1.2.1 From 65a4b4dfe93e9be0d9f753f063b8f77f8be5af77 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:23:17 +0200 Subject: trying to reduce "complexity" of #raise_inspect (#135) --- lib/coderay/scanner.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'lib/coderay/scanner.rb') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 9a5be58..68451ae 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -289,7 +289,7 @@ module CodeRay SCAN_ERROR_MESSAGE = <<-MESSAGE -***ERROR in %s: %s (after %d tokens) +***ERROR in %s: %s (after %s tokens) tokens: %s @@ -311,8 +311,8 @@ surrounding code: raise ScanError, SCAN_ERROR_MESSAGE % [ File.basename(caller[0]), msg, - tokens.respond_to?(:size) ? tokens.size : '[tokens.size undefined]', - tokens.respond_to?(:last) ? tokens.last(10).map(&:inspect).join("\n") : '[tokens.last undefined]', + tokens_size, + tokens_last(10).map(&:inspect).join("\n"), line, column, pos, matched, state || 'No state given!', bol?, eos?, @@ -321,6 +321,14 @@ surrounding code: ], backtrace end + def tokens_size + tokens.size if tokens.respond_to?(:size) + end + + def tokens_last n + tokens.respond_to?(:last) ? tokens.last(n) : [] + end + # Shorthand for scan_until(/\z/). # This method also avoids a JRuby 1.9 mode bug. def scan_rest -- cgit v1.2.1 From 65b8ab0fda2ac06a2d6109ea96485ab148bab1a4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:32:16 +0200 Subject: trying to reduce "complexity" of #raise_inspect (#135) --- lib/coderay/scanner.rb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'lib/coderay/scanner.rb') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 68451ae..26bcbf3 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -307,25 +307,27 @@ surrounding code: MESSAGE # Scanner error with additional status information - def raise_inspect msg, tokens, state = self.state, ambit = 30, backtrace = caller - raise ScanError, SCAN_ERROR_MESSAGE % [ - File.basename(caller[0]), - msg, - tokens_size, - tokens_last(10).map(&:inspect).join("\n"), + def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller + raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace + end + + def raise_inspect_arguments message, tokens, state, ambit + return File.basename(caller[0]), + message, + tokens_size(tokens), + tokens_last(tokens, 10).map(&:inspect).join("\n"), line, column, pos, matched, state || 'No state given!', bol?, eos?, binary_string[pos - ambit, ambit], - binary_string[pos, ambit], - ], backtrace + binary_string[pos, ambit] end - def tokens_size + def tokens_size tokens tokens.size if tokens.respond_to?(:size) end - def tokens_last n + def tokens_last tokens, n tokens.respond_to?(:last) ? tokens.last(n) : [] end -- cgit v1.2.1 From cbf002e7c5e78cee9c2ef7f565b064d9f666bc2b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 10 Jun 2013 18:39:19 +0200 Subject: trying to reduce "complexity" of #raise_inspect (#135) --- lib/coderay/scanner.rb | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'lib/coderay/scanner.rb') diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 26bcbf3..b3f7e17 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -294,9 +294,7 @@ module CodeRay tokens: %s -current line: %d column: %d pos: %d -matched: %p state: %p -bol?: %p, eos?: %p +%s surrounding code: %p ~~ %p @@ -306,21 +304,33 @@ surrounding code: MESSAGE - # Scanner error with additional status information - def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller - raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace - end - def raise_inspect_arguments message, tokens, state, ambit return File.basename(caller[0]), message, tokens_size(tokens), tokens_last(tokens, 10).map(&:inspect).join("\n"), + scanner_state_info(state), + binary_string[pos - ambit, ambit], + binary_string[pos, ambit] + end + + SCANNER_STATE_INFO = <<-INFO +current line: %d column: %d pos: %d +matched: %p state: %p +bol?: %p, eos?: %p + INFO + + def scanner_state_info state + SCANNER_STATE_INFO % [ line, column, pos, matched, state || 'No state given!', bol?, eos?, - binary_string[pos - ambit, ambit], - binary_string[pos, ambit] + ] + end + + # Scanner error with additional status information + def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller + raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace end def tokens_size tokens -- cgit v1.2.1