diff options
Diffstat (limited to 'lib')
29 files changed, 198 insertions, 253 deletions
diff --git a/lib/coderay.rb b/lib/coderay.rb index d2d7331..9779ff5 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -158,6 +158,7 @@ module CodeRay # # See also demo/demo_simple. def scan code, lang, options = {}, &block + # FIXME: return a proxy for direct-stream encoding scanner = Scanners[lang].new code, options, &block scanner.tokenize end @@ -187,7 +188,7 @@ module CodeRay # encodes it with the Encoder for +format+. # +options+ will be passed to the Encoder. # - # See CodeRay::Encoder.encode + # See CodeRay::Encoder.encode. def encode code, lang, format, options = {} encoder(format, options).encode code, lang, options end diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb index 85a2456..cc331d1 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoder.rb @@ -153,7 +153,17 @@ module CodeRay # # See the HTML Encoder for an example of option caching. def setup options - @out = '' + @out = get_output(options) + end + + def get_output options + options[:out] || '' + end + + # Append data.to_s to the output. Returns the argument. + def output data + @out << data.to_s + data end # Called with merged options after encoding starts. diff --git a/lib/coderay/encoders/_map.rb b/lib/coderay/encoders/_map.rb index 24ada0a..e5cbdeb 100644 --- a/lib/coderay/encoders/_map.rb +++ b/lib/coderay/encoders/_map.rb @@ -3,7 +3,6 @@ module Encoders map \ :loc => :lines_of_code, - :html => :page, :plain => :text, :plaintext => :text, :remove_comments => :comment_filter, diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb index 52d4bdd..98a427e 100644 --- a/lib/coderay/encoders/count.rb +++ b/lib/coderay/encoders/count.rb @@ -11,17 +11,23 @@ module Encoders protected def setup options - @out = 0 + super + + @count = 0 + end + + def finish options + output @count end public def text_token text, kind - @out += 1 + @count += 1 end def begin_group kind - @out += 1 + @count += 1 end alias end_group begin_group alias begin_line begin_group diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 08f1309..95d6138 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -27,7 +27,7 @@ module Encoders if kind == :space @out << text else - # FIXME: Escape ( + # TODO: Escape ( text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \ @out << kind.to_s << '(' << text << ')' end diff --git a/lib/coderay/encoders/filter.rb b/lib/coderay/encoders/filter.rb index a71e43e..e7f34d6 100644 --- a/lib/coderay/encoders/filter.rb +++ b/lib/coderay/encoders/filter.rb @@ -21,29 +21,35 @@ module Encoders protected def setup options - @out = options[:tokens] || Tokens.new + super + + @tokens = options[:tokens] || Tokens.new + end + + def finish options + output @tokens end public def text_token text, kind # :nodoc: - @out.text_token text, kind + @tokens.text_token text, kind end def begin_group kind # :nodoc: - @out.begin_group kind + @tokens.begin_group kind end def begin_line kind # :nodoc: - @out.begin_line kind + @tokens.begin_line kind end def end_group kind # :nodoc: - @out.end_group kind + @tokens.end_group kind end def end_line kind # :nodoc: - @out.end_line kind + @tokens.end_line kind end end diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ec73d2c..abbafad 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -164,6 +164,11 @@ module Encoders def setup options super + if options[:wrap] || options[:line_numbers] + @real_out = @out + @out = '' + end + @HTML_ESCAPE = HTML_ESCAPE.dup @HTML_ESCAPE["\t"] = ' ' * options[:tab_width] @@ -214,7 +219,7 @@ module Encoders def finish options unless @opened.empty? - warn '%d tokens still open: %p' % [@opened.size, @opened] + warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG @out << '</span>' while @opened.pop @last_opened = nil end @@ -227,6 +232,11 @@ module Encoders @out.wrap! options[:wrap] @out.apply_title! options[:title] + if defined?(@real_out) && @real_out + @real_out << @out + @out = @real_out + end + super end diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index c459222..6de4b46 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -44,7 +44,7 @@ module Encoders ( [^\}]+ )? # $2 = style \s* \} \s* | - ( . ) # $3 = error + ( [^\n]+ ) # $3 = error /mx def parse stylesheet stylesheet.scan CSS_CLASS_PATTERN do |selectors, style, error| diff --git a/lib/coderay/encoders/html/output.rb b/lib/coderay/encoders/html/output.rb index 004351b..4f65878 100644 --- a/lib/coderay/encoders/html/output.rb +++ b/lib/coderay/encoders/html/output.rb @@ -55,7 +55,6 @@ module Encoders end def apply_title! title - # FIXME: This may change the output! self.sub!(/(<title>)(<\/title>)/) { $1 + title + $2 } self end diff --git a/lib/coderay/encoders/json.rb b/lib/coderay/encoders/json.rb index ccad554..0a95397 100644 --- a/lib/coderay/encoders/json.rb +++ b/lib/coderay/encoders/json.rb @@ -36,32 +36,45 @@ module Encoders protected def setup options - @out = [] + super + + @first = true + @out << '[' end def finish options - @out.to_json + @out << ']' + end + + def append data + if @first + @first = false + else + @out << ',' + end + + @out << data.to_json end public def text_token text, kind - @out << { :type => 'text', :text => text, :kind => kind } + append :type => 'text', :text => text, :kind => kind end def begin_group kind - @out << { :type => 'block', :action => 'open', :kind => kind } + append :type => 'block', :action => 'open', :kind => kind end def end_group kind - @out << { :type => 'block', :action => 'close', :kind => kind } + append :type => 'block', :action => 'close', :kind => kind end def begin_line kind - @out << { :type => 'block', :action => 'begin_line', :kind => kind } + append :type => 'block', :action => 'begin_line', :kind => kind end def end_line kind - @out << { :type => 'block', :action => 'end_line', :kind => kind } + append :type => 'block', :action => 'end_line', :kind => kind end end diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb index c49cade..5f8422f 100644 --- a/lib/coderay/encoders/lines_of_code.rb +++ b/lib/coderay/encoders/lines_of_code.rb @@ -31,11 +31,12 @@ module Encoders end options[:exclude] = kinds_not_loc + super options end def finish options - @out.to_s.scan(NON_EMPTY_LINE).size + output @tokens.text.scan(NON_EMPTY_LINE).size end end diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb index e52f28f..2315d9e 100644 --- a/lib/coderay/encoders/statistic.rb +++ b/lib/coderay/encoders/statistic.rb @@ -15,15 +15,12 @@ module Encoders protected def setup options + super + @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 } @real_token_count = 0 end - def generate tokens, options - @tokens = tokens - super - end - STATS = <<-STATS # :nodoc: Code Statistics @@ -51,11 +48,13 @@ Token Types (%d): types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v| TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size] end.join - STATS % [ + @out << STATS % [ all_count, @real_token_count, all_size, @type_stats.delete_if { |k, v| k.is_a? String }.size, types_stats ] + + super end public diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb index 84e2215..15c66f9 100644 --- a/lib/coderay/encoders/text.rb +++ b/lib/coderay/encoders/text.rb @@ -24,19 +24,22 @@ module Encoders def text_token text, kind super - @out << @sep if @sep + + if @first + @first = false + else + @out << @sep + end if @sep end protected def setup options super + + @first = true @sep = options[:separator] end - def finish options - super.chomp @sep - end - end end diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb index 1ecf6ae..4773ea3 100644 --- a/lib/coderay/encoders/token_kind_filter.rb +++ b/lib/coderay/encoders/token_kind_filter.rb @@ -34,6 +34,7 @@ module Encoders protected def setup options super + @group_excluded = false @exclude = options[:exclude] @exclude = Array(@exclude) unless @exclude == :all diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb index f2e7c02..3d306a6 100644 --- a/lib/coderay/encoders/xml.rb +++ b/lib/coderay/encoders/xml.rb @@ -10,7 +10,7 @@ module Encoders FILE_EXTENSION = 'xml' - require 'rexml/document' + autoload :REXML, 'rexml/document' DEFAULT_OPTIONS = { :tab_width => 8, @@ -20,6 +20,8 @@ module Encoders protected def setup options + super + @doc = REXML::Document.new @doc << REXML::XMLDecl.new @tab_width = options[:tab_width] @@ -27,9 +29,9 @@ module Encoders end def finish options - @out = '' @doc.write @out, options[:pretty], options[:transitive], true - @out + + super end public diff --git a/lib/coderay/encoders/yaml.rb b/lib/coderay/encoders/yaml.rb index c12f8d0..1eb2523 100644 --- a/lib/coderay/encoders/yaml.rb +++ b/lib/coderay/encoders/yaml.rb @@ -6,39 +6,44 @@ module Encoders # Slow. class YAML < Encoder + autoload :YAML, 'yaml' + register_for :yaml FILE_EXTENSION = 'yaml' protected def setup options - require 'yaml' - @out = [] + super + + @data = [] end def finish options - @out.to_a.to_yaml + YAML.dump @data, @out + + super end public def text_token text, kind - @out << [text, kind] + @data << [text, kind] end def begin_group kind - @out << [:begin_group, kind] + @data << [:begin_group, kind] end def end_group kind - @out << [:end_group, kind] + @data << [:end_group, kind] end def begin_line kind - @out << [:begin_line, kind] + @data << [:begin_line, kind] end def end_line kind - @out << [:end_line, kind] + @data << [:end_line, kind] end end diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index ea7125f..c752a03 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -90,7 +90,7 @@ module CodeRay 'h' => :c, 'htm' => :page, 'html' => :page, - 'html.erb' => :rhtml, + 'html.erb' => :erb, 'java' => :java, 'js' => :java_script, 'json' => :json, @@ -109,7 +109,7 @@ module CodeRay 'raydebug' => :raydebug, 'rb' => :ruby, 'rbw' => :ruby, - 'rhtml' => :rhtml, + 'rhtml' => :erb, 'rjs' => :ruby, 'rpdf' => :ruby, 'ru' => :ruby, diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index 8a8e149..06c1233 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -114,9 +114,10 @@ module CodeRay def default id = nil if id id = validate_id id - plugin_hash[nil] = id + raise "The default plugin can't be named \"default\"." if id == :default + plugin_hash[:default] = id else - load nil + load :default end end @@ -179,10 +180,11 @@ module CodeRay require path rescue LoadError => boom if @plugin_map_loaded - if h.has_key?(nil) # default plugin - h[nil] + if h.has_key?(:default) + warn '%p could not load plugin %p; falling back to %p' % [self, id, h[:default]] + h[:default] else - raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom] + raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] end else load_plugin_map diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 0e0723c..e638c2c 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -74,7 +74,7 @@ module CodeRay if code.respond_to? :encoding code = encode_with_encoding code, self.encoding else - code = to_unix code if code.index ?\r + code = to_unix code end # code = code.dup if code.eql? original code @@ -100,7 +100,7 @@ module CodeRay def encode_with_encoding code, target_encoding if code.encoding == target_encoding if code.valid_encoding? - return to_unix(code) + return to_unix code else source_encoding = guess_encoding code end @@ -112,7 +112,7 @@ module CodeRay end def to_unix code - code.gsub(/\r\n?/, "\n") + code.index(?\r) ? code.gsub(/\r\n?/, "\n") : code end def guess_encoding s @@ -221,27 +221,39 @@ module CodeRay end include Enumerable - # The current line position of the scanner. See also #column. + # The current line position of the scanner, starting with 1. + # See also: #column. # # Beware, this is implemented inefficiently. It should be used # for debugging only. - def line - string[0..pos].count("\n") + 1 + def line pos = self.pos + return 1 if pos <= 0 + binary_string[0...pos].count("\n") + 1 end - # The current column position of the scanner. See also #line. + # The current column position of the scanner, starting with 1. + # See also: #line. # # Beware, this is implemented inefficiently. It should be used # for debugging only. def column pos = self.pos - return 0 if pos <= 0 - string = self.string - if string.respond_to?(:bytesize) && string.bytesize != string.size - #:nocov: - string = string.dup.force_encoding('binary') - #:nocov: - end - pos - (string.rindex(?\n, pos) || 0) + return 1 if pos <= 0 + pos - (binary_string.rindex(?\n, pos - 1) || -1) + end + + # The string in binary encoding. + # + # To be used with #pos, which is the index of the byte the scanner + # will scan next. + def binary_string + @binary_string ||= + if string.respond_to?(:bytesize) && string.bytesize != string.size + #:nocov: + string.dup.force_encoding('binary') + #:nocov: + else + string + end end protected @@ -267,7 +279,7 @@ module CodeRay def reset_instance @tokens.clear if @tokens.respond_to?(:clear) && !@options[:keep_tokens] @cached_tokens = nil - @bin_string = nil if defined? @bin_string + @binary_string = nil if defined? @binary_string end # Scanner error with additional status information @@ -297,8 +309,8 @@ surrounding code: tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '', line, column, pos, matched, state, bol?, eos?, - string[pos - ambit, ambit], - string[pos, ambit], + binary_string[pos - ambit, ambit], + binary_string[pos, ambit], ], backtrace end diff --git a/lib/coderay/scanners/_map.rb b/lib/coderay/scanners/_map.rb index b94d452..a240298 100644 --- a/lib/coderay/scanners/_map.rb +++ b/lib/coderay/scanners/_map.rb @@ -6,11 +6,11 @@ module Scanners :cplusplus => :cpp, :ecmascript => :java_script, :ecma_script => :java_script, - :erb => :rhtml, + :rhtml => :erb, + :eruby => :erb, :irb => :ruby, :javascript => :java_script, :js => :java_script, - :nitro => :nitro_xhtml, :pascal => :delphi, :patch => :diff, :plain => :text, diff --git a/lib/coderay/scanners/clojure.rb b/lib/coderay/scanners/clojure.rb index 89713de..f8fbf65 100644 --- a/lib/coderay/scanners/clojure.rb +++ b/lib/coderay/scanners/clojure.rb @@ -156,7 +156,7 @@ module CodeRay elsif match = scan(/['`\(\[\)\]\{\}]|\#[({]|~@?|[@\^]/) encoder.text_token match, :operator elsif match = scan(/;.*/) - encoder.text_token match, :comment # FIXME: recognize (comment ...) too + encoder.text_token match, :comment # TODO: recognize (comment ...) too elsif match = scan(/\#?\\(?:newline|space|.?)/) encoder.text_token match, :char elsif match = scan(/\#[ft]/) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 6413f8f..e5f03f5 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -2,9 +2,9 @@ module CodeRay module Scanners class CSS < Scanner - + register_for :css - + KINDS_NOT_LOC = [ :comment, :class, :pseudo_class, :type, @@ -20,28 +20,28 @@ module Scanners NMChar = /[-_a-zA-Z0-9]|#{Escape}/ NMStart = /[_a-zA-Z]|#{Escape}/ NL = /\r\n|\r|\n|\f/ - String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # FIXME: buggy regexp - String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # FIXME: buggy regexp + String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # TODO: buggy regexp + String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # TODO: buggy regexp String = /#{String1}|#{String2}/ - + HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/ Color = /#{HexColor}/ - + Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/ Name = /#{NMChar}+/ Ident = /-?#{NMStart}#{NMChar}*/ AtKeyword = /@#{Ident}/ Percentage = /#{Num}%/ - + reldimensions = %w[em ex px] absdimensions = %w[in cm mm pt pc] Unit = Regexp.union(*(reldimensions + absdimensions)) - + Dimension = /#{Num}#{Unit}/ - + Comment = %r! /\* (?: .*? \*/ | .* ) !mx Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/ - + Id = /##{Name}/ Class = /\.#{Name}/ PseudoClass = /:#{Name}/ @@ -64,20 +64,26 @@ module Scanners when :initial, :media if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox) encoder.text_token match, :type + next elsif match = scan(RE::Class) encoder.text_token match, :class + next elsif match = scan(RE::Id) encoder.text_token match, :constant + next elsif match = scan(RE::PseudoClass) encoder.text_token match, :pseudo_class + next elsif match = scan(RE::AttributeSelector) # TODO: Improve highlighting inside of attribute selectors. encoder.text_token match[0,1], :operator encoder.text_token match[1..-2], :attribute_name if match.size > 2 encoder.text_token match[-1,1], :operator if match[-1] == ?] + next elsif match = scan(/@media/) encoder.text_token match, :directive states.push :media_before_name + next end when :block @@ -87,18 +93,21 @@ module Scanners else encoder.text_token match, :key end + next end - + when :media_before_name if match = scan(RE::Ident) encoder.text_token match, :type states[-1] = :media_after_name + next end when :media_after_name if match = scan(/\{/) encoder.text_token match, :operator states[-1] = :media + next end else @@ -110,12 +119,12 @@ module Scanners elsif match = scan(/\/\*(?:.*?\*\/|\z)/m) encoder.text_token match, :comment - + elsif match = scan(/\{/) value_expected = false encoder.text_token match, :operator states.push :block - + elsif match = scan(/\}/) value_expected = false if states.last == :block || states.last == :media @@ -124,14 +133,14 @@ module Scanners else encoder.text_token match, :error end - + elsif match = scan(/#{RE::String}/o) encoder.begin_group :string encoder.text_token match[0, 1], :delimiter encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1, 1], :delimiter if match.size >= 2 encoder.end_group :string - + elsif match = scan(/#{RE::Function}/o) encoder.begin_group :string start = match[/^\w+\(/] @@ -143,22 +152,22 @@ module Scanners encoder.text_token match[start.size..-1], :content end encoder.end_group :string - + elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox) encoder.text_token match, :float - + elsif match = scan(/#{RE::Color}/o) encoder.text_token match, :color - + elsif match = scan(/! *important/) encoder.text_token match, :important - + elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/) encoder.text_token match, :color - + elsif match = scan(RE::AtKeyword) encoder.text_token match, :directive - + elsif match = scan(/ [+>:;,.=()\/] /x) if match == ':' value_expected = true @@ -166,18 +175,18 @@ module Scanners value_expected = false end encoder.text_token match, :operator - + else encoder.text_token getch, :error - + end - + end - + encoder end - + end - + end end diff --git a/lib/coderay/scanners/rhtml.rb b/lib/coderay/scanners/erb.rb index 9bfab5c..eaf3bba 100644 --- a/lib/coderay/scanners/rhtml.rb +++ b/lib/coderay/scanners/erb.rb @@ -5,9 +5,9 @@ module Scanners load :ruby # Scanner for HTML ERB templates. - class RHTML < Scanner + class ERB < Scanner - register_for :rhtml + register_for :erb title 'HTML ERB Template' KINDS_NOT_LOC = HTML::KINDS_NOT_LOC @@ -56,7 +56,7 @@ module Scanners if start_tag[/\A<%#/] encoder.text_token code, :comment else - @ruby_scanner.tokenize code + @ruby_scanner.tokenize code, :tokens => encoder end unless code.empty? encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? encoder.end_group :inline diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index 2334bc6..3501aaf 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -220,7 +220,7 @@ module Scanners encoder.text_token match, :content elsif match = scan(/ \\. /mx) - encoder.text_token match, :content # FIXME: Shouldn't this be :error? + encoder.text_token match, :content # TODO: Shouldn't this be :error? elsif match = scan(/ \\ | \n /x) encoder.end_group state diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 9ebda6f..9063c5e 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -89,7 +89,7 @@ module Scanners end elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim) - # FIXME: scan over nested tags + # TODO: scan over nested tags xml_scanner.tokenize match, :tokens => encoder value_expected = false next diff --git a/lib/coderay/scanners/nitro_xhtml.rb b/lib/coderay/scanners/nitro_xhtml.rb deleted file mode 100644 index b67b60c..0000000 --- a/lib/coderay/scanners/nitro_xhtml.rb +++ /dev/null @@ -1,136 +0,0 @@ -module CodeRay -module Scanners - - load :html - load :ruby - - # Nitro XHTML Scanner - # - # Alias: +nitro+ - class NitroXHTML < Scanner - - register_for :nitro_xhtml - file_extension :xhtml - title 'Nitro XHTML' - - KINDS_NOT_LOC = HTML::KINDS_NOT_LOC - - NITRO_RUBY_BLOCK = / - <\?r - (?> - [^\?]* - (?> \?(?!>) [^\?]* )* - ) - (?: \?> )? - | - <ruby> - (?> - [^<]* - (?> <(?!\/ruby>) [^<]* )* - ) - (?: <\/ruby> )? - | - <% - (?> - [^%]* - (?> %(?!>) [^%]* )* - ) - (?: %> )? - /mx # :nodoc: - - NITRO_VALUE_BLOCK = / - \# - (?: - \{ - [^{}]* - (?> - \{ [^}]* \} - (?> [^{}]* ) - )* - \}? - | \| [^|]* \|? - | \( [^)]* \)? - | \[ [^\]]* \]? - | \\ [^\\]* \\? - ) - /x # :nodoc: - - NITRO_ENTITY = / - % (?: \#\d+ | \w+ ) ; - / # :nodoc: - - START_OF_RUBY = / - (?=[<\#%]) - < (?: \?r | % | ruby> ) - | \# [{(|] - | % (?: \#\d+ | \w+ ) ; - /x # :nodoc: - - CLOSING_PAREN = Hash.new { |h, p| h[p] = p } # :nodoc: - CLOSING_PAREN.update( { - '(' => ')', - '[' => ']', - '{' => '}', - } ) - - protected - - def setup - @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true - @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true - end - - def reset_instance - super - @html_scanner.reset - end - - def scan_tokens encoder, options - - until eos? - - if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_rest) and not match.empty? - @html_scanner.tokenize match - - elsif match = scan(/#{NITRO_VALUE_BLOCK}/o) - start_tag = match[0,2] - delimiter = CLOSING_PAREN[start_tag[1,1]] - end_tag = match[-1,1] == delimiter ? delimiter : '' - encoder.begin_group :inline - encoder.text_token start_tag, :inline_delimiter - code = match[start_tag.size .. -1 - end_tag.size] - @ruby_scanner.tokenize code, :tokens => encoder - encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? - encoder.end_group :inline - - elsif match = scan(/#{NITRO_RUBY_BLOCK}/o) - start_tag = '<?r' - end_tag = match[-2,2] == '?>' ? '?>' : '' - encoder.begin_group :inline - encoder.text_token start_tag, :inline_delimiter - code = match[start_tag.size .. -(end_tag.size)-1] - @ruby_scanner.tokenize code, :tokens => encoder - encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? - encoder.end_group :inline - - elsif entity = scan(/#{NITRO_ENTITY}/o) - encoder.text_token entity, :entity - - elsif scan(/%/) - encoder.text_token matched, :error - - else - raise_inspect 'else-case reached!', encoder - - end - - end - - encoder - - end - - end - -end -end diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index c290dab..b2632a2 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -234,8 +234,8 @@ module Scanners def scan_tokens encoder, options if check(RE::PHP_START) || # starts with <? - (match?(/\s*<\S/) && check(/.{1,100}#{RE::PHP_START}/om)) || # starts with tag and contains <? - check(/.{1,100}#{RE::HTML_INDICATOR}/om) || + (match?(/\s*<\S/) && check(/.{1,1000}#{RE::PHP_START}/om)) || # starts with tag and contains <? + check(/.{0,1000}#{RE::HTML_INDICATOR}/om) || check(/.{1,100}#{RE::PHP_START}/om) # PHP start after max 100 chars # is HTML with embedded PHP, so start with HTML states = [:initial] diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index e1395ca..25da208 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -26,7 +26,7 @@ module Scanners state, heredocs = @state heredocs = heredocs.dup if heredocs.is_a?(Array) - if state && state.instance_of?(self.class::StringState) + if state && state.instance_of?(StringState) encoder.begin_group state.type end @@ -426,13 +426,18 @@ module Scanners end # cleaning up - if options[:keep_state] - heredocs = nil if heredocs && heredocs.empty? - @state = state, heredocs + if state.is_a? StringState + encoder.end_group state.type end - if state.is_a? self.class::StringState - encoder.end_group state.type + if options[:keep_state] + if state.is_a?(StringState) && state.heredoc + (heredocs ||= []).unshift state + state = :initial + elsif heredocs && heredocs.empty? + heredocs = nil + end + @state = state, heredocs end if inline_block_stack diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index f6f8845..1566276 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -73,9 +73,7 @@ module CodeRay encoder.encode_tokens self, options end - # Turn into a string using Encoders::Text. - # - # +options+ are passed to the encoder if given. + # Turn tokens into a string by concatenating them. def to_s encode CodeRay::Encoders::Encoder.new end |