diff options
Diffstat (limited to 'lib/coderay/encoders')
-rw-r--r-- | lib/coderay/encoders/count.rb | 44 | ||||
-rw-r--r-- | lib/coderay/encoders/debug.rb | 36 | ||||
-rw-r--r-- | lib/coderay/encoders/filter.rb | 26 | ||||
-rw-r--r-- | lib/coderay/encoders/html.rb | 171 | ||||
-rw-r--r-- | lib/coderay/encoders/json.rb | 18 | ||||
-rw-r--r-- | lib/coderay/encoders/lines_of_code.rb | 6 | ||||
-rw-r--r-- | lib/coderay/encoders/statistic.rb | 85 | ||||
-rw-r--r-- | lib/coderay/encoders/terminal.rb | 95 | ||||
-rw-r--r-- | lib/coderay/encoders/text.rb | 8 | ||||
-rw-r--r-- | lib/coderay/encoders/token_kind_filter.rb | 30 | ||||
-rw-r--r-- | lib/coderay/encoders/xml.rb | 12 |
11 files changed, 338 insertions, 193 deletions
diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb index 2e60a89..451a7f8 100644 --- a/lib/coderay/encoders/count.rb +++ b/lib/coderay/encoders/count.rb @@ -1,25 +1,55 @@ +($:.unshift '../..'; require 'coderay') unless defined? CodeRay module CodeRay module Encoders # Returns the number of tokens. # - # Text and block tokens (:open etc.) are counted. + # Text and block tokens are counted. class Count < Encoder - + include Streamable register_for :count - + protected - + def setup options @out = 0 end - - def token text, kind + + def text_token text, kind + @out += 1 + end + + def begin_group kind @out += 1 end + alias end_group begin_group + alias begin_line begin_group + alias end_line begin_group end - + end end + +if $0 == __FILE__ + $VERBOSE = true + $: << File.join(File.dirname(__FILE__), '..') + eval DATA.read, nil, $0, __LINE__ + 4 +end + +__END__ +require 'test/unit' + +class CountTest < Test::Unit::TestCase + + def test_count + tokens = CodeRay.scan <<-RUBY.strip, :ruby +#!/usr/bin/env ruby +# a minimal Ruby program +puts "Hello world!" + RUBY + assert_equal 9, tokens.encode_with(:count) + end + +end
\ No newline at end of file diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 4c680d3..89e430f 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -19,31 +19,43 @@ module Encoders register_for :debug FILE_EXTENSION = 'raydebug' + + def initialize options = {} + super + @opened = [] + end - protected + public + def text_token text, kind if kind == :space - text + @out << text else text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \ - "#{kind}(#{text})" + @out << kind.to_s << '(' << text << ')' end end - def open_token kind - "#{kind}<" + def begin_group kind + @opened << kind + @out << kind.to_s << '<' end - def close_token kind - '>' + def end_group kind + if @opened.last != kind + puts @out + raise "we are inside #{@opened.inspect}, not #{kind}" + end + @opened.pop + @out << '>' end def begin_line kind - "#{kind}[" + @out << kind.to_s << '[' end def end_line kind - ']' + @out << ']' end end @@ -74,16 +86,16 @@ class DebugEncoderTest < Test::Unit::TestCase TEST_INPUT = CodeRay::Tokens[ ['10', :integer], ['(\\)', :operator], - [:open, :string], + [:begin_group, :string], ['test', :content], - [:close, :string], + [:end_group, :string], [:begin_line, :test], ["\n", :space], ["\n \t", :space], [" \n", :space], ["[]", :method], [:end_line, :test], - ] + ].flatten TEST_OUTPUT = <<-'DEBUG'.chomp integer(10)operator((\\\))string<content(test)>test[ diff --git a/lib/coderay/encoders/filter.rb b/lib/coderay/encoders/filter.rb index c1991cf..6b78ad3 100644 --- a/lib/coderay/encoders/filter.rb +++ b/lib/coderay/encoders/filter.rb @@ -16,15 +16,27 @@ module Encoders end def text_token text, kind - [text, kind] if include_text_token? text, kind + @out.text_token text, kind if include_text_token? text, kind end def include_text_token? text, kind true end - def block_token action, kind - [action, kind] if include_block_token? action, kind + def begin_group kind + @out.begin_group kind if include_block_token? :begin_group, kind + end + + def end_group kind + @out.end_group kind if include_block_token? :end_group, kind + end + + def begin_line kind + @out.begin_line kind if include_block_token? :begin_line, kind + end + + def end_line kind + @out.end_line kind if include_block_token? :end_line, kind end def include_block_token? action, kind @@ -59,7 +71,7 @@ class FilterTest < Test::Unit::TestCase def test_filtering_text_tokens tokens = CodeRay::Tokens.new 10.times do |i| - tokens << [i.to_s, :index] + tokens.text_token i.to_s, :index end assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) assert_equal tokens, tokens.filter @@ -68,9 +80,9 @@ class FilterTest < Test::Unit::TestCase def test_filtering_block_tokens tokens = CodeRay::Tokens.new 10.times do |i| - tokens << [:open, :index] - tokens << [i.to_s, :content] - tokens << [:close, :index] + tokens.begin_group :index + tokens.text_token i.to_s, :content + tokens.end_group :index end assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) assert_equal tokens, tokens.filter diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index dcdffa1..807fb42 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -83,7 +83,7 @@ module Encoders # # === :hint # Include some information into the output using the title attribute. - # Can be :info (show token type on mouse-over), :info_long (with full path) + # Can be :info (show token kind on mouse-over), :info_long (with full path) # or :debug (via inspect). # # Default: false @@ -153,12 +153,18 @@ module Encoders # # +hint+ may be :info, :info_long or :debug. def self.token_path_to_hint hint, kinds + # FIXME: TRANSPARENT_TOKEN_KINDS? + # if TRANSPARENT_TOKEN_KINDS.include? kinds.first + # kinds = kinds[1..-1] + # else + # kinds = kinds[1..-1] + kinds.first + # end title = case hint when :info TOKEN_KIND_TO_INFO[kinds.first] when :info_long - kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/') + kinds.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/') when :debug kinds.inspect end @@ -167,13 +173,13 @@ module Encoders def setup options super - + @HTML_ESCAPE = HTML_ESCAPE.dup @HTML_ESCAPE["\t"] = ' ' * options[:tab_width] - + @opened = [nil] @css = CSS.new options[:style] - + hint = options[:hint] if hint and not [:debug, :info, :info_long].include? hint raise ArgumentError, "Unknown value %p for :hint; \ @@ -184,45 +190,33 @@ module Encoders when :class @css_style = Hash.new do |h, k| - c = CodeRay::Tokens::AbbreviationForKind[k.first] - if c == :NO_HIGHLIGHT and not hint - h[k.dup] = false - else - title = if hint - HTML.token_path_to_hint(hint, k[1..-1] << k.first) - else - '' - end - if c == :NO_HIGHLIGHT - h[k.dup] = '<span%s>' % [title] - else - h[k.dup] = '<span%s class="%s">' % [title, c] + c = Tokens::AbbreviationForKind[k.first] + h[k.dup] = + if c != :NO_HIGHLIGHT or hint + if hint + title = HTML.token_path_to_hint hint, k + end + if c == :NO_HIGHLIGHT + '<span%s>' % [title] + else + '<span%s class="%s">' % [title, c] + end end - end end when :style @css_style = Hash.new do |h, k| - if k.is_a? ::Array - styles = k.dup - else - styles = [k] - end - type = styles.first - classes = styles.map { |c| Tokens::AbbreviationForKind[c] } - if classes.first == :NO_HIGHLIGHT and not hint - h[k] = false - else - styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first - title = HTML.token_path_to_hint hint, styles - style = @css[*classes] - h[k] = + classes = k.map { |c| Tokens::AbbreviationForKind[c] } + h[k.dup] = + if classes.first != :NO_HIGHLIGHT or hint + if hint + title = HTML.token_path_to_hint hint, k + end + style = @css[*classes] if style '<span%s style="%s">' % [title, style] - else - false end - end + end end else @@ -233,80 +227,81 @@ module Encoders def finish options not_needed = @opened.shift - @out << '</span>' * @opened.size unless @opened.empty? warn '%d tokens still open: %p' % [@opened.size, @opened] + @out << '</span>' * @opened.size end - + @out.extend Output @out.css = @css @out.numerize! options[:line_numbers], options @out.wrap! options[:wrap] @out.apply_title! options[:title] - + super end - - def token text, type - case text - - when nil - # raise 'Token with nil as text was given: %p' % [[text, type]] - - when String - if text =~ /#{HTML_ESCAPE_PATTERN}/o - text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } - end - @opened[0] = type - if text != "\n" && style = @css_style[@opened] - @out << style << text << '</span>' + + public + + def text_token text, kind + if text =~ /#{HTML_ESCAPE_PATTERN}/o + text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } + end + @opened[0] = kind + @out << + if style = @css_style[@opened] + style + text + '</span>' else - @out << text - end - - - # token groups, eg. strings - when :open - @opened[0] = type - @out << (@css_style[@opened] || '<span>') - @opened << type - when :close - if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type) - warn 'Malformed token stream: Trying to close a token (%p) ' \ - 'that is not open. Open are: %p.' % [type, @opened[1..-1]] + text end + end + + # token groups, eg. strings + def begin_group kind + @opened[0] = kind + @opened << kind + @out << (@css_style[@opened] || '<span>') + end + + def end_group kind + if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != kind) + warn 'Malformed token stream: Trying to close a token (%p) ' \ + 'that is not open. Open are: %p.' % [kind, @opened[1..-1]] + end + @out << if @opened.empty? - # nothing to close + '' # nothing to close else - @out << '</span>' @opened.pop + '</span>' end - - # whole lines to be highlighted, eg. a deleted line in a diff - when :begin_line - @opened[0] = type - if style = @css_style[@opened] - @out << style.sub('<span', '<div') + end + + # whole lines to be highlighted, eg. a deleted line in a diff + def begin_line kind + @opened[0] = kind + style = @css_style[@opened] + @opened << kind + @out << + if style + style.sub '<span', '<div' else - @out << '<div>' - end - @opened << type - when :end_line - if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type) - warn 'Malformed token stream: Trying to close a line (%p) ' \ - 'that is not open. Open are: %p.' % [type, @opened[1..-1]] + '<div>' end + end + + def end_line kind + if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != kind) + warn 'Malformed token stream: Trying to close a line (%p) ' \ + 'that is not open. Open are: %p.' % [kind, @opened[1..-1]] + end + @out << if @opened.empty? - # nothing to close + '' # nothing to close else - @out << '</div>' @opened.pop + '</div>' end - - else - raise 'unknown token kind: %p' % [text] - - end end end diff --git a/lib/coderay/encoders/json.rb b/lib/coderay/encoders/json.rb index 78f0ec0..bb09809 100644 --- a/lib/coderay/encoders/json.rb +++ b/lib/coderay/encoders/json.rb @@ -33,11 +33,23 @@ module Encoders end def text_token text, kind - { :type => 'text', :text => text, :kind => kind } + @out << { :type => 'text', :text => text, :kind => kind } end - def block_token action, kind - { :type => 'block', :action => action, :kind => kind } + def begin_group kind + @out << { :type => 'block', :action => 'open', :kind => kind } + end + + def end_group kind + @out << { :type => 'block', :action => 'close', :kind => kind } + end + + def begin_line kind + @out << { :type => 'block', :action => 'begin_line', :kind => kind } + end + + def end_line kind + @out << { :type => 'block', :action => 'end_line', :kind => kind } end def finish options diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb index c6ed4de..6b36aef 100644 --- a/lib/coderay/encoders/lines_of_code.rb +++ b/lib/coderay/encoders/lines_of_code.rb @@ -79,9 +79,9 @@ puts "Hello world!" def test_filtering_block_tokens tokens = CodeRay::Tokens.new - tokens << ["Hello\n", :world] - tokens << ["Hello\n", :space] - tokens << ["Hello\n", :comment] + tokens.concat ["Hello\n", :world] + tokens.concat ["Hello\n", :space] + tokens.concat ["Hello\n", :comment] assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens) assert_equal 2, tokens.lines_of_code assert_equal 2, tokens.loc diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb index 1b38938..d267b21 100644 --- a/lib/coderay/encoders/statistic.rb +++ b/lib/coderay/encoders/statistic.rb @@ -1,3 +1,4 @@ +($:.unshift '../..'; require 'coderay') unless defined? CodeRay module CodeRay module Encoders @@ -34,9 +35,25 @@ module Encoders end # TODO Hierarchy handling - def block_token action, kind + def begin_group kind + block_token 'begin_group' + end + + def end_group kind + block_token 'end_group' + end + + def begin_line kind + block_token 'begin_line' + end + + def end_line kind + block_token 'end_line' + end + + def block_token action @type_stats['TOTAL'].count += 1 - @type_stats['open/close'].count += 1 + @type_stats[action].count += 1 end STATS = <<-STATS # :nodoc: @@ -77,3 +94,67 @@ Token Types (%d): end end + +if $0 == __FILE__ + $VERBOSE = true + $: << File.join(File.dirname(__FILE__), '..') + eval DATA.read, nil, $0, __LINE__ + 4 +end + +__END__ +require 'test/unit' + +class StatisticEncoderTest < Test::Unit::TestCase + + def test_creation + assert CodeRay::Encoders::Statistic < CodeRay::Encoders::Encoder + stats = nil + assert_nothing_raised do + stats = CodeRay.encoder :statistic + end + assert_kind_of CodeRay::Encoders::Encoder, stats + end + + TEST_INPUT = CodeRay::Tokens[ + ['10', :integer], + ['(\\)', :operator], + [:begin_group, :string], + ['test', :content], + [:end_group, :string], + [:begin_line, :test], + ["\n", :space], + ["\n \t", :space], + [" \n", :space], + ["[]", :method], + [:end_line, :test], + ].flatten + TEST_OUTPUT = <<-'DEBUG' + +Code Statistics + +Tokens 11 + Non-Whitespace 4 +Bytes Total 20 + +Token Types (5): + type count ratio size (average) +------------------------------------------------------------- + TOTAL 11 100.00 % 1.8 + space 3 27.27 % 3.0 + begin_group 1 9.09 % 0.0 + begin_line 1 9.09 % 0.0 + content 1 9.09 % 4.0 + end_group 1 9.09 % 0.0 + end_line 1 9.09 % 0.0 + integer 1 9.09 % 2.0 + method 1 9.09 % 2.0 + operator 1 9.09 % 3.0 + + DEBUG + + def test_filtering_text_tokens + assert_equal TEST_OUTPUT, CodeRay::Encoders::Statistic.new.encode_tokens(TEST_INPUT) + assert_equal TEST_OUTPUT, TEST_INPUT.statistic + end + +end
\ No newline at end of file diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb index 7224218..3a774a0 100644 --- a/lib/coderay/encoders/terminal.rb +++ b/lib/coderay/encoders/terminal.rb @@ -92,41 +92,72 @@ module CodeRay TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved] TOKEN_COLORS[:method] = TOKEN_COLORS[:function] TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex] - TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] + TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] = + TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = + TOKEN_COLORS[:delimiter] protected def setup(options) super @opened = [] + @subcolors = nil end - - def finish(options) - super - end - - def text_token text, type - if color = (@subcolors || TOKEN_COLORS)[type] + + public + + def text_token text, kind + if color = (@subcolors || TOKEN_COLORS)[kind] if Hash === color if color[:self] color = color[:self] else - return text + @out << text + return end end - - out = ansi_colorize(color) - out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color)) - out << ansi_clear - out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self] - out + + @out << ansi_colorize(color) + @out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color)) + @out << ansi_clear + @out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self] else - text + @out << text end end - def open_token type - if color = TOKEN_COLORS[type] + def begin_group kind + @opened << kind + @out << open_token(kind) + end + alias begin_line begin_group + + def end_group kind + if @opened.empty? + # nothing to close + else + @opened.pop + @out << ansi_clear + @out << open_token(@opened.last) + end + end + + def end_line kind + if @opened.empty? + # nothing to close + else + @opened.pop + # whole lines to be highlighted, + # eg. added/modified/deleted lines in a diff + @out << "\t" * 100 + ansi_clear + @out << open_token(@opened.last) + end + end + + private + + def open_token kind + if color = TOKEN_COLORS[kind] if Hash === color @subcolors = color ansi_colorize(color[:self]) if color[:self] @@ -140,34 +171,6 @@ module CodeRay end end - def block_token action, type - case action - - when :open, :begin_line - @opened << type - open_token type - when :close, :end_line - if @opened.empty? - # nothing to close - else - @opened.pop - if action == :end_line - # whole lines to be highlighted, - # eg. added/modified/deleted lines in a diff - "\t" * 100 + ansi_clear - else - ansi_clear - end + - open_token(@opened.last) - end - - else - raise 'unknown token kind: %p' % [text] - end - end - - private - def ansi_colorize(color) Array(color).map { |c| "\e[#{c}m" }.join end diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb index 26fef84..ecbf624 100644 --- a/lib/coderay/encoders/text.rb +++ b/lib/coderay/encoders/text.rb @@ -23,16 +23,16 @@ module Encoders :separator => '' } + def text_token text, kind + @out << text + @sep + end + protected def setup options super @sep = options[:separator] end - def text_token text, kind - text + @sep - end - def finish options super.chomp @sep end diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb index 4b2f582..fd3df44 100644 --- a/lib/coderay/encoders/token_kind_filter.rb +++ b/lib/coderay/encoders/token_kind_filter.rb @@ -76,28 +76,28 @@ class TokenKindFilterTest < Test::Unit::TestCase def test_filtering_text_tokens tokens = CodeRay::Tokens.new for i in 1..10 - tokens << [i.to_s, :index] - tokens << [' ', :space] if i < 10 + tokens.text_token i.to_s, :index + tokens.text_token ' ', :space if i < 10 end - assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).size - assert_equal 10, tokens.token_kind_filter(:exclude => :space).size - assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).size - assert_equal 9, tokens.token_kind_filter(:include => :space).size - assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).size - assert_equal 0, tokens.token_kind_filter(:exclude => :all).size + assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).count + assert_equal 10, tokens.token_kind_filter(:exclude => :space).count + assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).count + assert_equal 9, tokens.token_kind_filter(:include => :space).count + assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).count + assert_equal 0, tokens.token_kind_filter(:exclude => :all).count end def test_filtering_block_tokens tokens = CodeRay::Tokens.new 10.times do |i| - tokens << [:open, :index] - tokens << [i.to_s, :content] - tokens << [:close, :index] + tokens.begin_group :index + tokens.text_token i.to_s, :content + tokens.end_group :index end - assert_equal 20, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).size - assert_equal 20, tokens.token_kind_filter(:include => :blubb).size - assert_equal 30, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).size - assert_equal 30, tokens.token_kind_filter(:exclude => :index).size + assert_equal 20, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).count + assert_equal 20, tokens.token_kind_filter(:include => :blubb).count + assert_equal 30, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).count + assert_equal 30, tokens.token_kind_filter(:exclude => :index).count end end diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb index f32c967..0006d75 100644 --- a/lib/coderay/encoders/xml.rb +++ b/lib/coderay/encoders/xml.rb @@ -53,19 +53,19 @@ module Encoders end end end - - def open_token kind + + def begin_group kind @node = @node.add_element kind.to_s end - - def close_token kind + + def end_group kind if @node == @root raise 'no token to close!' end @node = @node.parent end - + end - + end end |