From 476bc1bfe2f2000acc665f034e7da19bd3f0cc6b Mon Sep 17 00:00:00 2001 From: murphy Date: Tue, 30 Mar 2010 01:10:25 +0000 Subject: Renamed token_classes to token_kinds (closes #122). * *RENAMED* from ClassOfKind. The term "token class" is no longer used in CodeRay. Instead, tokens have _kinds_. * *RENAMED* TokenKindFilter to TokenClassFilter. * *ADDED* documentation on encoders affected by the term change. * *REMOVED* token kinds :attribute_name_fat, :attribute_value_fat, :operator_fat, :tag_fat, and :xml_text. * *ADDED* token kind :filename. --- lib/coderay/encoders/comment_filter.rb | 13 +++- lib/coderay/encoders/html.rb | 35 +++++----- lib/coderay/encoders/lines_of_code.rb | 6 +- lib/coderay/encoders/token_class_filter.rb | 84 ----------------------- lib/coderay/encoders/token_kind_filter.rb | 103 +++++++++++++++++++++++++++++ lib/coderay/styles/cycnus.rb | 10 ++- lib/coderay/styles/murphy.rb | 7 +- lib/coderay/token_classes.rb | 86 ------------------------ lib/coderay/token_kinds.rb | 82 +++++++++++++++++++++++ 9 files changed, 223 insertions(+), 203 deletions(-) delete mode 100644 lib/coderay/encoders/token_class_filter.rb create mode 100644 lib/coderay/encoders/token_kind_filter.rb delete mode 100755 lib/coderay/token_classes.rb create mode 100755 lib/coderay/token_kinds.rb (limited to 'lib/coderay') diff --git a/lib/coderay/encoders/comment_filter.rb b/lib/coderay/encoders/comment_filter.rb index 4d3fb54..819c619 100644 --- a/lib/coderay/encoders/comment_filter.rb +++ b/lib/coderay/encoders/comment_filter.rb @@ -2,9 +2,18 @@ module CodeRay module Encoders - load :token_class_filter + load :token_kind_filter - class CommentFilter < TokenClassFilter + # A simple Filter that removes all tokens of the :comment kind. + # + # Alias: +remove_comments+ + # + # Usage: + # CodeRay.scan('print # foo', :ruby).remove_comments.text + # #-> "print " + # + # See also: TokenKindFilter, LinesOfCode + class CommentFilter < TokenKindFilter register_for :comment_filter diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ffbc6ca..f1fe7ce 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -27,6 +27,7 @@ module Encoders # # === :tab_width # Convert \t characters to +n+ spaces (a number.) + # # Default: 8 # # === :css @@ -125,7 +126,7 @@ module Encoders #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/ HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/ - TOKEN_KIND_TO_INFO = Hash.new { |h, kind| + TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = case kind when :pre_constant @@ -133,24 +134,24 @@ module Encoders else kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } end - } + end TRANSPARENT_TOKEN_KINDS = [ :delimiter, :modifier, :content, :escape, :inline_delimiter, ].to_set - # Generate a hint about the given +classes+ in a +hint+ style. + # Generate a hint about the given +kinds+ in a +hint+ style. # # +hint+ may be :info, :info_long or :debug. - def self.token_path_to_hint hint, classes + def self.token_path_to_hint hint, kinds title = case hint when :info - TOKEN_KIND_TO_INFO[classes.first] + TOKEN_KIND_TO_INFO[kinds.first] when :info_long - classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/') + kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/') when :debug - classes.inspect + kinds.inspect end title ? " title=\"#{title}\"" : '' end @@ -174,7 +175,7 @@ module Encoders when :class @css_style = Hash.new do |h, k| - c = CodeRay::Tokens::ClassOfKind[k.first] + c = CodeRay::Tokens::AbbreviationForKind[k.first] if c == :NO_HIGHLIGHT and not hint h[k.dup] = false else @@ -199,7 +200,7 @@ module Encoders styles = [k] end type = styles.first - classes = styles.map { |c| Tokens::ClassOfKind[c] } + classes = styles.map { |c| Tokens::AbbreviationForKind[c] } if classes.first == :NO_HIGHLIGHT and not hint h[k] = false else @@ -261,13 +262,13 @@ module Encoders @out << (@css_style[@opened] || '') @opened << type when :close + if $CODERAY_DEBUG and @opened.last != type + warn 'Malformed token stream: Trying to close a token (%p) ' \ + 'that is not open. Open are: %p.' % [type, @opened[1..-1]] + end if @opened.empty? # nothing to close else - if $DEBUG and (@opened.size == 1 or @opened.last != type) - raise 'Malformed token stream: Trying to close a token (%p) \ - that is not open. Open are: %p.' % [type, @opened[1..-1]] - end @out << '' @opened.pop end @@ -282,13 +283,13 @@ module Encoders end @opened << type when :end_line + if $CODERAY_DEBUG and @opened.last != type + warn 'Malformed token stream: Trying to close a line (%p) ' \ + 'that is not open. Open are: %p.' % [type, @opened[1..-1]] + end if @opened.empty? # nothing to close else - if $DEBUG and (@opened.size == 1 or @opened.last != type) - raise 'Malformed token stream: Trying to close a line (%p) \ - that is not open. Open are: %p.' % [type, @opened[1..-1]] - end @out << '' @opened.pop end diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb index b24145e..60918c4 100644 --- a/lib/coderay/encoders/lines_of_code.rb +++ b/lib/coderay/encoders/lines_of_code.rb @@ -4,7 +4,7 @@ module Encoders # Counts the LoC (Lines of Code). Returns an Integer >= 0. # - # Alias: :loc + # Alias: +loc+ # # Everything that is not comment, markup, doctype/shebang, or an empty line, # is considered to be code. @@ -25,10 +25,10 @@ module Encoders if scanner = tokens.scanner kinds_not_loc = scanner.class::KINDS_NOT_LOC else - warn ArgumentError, 'Tokens have no scanner.' if $DEBUG + warn ArgumentError, 'Tokens have no scanner.' if $VERBOSE kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC end - code = tokens.token_class_filter :exclude => kinds_not_loc + code = tokens.token_kind_filter :exclude => kinds_not_loc @loc = code.text.scan(NON_EMPTY_LINE).size end diff --git a/lib/coderay/encoders/token_class_filter.rb b/lib/coderay/encoders/token_class_filter.rb deleted file mode 100644 index a9e8673..0000000 --- a/lib/coderay/encoders/token_class_filter.rb +++ /dev/null @@ -1,84 +0,0 @@ -($:.unshift '../..'; require 'coderay') unless defined? CodeRay -module CodeRay -module Encoders - - load :filter - - class TokenClassFilter < Filter - - include Streamable - register_for :token_class_filter - - DEFAULT_OPTIONS = { - :exclude => [], - :include => :all - } - - protected - def setup options - super - @exclude = options[:exclude] - @exclude = Array(@exclude) unless @exclude == :all - @include = options[:include] - @include = Array(@include) unless @include == :all - end - - def include_text_token? text, kind - (@include == :all || @include.include?(kind)) && - !(@exclude == :all || @exclude.include?(kind)) - end - - end - -end -end - -if $0 == __FILE__ - $VERBOSE = true - $: << File.join(File.dirname(__FILE__), '..') - eval DATA.read, nil, $0, __LINE__ + 4 -end - -__END__ -require 'test/unit' - -class TokenClassFilterTest < Test::Unit::TestCase - - def test_creation - assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder - assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter - filter = nil - assert_nothing_raised do - filter = CodeRay.encoder :token_class_filter - end - assert_instance_of CodeRay::Encoders::TokenClassFilter, filter - end - - def test_filtering_text_tokens - tokens = CodeRay::Tokens.new - for i in 1..10 - tokens << [i.to_s, :index] - tokens << [' ', :space] if i < 10 - end - assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size - assert_equal 10, tokens.token_class_filter(:exclude => :space).size - assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size - assert_equal 9, tokens.token_class_filter(:include => :space).size - assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size - assert_equal 0, tokens.token_class_filter(:exclude => :all).size - end - - def test_filtering_block_tokens - tokens = CodeRay::Tokens.new - 10.times do |i| - tokens << [:open, :index] - tokens << [i.to_s, :content] - tokens << [:close, :index] - end - assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size - assert_equal 20, tokens.token_class_filter(:include => :blubb).size - assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size - assert_equal 30, tokens.token_class_filter(:exclude => :index).size - end - -end diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb new file mode 100644 index 0000000..4b2f582 --- /dev/null +++ b/lib/coderay/encoders/token_kind_filter.rb @@ -0,0 +1,103 @@ +($:.unshift '../..'; require 'coderay') unless defined? CodeRay +module CodeRay +module Encoders + + load :filter + + # A Filter that selects tokens based on their token kind. + # + # == Options + # + # === :exclude + # + # One or many symbols (in an Array) which shall be excluded. + # + # Default: [] + # + # === :include + # + # One or many symbols (in an array) which shall be included. + # + # Default: :all, which means all tokens are included. + # + # Exclusion wins over inclusion. + # + # See also: CommentFilter + class TokenKindFilter < Filter + + include Streamable + register_for :token_kind_filter + + DEFAULT_OPTIONS = { + :exclude => [], + :include => :all + } + + protected + def setup options + super + @exclude = options[:exclude] + @exclude = Array(@exclude) unless @exclude == :all + @include = options[:include] + @include = Array(@include) unless @include == :all + end + + def include_text_token? text, kind + (@include == :all || @include.include?(kind)) && + !(@exclude == :all || @exclude.include?(kind)) + end + + end + +end +end + +if $0 == __FILE__ + $VERBOSE = true + $: << File.join(File.dirname(__FILE__), '..') + eval DATA.read, nil, $0, __LINE__ + 4 +end + +__END__ +require 'test/unit' + +class TokenKindFilterTest < Test::Unit::TestCase + + def test_creation + assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Encoder + assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Filter + filter = nil + assert_nothing_raised do + filter = CodeRay.encoder :token_kind_filter + end + assert_instance_of CodeRay::Encoders::TokenKindFilter, filter + end + + def test_filtering_text_tokens + tokens = CodeRay::Tokens.new + for i in 1..10 + tokens << [i.to_s, :index] + tokens << [' ', :space] if i < 10 + end + assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).size + assert_equal 10, tokens.token_kind_filter(:exclude => :space).size + assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).size + assert_equal 9, tokens.token_kind_filter(:include => :space).size + assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).size + assert_equal 0, tokens.token_kind_filter(:exclude => :all).size + end + + def test_filtering_block_tokens + tokens = CodeRay::Tokens.new + 10.times do |i| + tokens << [:open, :index] + tokens << [i.to_s, :content] + tokens << [:close, :index] + end + assert_equal 20, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).size + assert_equal 20, tokens.token_kind_filter(:include => :blubb).size + assert_equal 30, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).size + assert_equal 30, tokens.token_kind_filter(:exclude => :index).size + end + +end diff --git a/lib/coderay/styles/cycnus.rb b/lib/coderay/styles/cycnus.rb index d2a4353..8196c9f 100644 --- a/lib/coderay/styles/cycnus.rb +++ b/lib/coderay/styles/cycnus.rb @@ -1,6 +1,7 @@ module CodeRay module Styles - + + # A colorful theme that is also the default for CodeRay output. class Cycnus < Style register_for :cycnus @@ -10,7 +11,7 @@ module Styles border_color = 'silver' normal_color = '#000' - CSS_MAIN_STYLES = <<-MAIN + CSS_MAIN_STYLES = <<-MAIN # :nodoc: .CodeRay { background-color: #{code_background}; border: 1px solid #{border_color}; @@ -45,11 +46,9 @@ ol.CodeRay li { white-space: pre } TOKEN_COLORS = <<-'TOKENS' .debug { color:white ! important; background:blue ! important; } -.af { color:#00C } .an { color:#007 } .at { color:#f08 } .av { color:#700 } -.aw { color:#C00 } .bi { color:#509; font-weight:bold } .c { color:#888; } @@ -92,7 +91,6 @@ ol.CodeRay li { white-space: pre } .la { color:#970; font-weight:bold } .lv { color:#963 } .oc { color:#40E; font-weight:bold } -.of { color:#000; font-weight:bold } .op { } .pc { color:#038; font-weight:bold } .pd { color:#369; font-weight:bold } @@ -128,7 +126,6 @@ ol.CodeRay li { white-space: pre } .sy .dl { color:#630 } .ta { color:#070 } -.tf { color:#070; font-weight:bold } .ts { color:#D70; font-weight:bold } .ty { color:#339; font-weight:bold } .v { color:#036 } @@ -138,6 +135,7 @@ ol.CodeRay li { white-space: pre } .del { background: #faa; } .chg { color: #aaf; background: #007; } .head { color: #f8f; background: #505 } +.head .filename { color: white; } .ins .ins { color: #080; font-weight:bold } .del .del { color: #800; font-weight:bold } diff --git a/lib/coderay/styles/murphy.rb b/lib/coderay/styles/murphy.rb index 211ab6b..033d949 100644 --- a/lib/coderay/styles/murphy.rb +++ b/lib/coderay/styles/murphy.rb @@ -1,6 +1,7 @@ module CodeRay module Styles + # A alternative color theme. class Murphy < Style register_for :murphy @@ -10,7 +11,7 @@ module Styles border_color = 'silver' normal_color = '#C0C0C0' - CSS_MAIN_STYLES = <<-MAIN + CSS_MAIN_STYLES = <<-MAIN # :nodoc: .CodeRay { background-color: #{code_background}; border: 1px solid #{border_color}; @@ -42,10 +43,8 @@ ol.CodeRay li { white-space: pre; } MAIN TOKEN_COLORS = <<-'TOKENS' -.af { color:#00C; } .an { color:#007; } .av { color:#700; } -.aw { color:#C00; } .bi { color:#509; font-weight:bold; } .c { color:#555; background-color: black; } @@ -77,7 +76,6 @@ ol.CodeRay li { white-space: pre; } .la { color:#970; font-weight:bold; } .lv { color:#963; } .oc { color:#40E; font-weight:bold; } -.of { color:#000; font-weight:bold; } .op { } .pc { color:#08f; font-weight:bold; } .pd { color:#369; font-weight:bold; } @@ -109,7 +107,6 @@ ol.CodeRay li { white-space: pre; } .sy .dl { color:#F84; } .ta { color:#070; } -.tf { color:#070; font-weight:bold; } .ts { color:#D70; font-weight:bold; } .ty { color:#339; font-weight:bold; } .v { color:#036; } diff --git a/lib/coderay/token_classes.rb b/lib/coderay/token_classes.rb deleted file mode 100755 index ae35c0f..0000000 --- a/lib/coderay/token_classes.rb +++ /dev/null @@ -1,86 +0,0 @@ -module CodeRay - class Tokens - ClassOfKind = Hash.new do |h, k| - h[k] = k.to_s - end - ClassOfKind.update with = { - :annotation => 'at', - :attribute_name => 'an', - :attribute_name_fat => 'af', - :attribute_value => 'av', - :attribute_value_fat => 'aw', - :bin => 'bi', - :char => 'ch', - :class => 'cl', - :class_variable => 'cv', - :color => 'cr', - :comment => 'c', - :complex => 'cm', - :constant => 'co', - :content => 'k', - :decorator => 'de', - :definition => 'df', - :delimiter => 'dl', - :directive => 'di', - :doc => 'do', - :doctype => 'dt', - :doc_string => 'ds', - :entity => 'en', - :error => 'er', - :escape => 'e', - :exception => 'ex', - :float => 'fl', - :function => 'fu', - :global_variable => 'gv', - :hex => 'hx', - :imaginary => 'cm', - :important => 'im', - :include => 'ic', - :inline => 'il', - :inline_delimiter => 'idl', - :instance_variable => 'iv', - :integer => 'i', - :interpreted => 'in', - :keyword => 'kw', - :key => 'ke', - :label => 'la', - :local_variable => 'lv', - :modifier => 'mod', - :oct => 'oc', - :operator_fat => 'of', - :pre_constant => 'pc', - :pre_type => 'pt', - :predefined => 'pd', - :preprocessor => 'pp', - :pseudo_class => 'ps', - :regexp => 'rx', - :reserved => 'r', - :shell => 'sh', - :string => 's', - :symbol => 'sy', - :tag => 'ta', - :tag_fat => 'tf', - :tag_special => 'ts', - :type => 'ty', - :variable => 'v', - :value => 'vl', - :xml_text => 'xt', - - :insert => 'ins', - :delete => 'del', - :change => 'chg', - :head => 'head', - - :ident => :NO_HIGHLIGHT, # 'id' - #:operator => 'op', - :operator => :NO_HIGHLIGHT, # 'op' - :space => :NO_HIGHLIGHT, # 'sp' - :plain => :NO_HIGHLIGHT, - } - ClassOfKind[:method] = ClassOfKind[:function] - ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter] - ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter] - ClassOfKind[:escape] = ClassOfKind[:delimiter] - #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!' - end -end \ No newline at end of file diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb new file mode 100755 index 0000000..c8e370c --- /dev/null +++ b/lib/coderay/token_kinds.rb @@ -0,0 +1,82 @@ +module CodeRay + class Tokens + AbbreviationForKind = Hash.new do |h, k| # :nodoc: + raise 'Undefined Token kind: %p' % [k] # :nodoc: + end + AbbreviationForKind.update with = { # :nodoc: + :annotation => 'at', + :attribute_name => 'an', + :attribute_value => 'av', + :bin => 'bi', + :char => 'ch', + :class => 'cl', + :class_variable => 'cv', + :color => 'cr', + :comment => 'c', + :complex => 'cm', + :constant => 'co', + :content => 'k', + :decorator => 'de', + :definition => 'df', + :delimiter => 'dl', + :directive => 'di', + :doc => 'do', + :doctype => 'dt', + :doc_string => 'ds', + :entity => 'en', + :error => 'er', + :escape => 'e', + :exception => 'ex', + :filename => 'filename', + :float => 'fl', + :function => 'fu', + :global_variable => 'gv', + :hex => 'hx', + :imaginary => 'cm', + :important => 'im', + :include => 'ic', + :inline => 'il', + :inline_delimiter => 'idl', + :instance_variable => 'iv', + :integer => 'i', + :interpreted => 'in', + :key => 'ke', + :keyword => 'kw', + :label => 'la', + :local_variable => 'lv', + :modifier => 'mod', + :oct => 'oc', + :predefined => 'pd', + :preprocessor => 'pp', + :pre_constant => 'pc', + :pre_type => 'pt', + :pseudo_class => 'ps', + :regexp => 'rx', + :reserved => 'r', + :shell => 'sh', + :string => 's', + :symbol => 'sy', + :tag => 'ta', + :tag_special => 'ts', + :type => 'ty', + :value => 'vl', + :variable => 'v', + + :insert => 'ins', + :delete => 'del', + :change => 'chg', + :head => 'head', + + :ident => :NO_HIGHLIGHT, # 'id' + #:operator => 'op', + :operator => :NO_HIGHLIGHT, # 'op' + :space => :NO_HIGHLIGHT, # 'sp' + :plain => :NO_HIGHLIGHT, + } + AbbreviationForKind[:method] = AbbreviationForKind[:function] + AbbreviationForKind[:open] = AbbreviationForKind[:close] = AbbreviationForKind[:delimiter] + AbbreviationForKind[:nesting_delimiter] = AbbreviationForKind[:delimiter] + AbbreviationForKind[:escape] = AbbreviationForKind[:delimiter] + #AbbreviationForKind.default = AbbreviationForKind[:error] or raise 'no class found for :error!' + end +end -- cgit v1.2.1