From 84b8431608174e74a4c0d2394eb330a6621bc74b Mon Sep 17 00:00:00 2001 From: no author Date: Mon, 26 Sep 2005 02:58:54 +0000 Subject: New Repository, initial import --- lib/coderay/encoders/count.rb | 20 +++ lib/coderay/encoders/div.rb | 16 ++ lib/coderay/encoders/helpers/html_css.rb | 168 +++++++++++++++++++ lib/coderay/encoders/helpers/html_helper.rb | 68 ++++++++ lib/coderay/encoders/helpers/html_output.rb | 240 ++++++++++++++++++++++++++++ lib/coderay/encoders/html.rb | 167 +++++++++++++++++++ lib/coderay/encoders/null.rb | 20 +++ lib/coderay/encoders/span.rb | 17 ++ lib/coderay/encoders/statistic.rb | 74 +++++++++ lib/coderay/encoders/text.rb | 33 ++++ lib/coderay/encoders/tokens.rb | 44 +++++ lib/coderay/encoders/yaml.rb | 19 +++ 12 files changed, 886 insertions(+) create mode 100644 lib/coderay/encoders/count.rb create mode 100644 lib/coderay/encoders/div.rb create mode 100644 lib/coderay/encoders/helpers/html_css.rb create mode 100644 lib/coderay/encoders/helpers/html_helper.rb create mode 100644 lib/coderay/encoders/helpers/html_output.rb create mode 100644 lib/coderay/encoders/html.rb create mode 100644 lib/coderay/encoders/null.rb create mode 100644 lib/coderay/encoders/span.rb create mode 100644 lib/coderay/encoders/statistic.rb create mode 100644 lib/coderay/encoders/text.rb create mode 100644 lib/coderay/encoders/tokens.rb create mode 100644 lib/coderay/encoders/yaml.rb (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb new file mode 100644 index 0000000..80aec57 --- /dev/null +++ b/lib/coderay/encoders/count.rb @@ -0,0 +1,20 @@ +module CodeRay +module Encoders + + class Count < Encoder + + register_for :count + + protected + + def setup options + @out = 0 + end + + def token text, kind + @out += 1 + end + end + +end +end diff --git a/lib/coderay/encoders/div.rb b/lib/coderay/encoders/div.rb new file mode 100644 index 0000000..640df0e --- /dev/null +++ b/lib/coderay/encoders/div.rb @@ -0,0 +1,16 @@ +module CodeRay module Encoders + + require 'coderay/encoders/html' + class Div < HTML + + FILE_EXTENSION = 'div.html' + + register_for :div + + DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({ + :css => :style, + :wrap => :div, + }) + end + +end end diff --git a/lib/coderay/encoders/helpers/html_css.rb b/lib/coderay/encoders/helpers/html_css.rb new file mode 100644 index 0000000..f9cadf7 --- /dev/null +++ b/lib/coderay/encoders/helpers/html_css.rb @@ -0,0 +1,168 @@ +module CodeRay module Encoders + + class HTML + class CSS + + def initialize stylesheet = TOKENS + @classes = Hash.new + parse stylesheet + end + + def [] *styles + cl = @classes[styles.first] + return '' unless cl + style = false + 1.upto(cl.size + 1) do |offset| + break if style = cl[styles[offset .. -1]] + end + return style + end + + private + + CSS_CLASS = / + ( (?: # $1 = classes + \s* \. [-\w]+ + )+ ) + \s* \{ + ( [^\}]* ) # $2 = style + \} \s* + | + ( . ) # $3 = error + /mx + def parse stylesheet + stylesheet.scan CSS_CLASS do |classes, style, error| + raise "CSS parse error: '#{error}' not recognized" if error + styles = classes.scan(/[-\w]+/) + cl = styles.pop + @classes[cl] ||= Hash.new + @classes[cl][styles] = style.strip + end + end + + MAIN = <<-'MAIN' +.code { + background-color: #FAFAFA; + border: 1px solid #D1D7DC; + font-family: 'Courier New', 'Terminal', monospace; + font-size: 10pt; + color: black; + vertical-align: top; + text-align: left; + padding: 0px; +} +span.code { white-space: pre; } +.code tt { font-weight: bold; } +.code pre { + font-size: 10pt; + margin: 0px 5px; +} +.code .code_table { + margin: 0px; +} +.code .line_numbers { + margin: 0px; + background-color:#DEF; color: #777; + vertical-align: top; + text-align: right; +} +.code .code_cell { + width: 100%; + background-color:#FAFAFA; + color: black; + vertical-align: top; + text-align: left; +} +.code .no { + background-color:#DEF; + color: #777; + padding: 0px 5px; + font-weight: normal; + font-style: normal; +} + +.code tt { display: hidden; } + + MAIN + + TOKENS = <<-'TOKENS' +.af { color:#00C; } +.an { color:#007; } +.av { color:#700; } +.aw { color:#C00; } +.bi { color:#509; font-weight:bold; } +.c { color:#888; } + +.ch { color:#04D; /* background-color:#f0f0ff; */ } +.ch .k { color:#04D; } +.ch .dl { color:#039; } + +.cl { color:#B06; font-weight:bold; } +.co { color:#036; font-weight:bold; } +.cr { color:#0A0; } +.cv { color:#369; } +.df { color:#099; font-weight:bold; } +.di { color:#088; font-weight:bold; } +.dl { color:black; } +.do { color:#970; } +.ds { color:#D42; font-weight:bold; } +.e { color:#666; font-weight:bold; } +.er { color:#F00; background-color:#FAA; } +.ex { color:#F00; font-weight:bold; } +.fl { color:#60E; font-weight:bold; } +.fu { color:#06B; font-weight:bold; } +.gv { color:#d70; font-weight:bold; } +.hx { color:#058; font-weight:bold; } +.i { color:#00D; font-weight:bold; } +.ic { color:#B44; font-weight:bold; } +.in { color:#B2B; font-weight:bold; } +.iv { color:#33B; } +.la { color:#970; font-weight:bold; } +.lv { color:#963; } +.oc { color:#40E; font-weight:bold; } +.on { color:#000; font-weight:bold; } +.pc { color:#038; font-weight:bold; } +.pd { color:#369; font-weight:bold; } +.pp { color:#579; } +.pt { color:#339; font-weight:bold; } +.r { color:#080; font-weight:bold; } + +.rx { background-color:#fff0ff; } +.rx .k { color:#808; } +.rx .dl { color:#404; } +.rx .mod { color:#C2C; } +.rx .fu { color:#404; font-weight: bold; } + +.s { background-color:#fff0f0; } +.s .s { background-color:#ffe0e0; } +.s .s .s { background-color:#ffd0d0; } +.s .k { color:#D20; } +.s .dl { color:#710; } + +.sh { background-color:#f0fff0; } +.sh .k { color:#2B2; } +.sh .dl { color:#161; } + +.sy { color:#A60; } +.sy .k { color:#A60; } +.sy .dl { color:#630; } + +.ta { color:#070; } +.tf { color:#070; font-weight:bold; } +.ts { color:#D70; font-weight:bold; } +.ty { color:#339; font-weight:bold; } +.v { color:#036; } +.xt { color:#444; } + TOKENS + + DEFAULT_STYLESHEET = MAIN + TOKENS + + end + end + +end end + +if $0 == __FILE__ + require 'pp' + pp CodeRay::Encoders::HTML::CSS.new +end diff --git a/lib/coderay/encoders/helpers/html_helper.rb b/lib/coderay/encoders/helpers/html_helper.rb new file mode 100644 index 0000000..03ea0a2 --- /dev/null +++ b/lib/coderay/encoders/helpers/html_helper.rb @@ -0,0 +1,68 @@ +module CodeRay module Encoders + + class HTML + + ClassOfKind = { + :attribute_name => 'an', + :attribute_name_fat => 'af', + :attribute_value => 'av', + :attribute_value_fat => 'aw', + :bin => 'bi', + :char => 'ch', + :class => 'cl', + :class_variable => 'cv', + :color => 'cr', + :comment => 'c', + :constant => 'co', + :content => 'k', + :definition => 'df', + :delimiter => 'dl', + :directive => 'di', + :doc => 'do', + :doc_string => 'ds', + :error => 'er', + :escape => 'e', + :exception => 'ex', + :float => 'fl', + :function => 'fu', + :global_variable => 'gv', + :hex => 'hx', + :include => 'ic', + :instance_variable => 'iv', + :integer => 'i', + :interpreted => 'in', + :label => 'la', + :local_variable => 'lv', + :modifier => 'mod', + :oct => 'oc', + :operator_name => 'on', + :pre_constant => 'pc', + :pre_type => 'pt', + :predefined => 'pd', + :preprocessor => 'pp', + :regexp => 'rx', + :reserved => 'r', + :shell => 'sh', + :string => 's', + :symbol => 'sy', + :tag => 'ta', + :tag_fat => 'tf', + :tag_special => 'ts', + :type => 'ty', + :variable => 'v', + :xml_text => 'xt', + + :ident => :NO_HIGHLIGHT, # 'id' + :operator => :NO_HIGHLIGHT, # 'op' + :space => :NO_HIGHLIGHT, # 'sp' + :plain => :NO_HIGHLIGHT, + } + ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function] + ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter] + ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter] + ClassOfKind[:escape] = ClassOfKind[:delimiter] + ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!' + + end + +end end diff --git a/lib/coderay/encoders/helpers/html_output.rb b/lib/coderay/encoders/helpers/html_output.rb new file mode 100644 index 0000000..e2b26e7 --- /dev/null +++ b/lib/coderay/encoders/helpers/html_output.rb @@ -0,0 +1,240 @@ +module CodeRay + module Encoders + + class HTML + + # This module is included in the output String from thew HTML Encoder. + # + # It provides methods like wrap, div, page etc. + # + # Remember to use #clone instead of #dup to keep the modules the object was + # extended with. + # + # TODO: more doc. + module Output + + class << self + + # This makes Output look like a class. + # + # Example: + # + # a = Output.new 'Code' + # a.wrap! :page + def new string, element = nil + output = string.clone.extend self + output.wrapped_in = element + output + end + + # Raises an exception if an object that doesn't respond to to_str is extended by Output, + # to prevent users from misuse. Use Module#remove_method to disable. + def extended o + warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str + end + + def page_template_for_css css = :default + css = CSS::DEFAULT_STYLESHEET if css == :default + PAGE.apply 'CSS', css + end + + # Define a new wrapper. This is meta programming. + def wrapper *wrappers + wrappers.each do |wrapper| + define_method wrapper do |*args| + wrap wrapper, *args + end + define_method(:"#{wrapper}!") do |*args| + wrap! wrapper, *args + end + end + end + end + + wrapper :div, :span, :page + + def wrapped_in + @wrapped_in || nil + end + attr_writer :wrapped_in + + def wrapped_in? element + wrapped_in == element + end + + def wrap_in template + clone.wrap_in! template + end + + def wrap_in! template + Template.wrap! self, template, 'CONTENT' + self + end + + def wrap! element, *args + return self if not element or element == wrapped_in + case element + when :div + raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil + wrap_in! DIV + when :span + raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil + wrap_in! SPAN + when :page + wrap! :div if wrapped_in? nil + raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div + wrap_in! Output.page_template_for_css + when nil + return self + else + raise "Unknown value %p for :wrap" % element + end + @wrapped_in = element + self + end + + def wrap *args + clone.wrap!(*args) + end + + def numerize! mode = :table, options = {} + return self unless mode + + offset = options.fetch :line_numbers_offset, DEFAULT_OPTIONS[:line_numbers_offset] + unless offset.is_a? Integer + raise ArgumentError, "Invalid value %p for :offset; Integer expected." % offset + end + + unless NUMERIZABLE_WRAPPINGS.include? options[:wrap] + raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]] + end + + bold_every = options.fetch :bold_every, DEFAULT_OPTIONS[:bold_every] + bolding = + if bold_every == :no_bolding or bold_every == 0 + proc { |line| line.to_s } + elsif bold_every.is_a? Integer + proc do |line| + if line % bold_every == 0 + "#{line}" # every bold_every-th number in bold + else + line.to_s + end + end + else + raise ArgumentError, "Invalid value %p for :bolding; :no_bolding or Integer expected." % bolding + end + + line_count = count("\n") + line_count += 1 if self[-1] != ?\n + + case mode + when :inline + max_width = line_count.to_s.size + line = offset - 1 + gsub!(/^/) do + line += 1 + line_number = bolding.call line + "#{ line_number.rjust(max_width) } " + end + wrap! :div + + when :table + # This is really ugly. + # Because even monospace fonts seem to have different heights when bold, + # I make the newline bold, both in the code and the line numbers. + # FIXME Still not working perfect for Mr. Internet Exploder + line_numbers = (offset ... offset + line_count).to_a.map(&bolding).join("\n") + line_numbers << "\n" # also for Mr. MS Internet Exploder :-/ + line_numbers.gsub!(/\n/) { "\n" } + + line_numbers_tpl = DIV_TABLE.apply('LINE_NUMBERS', line_numbers) + gsub!(/\n/) { "\n" } + wrap_in! line_numbers_tpl + @wrapped_in = :div + + else + raise ArgumentError, "Unknown value %p for mode: :inline or :table expected" % mode + end + + self + end + + def numerize *args + clone.numerize!(*args) + end + + class Template < String + + def self.wrap! str, template, target + target = Regexp.new(Regexp.escape("<%#{target}%>")) + if template =~ target + str[0,0] = $` + str << $' + else + raise "Template target <%%%p%%> not found" % target + end + end + + def apply target, replacement + target = Regexp.new(Regexp.escape("<%#{target}%>")) + if self =~ target + Template.new($` + replacement + $') + else + raise "Template target <%%%p%%> not found" % target + end + end + + module Simple + def ` str #` + Template.new str + end + end + end + + extend Template::Simple + +#-- don't include the templates in docu + + SPAN = `<%CONTENT%>` + + DIV, DIV_TABLE, PAGE = + <<-`DIV`, <<-`DIV_TABLE`, <<-`PAGE` + +
+
<%CONTENT%>
+
+ DIV + +
+ + + + + + 
<%LINE_NUMBERS%>
<%CONTENT%>
+
+ DIV_TABLE + + + + + + CodeRay HTML Encoder Example + + + +<%CONTENT%> + + + PAGE + + end + + end + +end +end diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb new file mode 100644 index 0000000..69b6e22 --- /dev/null +++ b/lib/coderay/encoders/html.rb @@ -0,0 +1,167 @@ +module CodeRay +module Encoders + + class HTML < Encoder + + include Streamable + register_for :html + + FILE_EXTENSION = 'html' + + DEFAULT_OPTIONS = { + :tab_width => 8, + + :level => :xhtml, + :css => :class, + + :wrap => :page, + :line_numbers => :table, + :line_numbers_offset => 1, + :bold_every => 10, + } + NUMERIZABLE_WRAPPINGS = [:div, :page] + + require 'coderay/encoders/helpers/html_helper' + require 'coderay/encoders/helpers/html_output' + require 'coderay/encoders/helpers/html_css' + + def initialize(*) + super + @last_options = nil + end + + protected + + HTML_ESCAPE = { #:nodoc: + '&' => '&', + '"' => '"', + '>' => '>', + '<' => '<', + } + + # This is to prevent illegal HTML. + # Strange chars should still be avoided in codes. + evil_chars = Array(0x00...0x20) - [?n, ?t] + evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' } + ansi_chars = Array(0x7f..0xff) + ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i } + # \x9 (\t) and \xA (\n) not included + HTML_ESCAPE_PATTERN = /[&"><\0-\x8\xB-\x1f\x7f-\xff]/ + + def setup options + if options[:line_numbers] and not NUMERIZABLE_WRAPPINGS.include? options[:wrap] + warn ':line_numbers wanted, but :wrap is %p' % options[:wrap] + end + super + return if options == @last_options + @last_options = options + + @HTML_ESCAPE = HTML_ESCAPE.dup + @HTML_ESCAPE["\t"] = ' ' * options[:tab_width] + + @opened = [nil] + @css = CSS.new + + case options[:css] + + when :class + @css_style = Hash.new do |h, k| + if k.is_a? Array + type = k.first + else + type = k + end + c = ClassOfKind[type] + if c == :NO_HIGHLIGHT + h[k] = false + else + if options[:debug] + debug_info = ' title="%p"' % [ k ] + else + debug_info = '' + end + h[k] = '' % [debug_info, c] + end + end + + when :style + @css_style = Hash.new do |h, k| + if k.is_a? Array + styles = k.dup + else + styles = [k] + end + styles.map! { |c| ClassOfKind[c] } + if styles.first == :NO_HIGHLIGHT + h[k] = false + else + if options[:debug] + debug_info = ' title="%s"' % [ styles.inspect.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } ] + else + debug_info = '' + end + style = @css[*styles] + h[k] = + if style + '' % [debug_info, style] + else + false + end + end + end + + else + raise "Unknown value %p for :css." % options[:css] + + end + end + + def finish options + not_needed = @opened.shift + @out << '' * @opened.size + + @out.extend Output + @out.numerize! options[:line_numbers], options # if options[:line_numbers] + @out.wrap! options[:wrap] # if options[:wrap] + + #require 'pp' + #pp @css_style, @css_style.size + + super + end + + def token text, type + if text.is_a? String + # be careful when streaming: text is changed! + text.gsub!(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } + @opened[0] = type + style = @css_style[@opened] + if style + @out << style << text << '' + else + @out << text + end + else + case text + when :open + @opened[0] = type + @out << @css_style[@opened] + @opened << type + when :close + unless @opened.empty? + raise 'Not Token to be closed.' unless @opened.size > 1 + @out << '' + @opened.pop + end + when nil + raise 'Token with nil as text was given: %p' % [[text, type]] + else + raise 'unknown token kind: %p' % text + end + end + end + + end + +end +end diff --git a/lib/coderay/encoders/null.rb b/lib/coderay/encoders/null.rb new file mode 100644 index 0000000..67c4987 --- /dev/null +++ b/lib/coderay/encoders/null.rb @@ -0,0 +1,20 @@ +module CodeRay + module Encoders + + class Null < Encoder + + include Streamable + register_for :null + + protected + + def token(*) + # do nothing + end + + end + + end +end + + diff --git a/lib/coderay/encoders/span.rb b/lib/coderay/encoders/span.rb new file mode 100644 index 0000000..a7715f4 --- /dev/null +++ b/lib/coderay/encoders/span.rb @@ -0,0 +1,17 @@ +module CodeRay module Encoders + + require 'coderay/encoders/html' + class Span < HTML + + FILE_EXTENSION = 'span.html' + + register_for :span + + DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({ + :css => :style, + :wrap => :span, + :line_numbers => nil, + }) + end + +end end diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb new file mode 100644 index 0000000..0685c03 --- /dev/null +++ b/lib/coderay/encoders/statistic.rb @@ -0,0 +1,74 @@ +module CodeRay module Encoders + + # Makes a statistic for the given tokens. + class Statistic < Encoder + + include Streamable + register_for :stats, :statistic + + attr_reader :type_stats, :real_token_count + + protected + + TypeStats = Struct.new :count, :size + + def setup options + @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 } + @real_token_count = 0 + end + + def generate tokens, options + @tokens = tokens + super + end + + def token text, type + @type_stats['TOTAL'].count += 1 + if text.is_a? String + @real_token_count += 1 unless type == :space + @type_stats[type].count += 1 + @type_stats[type].size += text.size + @type_stats['TOTAL'].size += text.size + else + @content_type = type + @type_stats['open/close'].count += 1 + end + end + + STATS = <<-STATS + +Code Statistics + +Tokens %8d + Non-Whitespace %8d +Bytes Total %8d + +Token Types (%d): + type count ratio size (average) +------------------------------------------------------------- +%s + STATS +# space 12007 33.81 % 1.7 + TOKEN_TYPES_ROW = <<-TKR + %-20s %8d %6.2f %% %5.1f + TKR + + def finish options + all = @type_stats['TOTAL'] + all_count, all_size = all.count, all.size + @type_stats.each do |type, stat| + stat.size /= stat.count.to_f + end + types_stats = @type_stats.sort_by { |k, v| -v.count }.map do |k, v| + TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size] + end.join + STATS % [ + all_count, @real_token_count, all_size, + @type_stats.delete_if { |k, v| k.is_a? String }.size, + types_stats + ] + end + + end + +end end diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb new file mode 100644 index 0000000..4f0a754 --- /dev/null +++ b/lib/coderay/encoders/text.rb @@ -0,0 +1,33 @@ +module CodeRay + module Encoders + + class Text < Encoder + + include Streamable + register_for :text + + FILE_EXTENSION = 'txt' + + DEFAULT_OPTIONS = { + :separator => '' + } + + protected + def setup options + super + @sep = options[:separator] + end + + def token text, kind + return unless text.respond_to :to_str + @out << text + @sep + end + + def finish options + @out.chomp @sep + end + + end + + end +end diff --git a/lib/coderay/encoders/tokens.rb b/lib/coderay/encoders/tokens.rb new file mode 100644 index 0000000..4573307 --- /dev/null +++ b/lib/coderay/encoders/tokens.rb @@ -0,0 +1,44 @@ +module CodeRay + module Encoders + + # The Tokens encoder converts the tokens to a simple + # readable format. It doesn't use colors and is mainly + # intended for console output. + # + # The tokens are converted with Tokens.write_token. + # + # The format is: + # + # \t \n + # + # Example: + # + # require 'coderay' + # puts CodeRay.scan("puts 3 + 4", :ruby).tokens + # + # prints: + # + # ident puts + # space + # integer 3 + # space + # operator + + # space + # integer 4 + # + class Tokens < Encoder + + include Streamable + register_for :tokens + + FILE_EXTENSION = 'tok' + + protected + def token *args + @out << CodeRay::Tokens.write_token(*args) + end + + end + + end +end diff --git a/lib/coderay/encoders/yaml.rb b/lib/coderay/encoders/yaml.rb new file mode 100644 index 0000000..4e2b7a1 --- /dev/null +++ b/lib/coderay/encoders/yaml.rb @@ -0,0 +1,19 @@ +module CodeRay + module Encoders + + class YAML < Encoder + + register_for :yaml + + FILE_EXTENSION = 'yaml' + + protected + def compile tokens, options + require 'yaml' + @out = tokens.to_a.to_yaml + end + + end + + end +end -- cgit v1.2.1