diff options
| author | no author <noone@nowhere> | 2005-09-26 02:58:54 +0000 |
|---|---|---|
| committer | no author <noone@nowhere> | 2005-09-26 02:58:54 +0000 |
| commit | 84b8431608174e74a4c0d2394eb330a6621bc74b (patch) | |
| tree | ffc2bd7ce21708a9147247c80b0e7fc7728ea063 /lib/coderay/encoders | |
| download | coderay-84b8431608174e74a4c0d2394eb330a6621bc74b.tar.gz | |
New Repository, initial import
Diffstat (limited to 'lib/coderay/encoders')
| -rw-r--r-- | lib/coderay/encoders/count.rb | 20 | ||||
| -rw-r--r-- | lib/coderay/encoders/div.rb | 16 | ||||
| -rw-r--r-- | lib/coderay/encoders/helpers/html_css.rb | 168 | ||||
| -rw-r--r-- | lib/coderay/encoders/helpers/html_helper.rb | 68 | ||||
| -rw-r--r-- | lib/coderay/encoders/helpers/html_output.rb | 240 | ||||
| -rw-r--r-- | lib/coderay/encoders/html.rb | 167 | ||||
| -rw-r--r-- | lib/coderay/encoders/null.rb | 20 | ||||
| -rw-r--r-- | lib/coderay/encoders/span.rb | 17 | ||||
| -rw-r--r-- | lib/coderay/encoders/statistic.rb | 74 | ||||
| -rw-r--r-- | lib/coderay/encoders/text.rb | 33 | ||||
| -rw-r--r-- | lib/coderay/encoders/tokens.rb | 44 | ||||
| -rw-r--r-- | lib/coderay/encoders/yaml.rb | 19 |
12 files changed, 886 insertions, 0 deletions
diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb new file mode 100644 index 0000000..80aec57 --- /dev/null +++ b/lib/coderay/encoders/count.rb @@ -0,0 +1,20 @@ +module CodeRay
+module Encoders
+
+ class Count < Encoder
+
+ register_for :count
+
+ protected
+
+ def setup options
+ @out = 0
+ end
+
+ def token text, kind
+ @out += 1
+ end
+ end
+
+end
+end
diff --git a/lib/coderay/encoders/div.rb b/lib/coderay/encoders/div.rb new file mode 100644 index 0000000..640df0e --- /dev/null +++ b/lib/coderay/encoders/div.rb @@ -0,0 +1,16 @@ +module CodeRay module Encoders
+
+ require 'coderay/encoders/html'
+ class Div < HTML
+
+ FILE_EXTENSION = 'div.html'
+
+ register_for :div
+
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+ :css => :style,
+ :wrap => :div,
+ })
+ end
+
+end end
diff --git a/lib/coderay/encoders/helpers/html_css.rb b/lib/coderay/encoders/helpers/html_css.rb new file mode 100644 index 0000000..f9cadf7 --- /dev/null +++ b/lib/coderay/encoders/helpers/html_css.rb @@ -0,0 +1,168 @@ +module CodeRay module Encoders
+
+ class HTML
+ class CSS
+
+ def initialize stylesheet = TOKENS
+ @classes = Hash.new
+ parse stylesheet
+ end
+
+ def [] *styles
+ cl = @classes[styles.first]
+ return '' unless cl
+ style = false
+ 1.upto(cl.size + 1) do |offset|
+ break if style = cl[styles[offset .. -1]]
+ end
+ return style
+ end
+
+ private
+
+ CSS_CLASS = /
+ ( (?: # $1 = classes
+ \s* \. [-\w]+
+ )+ )
+ \s* \{
+ ( [^\}]* ) # $2 = style
+ \} \s*
+ |
+ ( . ) # $3 = error
+ /mx
+ def parse stylesheet
+ stylesheet.scan CSS_CLASS do |classes, style, error|
+ raise "CSS parse error: '#{error}' not recognized" if error
+ styles = classes.scan(/[-\w]+/)
+ cl = styles.pop
+ @classes[cl] ||= Hash.new
+ @classes[cl][styles] = style.strip
+ end
+ end
+
+ MAIN = <<-'MAIN'
+.code {
+ background-color: #FAFAFA;
+ border: 1px solid #D1D7DC;
+ font-family: 'Courier New', 'Terminal', monospace;
+ font-size: 10pt;
+ color: black;
+ vertical-align: top;
+ text-align: left;
+ padding: 0px;
+}
+span.code { white-space: pre; }
+.code tt { font-weight: bold; }
+.code pre {
+ font-size: 10pt;
+ margin: 0px 5px;
+}
+.code .code_table {
+ margin: 0px;
+}
+.code .line_numbers {
+ margin: 0px;
+ background-color:#DEF; color: #777;
+ vertical-align: top;
+ text-align: right;
+}
+.code .code_cell {
+ width: 100%;
+ background-color:#FAFAFA;
+ color: black;
+ vertical-align: top;
+ text-align: left;
+}
+.code .no {
+ background-color:#DEF;
+ color: #777;
+ padding: 0px 5px;
+ font-weight: normal;
+ font-style: normal;
+}
+
+.code tt { display: hidden; }
+
+ MAIN
+
+ TOKENS = <<-'TOKENS'
+.af { color:#00C; }
+.an { color:#007; }
+.av { color:#700; }
+.aw { color:#C00; }
+.bi { color:#509; font-weight:bold; }
+.c { color:#888; }
+
+.ch { color:#04D; /* background-color:#f0f0ff; */ }
+.ch .k { color:#04D; }
+.ch .dl { color:#039; }
+
+.cl { color:#B06; font-weight:bold; }
+.co { color:#036; font-weight:bold; }
+.cr { color:#0A0; }
+.cv { color:#369; }
+.df { color:#099; font-weight:bold; }
+.di { color:#088; font-weight:bold; }
+.dl { color:black; }
+.do { color:#970; }
+.ds { color:#D42; font-weight:bold; }
+.e { color:#666; font-weight:bold; }
+.er { color:#F00; background-color:#FAA; }
+.ex { color:#F00; font-weight:bold; }
+.fl { color:#60E; font-weight:bold; }
+.fu { color:#06B; font-weight:bold; }
+.gv { color:#d70; font-weight:bold; }
+.hx { color:#058; font-weight:bold; }
+.i { color:#00D; font-weight:bold; }
+.ic { color:#B44; font-weight:bold; }
+.in { color:#B2B; font-weight:bold; }
+.iv { color:#33B; }
+.la { color:#970; font-weight:bold; }
+.lv { color:#963; }
+.oc { color:#40E; font-weight:bold; }
+.on { color:#000; font-weight:bold; }
+.pc { color:#038; font-weight:bold; }
+.pd { color:#369; font-weight:bold; }
+.pp { color:#579; }
+.pt { color:#339; font-weight:bold; }
+.r { color:#080; font-weight:bold; }
+
+.rx { background-color:#fff0ff; }
+.rx .k { color:#808; }
+.rx .dl { color:#404; }
+.rx .mod { color:#C2C; }
+.rx .fu { color:#404; font-weight: bold; }
+
+.s { background-color:#fff0f0; }
+.s .s { background-color:#ffe0e0; }
+.s .s .s { background-color:#ffd0d0; }
+.s .k { color:#D20; }
+.s .dl { color:#710; }
+
+.sh { background-color:#f0fff0; }
+.sh .k { color:#2B2; }
+.sh .dl { color:#161; }
+
+.sy { color:#A60; }
+.sy .k { color:#A60; }
+.sy .dl { color:#630; }
+
+.ta { color:#070; }
+.tf { color:#070; font-weight:bold; }
+.ts { color:#D70; font-weight:bold; }
+.ty { color:#339; font-weight:bold; }
+.v { color:#036; }
+.xt { color:#444; }
+ TOKENS
+
+ DEFAULT_STYLESHEET = MAIN + TOKENS
+
+ end
+ end
+
+end end
+
+if $0 == __FILE__
+ require 'pp'
+ pp CodeRay::Encoders::HTML::CSS.new
+end
diff --git a/lib/coderay/encoders/helpers/html_helper.rb b/lib/coderay/encoders/helpers/html_helper.rb new file mode 100644 index 0000000..03ea0a2 --- /dev/null +++ b/lib/coderay/encoders/helpers/html_helper.rb @@ -0,0 +1,68 @@ +module CodeRay module Encoders
+
+ class HTML
+
+ ClassOfKind = {
+ :attribute_name => 'an',
+ :attribute_name_fat => 'af',
+ :attribute_value => 'av',
+ :attribute_value_fat => 'aw',
+ :bin => 'bi',
+ :char => 'ch',
+ :class => 'cl',
+ :class_variable => 'cv',
+ :color => 'cr',
+ :comment => 'c',
+ :constant => 'co',
+ :content => 'k',
+ :definition => 'df',
+ :delimiter => 'dl',
+ :directive => 'di',
+ :doc => 'do',
+ :doc_string => 'ds',
+ :error => 'er',
+ :escape => 'e',
+ :exception => 'ex',
+ :float => 'fl',
+ :function => 'fu',
+ :global_variable => 'gv',
+ :hex => 'hx',
+ :include => 'ic',
+ :instance_variable => 'iv',
+ :integer => 'i',
+ :interpreted => 'in',
+ :label => 'la',
+ :local_variable => 'lv',
+ :modifier => 'mod',
+ :oct => 'oc',
+ :operator_name => 'on',
+ :pre_constant => 'pc',
+ :pre_type => 'pt',
+ :predefined => 'pd',
+ :preprocessor => 'pp',
+ :regexp => 'rx',
+ :reserved => 'r',
+ :shell => 'sh',
+ :string => 's',
+ :symbol => 'sy',
+ :tag => 'ta',
+ :tag_fat => 'tf',
+ :tag_special => 'ts',
+ :type => 'ty',
+ :variable => 'v',
+ :xml_text => 'xt',
+
+ :ident => :NO_HIGHLIGHT, # 'id'
+ :operator => :NO_HIGHLIGHT, # 'op'
+ :space => :NO_HIGHLIGHT, # 'sp'
+ :plain => :NO_HIGHLIGHT,
+ }
+ ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+ ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
+ ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
+ ClassOfKind[:escape] = ClassOfKind[:delimiter]
+ ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+
+ end
+
+end end
diff --git a/lib/coderay/encoders/helpers/html_output.rb b/lib/coderay/encoders/helpers/html_output.rb new file mode 100644 index 0000000..e2b26e7 --- /dev/null +++ b/lib/coderay/encoders/helpers/html_output.rb @@ -0,0 +1,240 @@ +module CodeRay
+ module Encoders
+
+ class HTML
+
+ # This module is included in the output String from thew HTML Encoder.
+ #
+ # It provides methods like wrap, div, page etc.
+ #
+ # Remember to use #clone instead of #dup to keep the modules the object was
+ # extended with.
+ #
+ # TODO: more doc.
+ module Output
+
+ class << self
+
+ # This makes Output look like a class.
+ #
+ # Example:
+ #
+ # a = Output.new '<span class="co">Code</span>'
+ # a.wrap! :page
+ def new string, element = nil
+ output = string.clone.extend self
+ output.wrapped_in = element
+ output
+ end
+
+ # Raises an exception if an object that doesn't respond to to_str is extended by Output,
+ # to prevent users from misuse. Use Module#remove_method to disable.
+ def extended o
+ warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str
+ end
+
+ def page_template_for_css css = :default
+ css = CSS::DEFAULT_STYLESHEET if css == :default
+ PAGE.apply 'CSS', css
+ end
+
+ # Define a new wrapper. This is meta programming.
+ def wrapper *wrappers
+ wrappers.each do |wrapper|
+ define_method wrapper do |*args|
+ wrap wrapper, *args
+ end
+ define_method(:"#{wrapper}!") do |*args|
+ wrap! wrapper, *args
+ end
+ end
+ end
+ end
+
+ wrapper :div, :span, :page
+
+ def wrapped_in
+ @wrapped_in || nil
+ end
+ attr_writer :wrapped_in
+
+ def wrapped_in? element
+ wrapped_in == element
+ end
+
+ def wrap_in template
+ clone.wrap_in! template
+ end
+
+ def wrap_in! template
+ Template.wrap! self, template, 'CONTENT'
+ self
+ end
+
+ def wrap! element, *args
+ return self if not element or element == wrapped_in
+ case element
+ when :div
+ raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
+ wrap_in! DIV
+ when :span
+ raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
+ wrap_in! SPAN
+ when :page
+ wrap! :div if wrapped_in? nil
+ raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div
+ wrap_in! Output.page_template_for_css
+ when nil
+ return self
+ else
+ raise "Unknown value %p for :wrap" % element
+ end
+ @wrapped_in = element
+ self
+ end
+
+ def wrap *args
+ clone.wrap!(*args)
+ end
+
+ def numerize! mode = :table, options = {}
+ return self unless mode
+
+ offset = options.fetch :line_numbers_offset, DEFAULT_OPTIONS[:line_numbers_offset]
+ unless offset.is_a? Integer
+ raise ArgumentError, "Invalid value %p for :offset; Integer expected." % offset
+ end
+
+ unless NUMERIZABLE_WRAPPINGS.include? options[:wrap]
+ raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]
+ end
+
+ bold_every = options.fetch :bold_every, DEFAULT_OPTIONS[:bold_every]
+ bolding =
+ if bold_every == :no_bolding or bold_every == 0
+ proc { |line| line.to_s }
+ elsif bold_every.is_a? Integer
+ proc do |line|
+ if line % bold_every == 0
+ "<strong>#{line}</strong>" # every bold_every-th number in bold
+ else
+ line.to_s
+ end
+ end
+ else
+ raise ArgumentError, "Invalid value %p for :bolding; :no_bolding or Integer expected." % bolding
+ end
+
+ line_count = count("\n")
+ line_count += 1 if self[-1] != ?\n
+
+ case mode
+ when :inline
+ max_width = line_count.to_s.size
+ line = offset - 1
+ gsub!(/^/) do
+ line += 1
+ line_number = bolding.call line
+ "<span class=\"no\">#{ line_number.rjust(max_width) }</span> "
+ end
+ wrap! :div
+
+ when :table
+ # This is really ugly.
+ # Because even monospace fonts seem to have different heights when bold,
+ # I make the newline bold, both in the code and the line numbers.
+ # FIXME Still not working perfect for Mr. Internet Exploder
+ line_numbers = (offset ... offset + line_count).to_a.map(&bolding).join("\n")
+ line_numbers << "\n" # also for Mr. MS Internet Exploder :-/
+ line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
+
+ line_numbers_tpl = DIV_TABLE.apply('LINE_NUMBERS', line_numbers)
+ gsub!(/\n/) { "<tt>\n</tt>" }
+ wrap_in! line_numbers_tpl
+ @wrapped_in = :div
+
+ else
+ raise ArgumentError, "Unknown value %p for mode: :inline or :table expected" % mode
+ end
+
+ self
+ end
+
+ def numerize *args
+ clone.numerize!(*args)
+ end
+
+ class Template < String
+
+ def self.wrap! str, template, target
+ target = Regexp.new(Regexp.escape("<%#{target}%>"))
+ if template =~ target
+ str[0,0] = $`
+ str << $'
+ else
+ raise "Template target <%%%p%%> not found" % target
+ end
+ end
+
+ def apply target, replacement
+ target = Regexp.new(Regexp.escape("<%#{target}%>"))
+ if self =~ target
+ Template.new($` + replacement + $')
+ else
+ raise "Template target <%%%p%%> not found" % target
+ end
+ end
+
+ module Simple
+ def ` str #`
+ Template.new str
+ end
+ end
+ end
+
+ extend Template::Simple
+
+#-- don't include the templates in docu
+
+ SPAN = `<span class="code"><%CONTENT%></span>`
+
+ DIV, DIV_TABLE, PAGE =
+ <<-`DIV`, <<-`DIV_TABLE`, <<-`PAGE`
+
+<div class="code">
+<pre><%CONTENT%></pre>
+</div>
+ DIV
+
+<div class="code">
+ <table class="code_table">
+ <tr>
+ <td class="line_numbers"><pre><%LINE_NUMBERS%></pre></td>
+ <td class="code_cell"><div class="nowrap"><pre><%CONTENT%></pre></div></td>
+ </tr>
+ </table>
+</div>
+ DIV_TABLE
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
+<head>
+ <meta http-equiv="content-type" content="text/html; charset=iso-8859-1" />
+ <title>CodeRay HTML Encoder Example</title>
+ <style type="text/css">
+<%CSS%>
+ </style>
+</head>
+<body style="background-color: white;">
+<%CONTENT%>
+</body>
+</html>
+ PAGE
+
+ end
+
+ end
+
+end
+end
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb new file mode 100644 index 0000000..69b6e22 --- /dev/null +++ b/lib/coderay/encoders/html.rb @@ -0,0 +1,167 @@ +module CodeRay
+module Encoders
+
+ class HTML < Encoder
+
+ include Streamable
+ register_for :html
+
+ FILE_EXTENSION = 'html'
+
+ DEFAULT_OPTIONS = {
+ :tab_width => 8,
+
+ :level => :xhtml,
+ :css => :class,
+
+ :wrap => :page,
+ :line_numbers => :table,
+ :line_numbers_offset => 1,
+ :bold_every => 10,
+ }
+ NUMERIZABLE_WRAPPINGS = [:div, :page]
+
+ require 'coderay/encoders/helpers/html_helper'
+ require 'coderay/encoders/helpers/html_output'
+ require 'coderay/encoders/helpers/html_css'
+
+ def initialize(*)
+ super
+ @last_options = nil
+ end
+
+ protected
+
+ HTML_ESCAPE = { #:nodoc:
+ '&' => '&',
+ '"' => '"',
+ '>' => '>',
+ '<' => '<',
+ }
+
+ # This is to prevent illegal HTML.
+ # Strange chars should still be avoided in codes.
+ evil_chars = Array(0x00...0x20) - [?n, ?t]
+ evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
+ ansi_chars = Array(0x7f..0xff)
+ ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
+ # \x9 (\t) and \xA (\n) not included
+ HTML_ESCAPE_PATTERN = /[&"><\0-\x8\xB-\x1f\x7f-\xff]/
+
+ def setup options
+ if options[:line_numbers] and not NUMERIZABLE_WRAPPINGS.include? options[:wrap]
+ warn ':line_numbers wanted, but :wrap is %p' % options[:wrap]
+ end
+ super
+ return if options == @last_options
+ @last_options = options
+
+ @HTML_ESCAPE = HTML_ESCAPE.dup
+ @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
+
+ @opened = [nil]
+ @css = CSS.new
+
+ case options[:css]
+
+ when :class
+ @css_style = Hash.new do |h, k|
+ if k.is_a? Array
+ type = k.first
+ else
+ type = k
+ end
+ c = ClassOfKind[type]
+ if c == :NO_HIGHLIGHT
+ h[k] = false
+ else
+ if options[:debug]
+ debug_info = ' title="%p"' % [ k ]
+ else
+ debug_info = ''
+ end
+ h[k] = '<span%s class="%s">' % [debug_info, c]
+ end
+ end
+
+ when :style
+ @css_style = Hash.new do |h, k|
+ if k.is_a? Array
+ styles = k.dup
+ else
+ styles = [k]
+ end
+ styles.map! { |c| ClassOfKind[c] }
+ if styles.first == :NO_HIGHLIGHT
+ h[k] = false
+ else
+ if options[:debug]
+ debug_info = ' title="%s"' % [ styles.inspect.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } ]
+ else
+ debug_info = ''
+ end
+ style = @css[*styles]
+ h[k] =
+ if style
+ '<span%s style="%s">' % [debug_info, style]
+ else
+ false
+ end
+ end
+ end
+
+ else
+ raise "Unknown value %p for :css." % options[:css]
+
+ end
+ end
+
+ def finish options
+ not_needed = @opened.shift
+ @out << '</span>' * @opened.size
+
+ @out.extend Output
+ @out.numerize! options[:line_numbers], options # if options[:line_numbers]
+ @out.wrap! options[:wrap] # if options[:wrap]
+
+ #require 'pp'
+ #pp @css_style, @css_style.size
+
+ super
+ end
+
+ def token text, type
+ if text.is_a? String
+ # be careful when streaming: text is changed!
+ text.gsub!(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
+ @opened[0] = type
+ style = @css_style[@opened]
+ if style
+ @out << style << text << '</span>'
+ else
+ @out << text
+ end
+ else
+ case text
+ when :open
+ @opened[0] = type
+ @out << @css_style[@opened]
+ @opened << type
+ when :close
+ unless @opened.empty?
+ raise 'Not Token to be closed.' unless @opened.size > 1
+ @out << '</span>'
+ @opened.pop
+ end
+ when nil
+ raise 'Token with nil as text was given: %p' % [[text, type]]
+ else
+ raise 'unknown token kind: %p' % text
+ end
+ end
+ end
+
+ end
+
+end
+end
diff --git a/lib/coderay/encoders/null.rb b/lib/coderay/encoders/null.rb new file mode 100644 index 0000000..67c4987 --- /dev/null +++ b/lib/coderay/encoders/null.rb @@ -0,0 +1,20 @@ +module CodeRay
+ module Encoders
+
+ class Null < Encoder
+
+ include Streamable
+ register_for :null
+
+ protected
+
+ def token(*)
+ # do nothing
+ end
+
+ end
+
+ end
+end
+
+
diff --git a/lib/coderay/encoders/span.rb b/lib/coderay/encoders/span.rb new file mode 100644 index 0000000..a7715f4 --- /dev/null +++ b/lib/coderay/encoders/span.rb @@ -0,0 +1,17 @@ +module CodeRay module Encoders
+
+ require 'coderay/encoders/html'
+ class Span < HTML
+
+ FILE_EXTENSION = 'span.html'
+
+ register_for :span
+
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+ :css => :style,
+ :wrap => :span,
+ :line_numbers => nil,
+ })
+ end
+
+end end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb new file mode 100644 index 0000000..0685c03 --- /dev/null +++ b/lib/coderay/encoders/statistic.rb @@ -0,0 +1,74 @@ +module CodeRay module Encoders
+
+ # Makes a statistic for the given tokens.
+ class Statistic < Encoder
+
+ include Streamable
+ register_for :stats, :statistic
+
+ attr_reader :type_stats, :real_token_count
+
+ protected
+
+ TypeStats = Struct.new :count, :size
+
+ def setup options
+ @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
+ @real_token_count = 0
+ end
+
+ def generate tokens, options
+ @tokens = tokens
+ super
+ end
+
+ def token text, type
+ @type_stats['TOTAL'].count += 1
+ if text.is_a? String
+ @real_token_count += 1 unless type == :space
+ @type_stats[type].count += 1
+ @type_stats[type].size += text.size
+ @type_stats['TOTAL'].size += text.size
+ else
+ @content_type = type
+ @type_stats['open/close'].count += 1
+ end
+ end
+
+ STATS = <<-STATS
+
+Code Statistics
+
+Tokens %8d
+ Non-Whitespace %8d
+Bytes Total %8d
+
+Token Types (%d):
+ type count ratio size (average)
+-------------------------------------------------------------
+%s
+ STATS
+# space 12007 33.81 % 1.7
+ TOKEN_TYPES_ROW = <<-TKR
+ %-20s %8d %6.2f %% %5.1f
+ TKR
+
+ def finish options
+ all = @type_stats['TOTAL']
+ all_count, all_size = all.count, all.size
+ @type_stats.each do |type, stat|
+ stat.size /= stat.count.to_f
+ end
+ types_stats = @type_stats.sort_by { |k, v| -v.count }.map do |k, v|
+ TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
+ end.join
+ STATS % [
+ all_count, @real_token_count, all_size,
+ @type_stats.delete_if { |k, v| k.is_a? String }.size,
+ types_stats
+ ]
+ end
+
+ end
+
+end end
diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb new file mode 100644 index 0000000..4f0a754 --- /dev/null +++ b/lib/coderay/encoders/text.rb @@ -0,0 +1,33 @@ +module CodeRay
+ module Encoders
+
+ class Text < Encoder
+
+ include Streamable
+ register_for :text
+
+ FILE_EXTENSION = 'txt'
+
+ DEFAULT_OPTIONS = {
+ :separator => ''
+ }
+
+ protected
+ def setup options
+ super
+ @sep = options[:separator]
+ end
+
+ def token text, kind
+ return unless text.respond_to :to_str
+ @out << text + @sep
+ end
+
+ def finish options
+ @out.chomp @sep
+ end
+
+ end
+
+ end
+end
diff --git a/lib/coderay/encoders/tokens.rb b/lib/coderay/encoders/tokens.rb new file mode 100644 index 0000000..4573307 --- /dev/null +++ b/lib/coderay/encoders/tokens.rb @@ -0,0 +1,44 @@ +module CodeRay
+ module Encoders
+
+ # The Tokens encoder converts the tokens to a simple
+ # readable format. It doesn't use colors and is mainly
+ # intended for console output.
+ #
+ # The tokens are converted with Tokens.write_token.
+ #
+ # The format is:
+ #
+ # <token-kind> \t <escaped token-text> \n
+ #
+ # Example:
+ #
+ # require 'coderay'
+ # puts CodeRay.scan("puts 3 + 4", :ruby).tokens
+ #
+ # prints:
+ #
+ # ident puts
+ # space
+ # integer 3
+ # space
+ # operator +
+ # space
+ # integer 4
+ #
+ class Tokens < Encoder
+
+ include Streamable
+ register_for :tokens
+
+ FILE_EXTENSION = 'tok'
+
+ protected
+ def token *args
+ @out << CodeRay::Tokens.write_token(*args)
+ end
+
+ end
+
+ end
+end
diff --git a/lib/coderay/encoders/yaml.rb b/lib/coderay/encoders/yaml.rb new file mode 100644 index 0000000..4e2b7a1 --- /dev/null +++ b/lib/coderay/encoders/yaml.rb @@ -0,0 +1,19 @@ +module CodeRay
+ module Encoders
+
+ class YAML < Encoder
+
+ register_for :yaml
+
+ FILE_EXTENSION = 'yaml'
+
+ protected
+ def compile tokens, options
+ require 'yaml'
+ @out = tokens.to_a.to_yaml
+ end
+
+ end
+
+ end
+end
|