Index: lib/coderay/token_classes.rb
===================================================================
--- lib/coderay/token_classes.rb (revision 0)
+++ lib/coderay/token_classes.rb (revision 250)
@@ -0,0 +1,71 @@
+module CodeRay
+ class Tokens
+ ClassOfKind = Hash.new do |h, k|
+ h[k] = k.to_s
+ end
+ ClassOfKind.update with = {
+ :attribute_name => 'an',
+ :attribute_name_fat => 'af',
+ :attribute_value => 'av',
+ :attribute_value_fat => 'aw',
+ :bin => 'bi',
+ :char => 'ch',
+ :class => 'cl',
+ :class_variable => 'cv',
+ :color => 'cr',
+ :comment => 'c',
+ :constant => 'co',
+ :content => 'k',
+ :definition => 'df',
+ :delimiter => 'dl',
+ :directive => 'di',
+ :doc => 'do',
+ :doc_string => 'ds',
+ :entity => 'en',
+ :error => 'er',
+ :escape => 'e',
+ :exception => 'ex',
+ :float => 'fl',
+ :function => 'fu',
+ :global_variable => 'gv',
+ :hex => 'hx',
+ :include => 'ic',
+ :inline => 'il',
+ :inline_delimiter => 'idl',
+ :instance_variable => 'iv',
+ :integer => 'i',
+ :interpreted => 'in',
+ :label => 'la',
+ :local_variable => 'lv',
+ :modifier => 'mod',
+ :oct => 'oc',
+ :operator_fat => 'of',
+ :pre_constant => 'pc',
+ :pre_type => 'pt',
+ :predefined => 'pd',
+ :preprocessor => 'pp',
+ :regexp => 'rx',
+ :reserved => 'r',
+ :shell => 'sh',
+ :string => 's',
+ :symbol => 'sy',
+ :tag => 'ta',
+ :tag_fat => 'tf',
+ :tag_special => 'ts',
+ :type => 'ty',
+ :variable => 'v',
+ :xml_text => 'xt',
+
+ :ident => :NO_HIGHLIGHT, # 'id'
+ #:operator => 'op',
+ :operator => :NO_HIGHLIGHT, # 'op'
+ :space => :NO_HIGHLIGHT, # 'sp'
+ :plain => :NO_HIGHLIGHT,
+ }
+ ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+ ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
+ ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
+ ClassOfKind[:escape] = ClassOfKind[:delimiter]
+ #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+ end
+end
\ No newline at end of file
Property changes on: lib/coderay/token_classes.rb
___________________________________________________________________
Added: svn:executable
+ *
Index: lib/coderay/encoder.rb
===================================================================
--- lib/coderay/encoder.rb (revision 200)
+++ lib/coderay/encoder.rb (revision 250)
@@ -1,3 +1,5 @@
+require "stringio"
+
module CodeRay
# This module holds the Encoder class and its subclasses.
@@ -40,7 +42,7 @@
# downcase class name instead.
def const_missing sym
if sym == :FILE_EXTENSION
- sym.to_s.downcase
+ plugin_id
else
super
end
@@ -130,13 +132,15 @@
# By default, it calls text_token or block_token, depending on
# whether +text+ is a String.
def token text, kind
- if text.instance_of? ::String # Ruby 1.9: :open.is_a? String
- text_token text, kind
- elsif text.is_a? ::Symbol
- block_token text, kind
- else
- raise 'Unknown token text type: %p' % text
- end
+ out =
+ if text.is_a? ::String # Ruby 1.9: :open.is_a? String
+ text_token text, kind
+ elsif text.is_a? ::Symbol
+ block_token text, kind
+ else
+ raise 'Unknown token text type: %p' % text
+ end
+ @out << out if defined?(@out) && @out
end
def text_token text, kind
@@ -164,7 +168,8 @@
# The already created +tokens+ object must be used; it can be a
# TokenStream or a Tokens object.
def compile tokens, options
- tokens.each(&self)
+ tokens.each { |text, kind| token text, kind } # FIXME for Ruby 1.9?
+ #tokens.each(&self)
end
end
Index: lib/coderay/encoders/xml.rb
===================================================================
--- lib/coderay/encoders/xml.rb (revision 200)
+++ lib/coderay/encoders/xml.rb (revision 250)
@@ -22,7 +22,6 @@
protected
def setup options
- @out = ''
@doc = REXML::Document.new
@doc << REXML::XMLDecl.new
@tab_width = options[:tab_width]
@@ -33,7 +32,7 @@
@doc.write @out, options[:pretty], options[:transitive], true
@out
end
-
+
def text_token text, kind
if kind == :space
token = @node
Index: lib/coderay/encoders/html/classes.rb
===================================================================
--- lib/coderay/encoders/html/classes.rb (revision 200)
+++ lib/coderay/encoders/html/classes.rb (revision 250)
@@ -1,77 +0,0 @@
-module CodeRay
-module Encoders
-
- class HTML
-
- ClassOfKind = Hash.new do |h, k|
- h[k] = k.to_s
- end
- ClassOfKind.update with = {
- :attribute_name => 'an',
- :attribute_name_fat => 'af',
- :attribute_value => 'av',
- :attribute_value_fat => 'aw',
- :bin => 'bi',
- :char => 'ch',
- :class => 'cl',
- :class_variable => 'cv',
- :color => 'cr',
- :comment => 'c',
- :constant => 'co',
- :content => 'k',
- :definition => 'df',
- :delimiter => 'dl',
- :directive => 'di',
- :doc => 'do',
- :doc_string => 'ds',
- :entity => 'en',
- :error => 'er',
- :escape => 'e',
- :exception => 'ex',
- :float => 'fl',
- :function => 'fu',
- :global_variable => 'gv',
- :hex => 'hx',
- :include => 'ic',
- :inline => 'il',
- :inline_delimiter => 'idl',
- :instance_variable => 'iv',
- :integer => 'i',
- :interpreted => 'in',
- :label => 'la',
- :local_variable => 'lv',
- :modifier => 'mod',
- :oct => 'oc',
- :operator_name => 'on',
- :pre_constant => 'pc',
- :pre_type => 'pt',
- :predefined => 'pd',
- :preprocessor => 'pp',
- :regexp => 'rx',
- :reserved => 'r',
- :shell => 'sh',
- :string => 's',
- :symbol => 'sy',
- :tag => 'ta',
- :tag_fat => 'tf',
- :tag_special => 'ts',
- :type => 'ty',
- :variable => 'v',
- :xml_text => 'xt',
-
- :ident => :NO_HIGHLIGHT, # 'id'
- #:operator => 'op',
- :operator => :NO_HIGHLIGHT, # 'op'
- :space => :NO_HIGHLIGHT, # 'sp'
- :plain => :NO_HIGHLIGHT,
- }
- ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
- ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
- ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
- ClassOfKind[:escape] = ClassOfKind[:delimiter]
- #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
-
- end
-
-end
-end
Index: lib/coderay/encoders/html/numerization.rb
===================================================================
--- lib/coderay/encoders/html/numerization.rb (revision 200)
+++ lib/coderay/encoders/html/numerization.rb (revision 250)
@@ -51,12 +51,12 @@
case mode
when :inline
max_width = (start + line_count).to_s.size
- line = start
+ line_number = start
gsub!(/^/) do
- line_number = bolding.call line
- indent = ' ' * (max_width - line.to_s.size)
- res = "#{indent}#{line_number} "
- line += 1
+ line_number_text = bolding.call line_number
+ indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
+ res = "#{indent}#{line_number_text} "
+ line_number += 1
res
end
Index: lib/coderay/encoders/tokens.rb
===================================================================
--- lib/coderay/encoders/tokens.rb (revision 200)
+++ lib/coderay/encoders/tokens.rb (revision 250)
@@ -33,9 +33,9 @@
FILE_EXTENSION = 'tok'
- protected
- def token *args
- @out << CodeRay::Tokens.write_token(*args)
+ protected
+ def token text, kind
+ @out << CodeRay::Tokens.write_token(text, kind)
end
end
Index: lib/coderay/encoders/html.rb
===================================================================
--- lib/coderay/encoders/html.rb (revision 200)
+++ lib/coderay/encoders/html.rb (revision 250)
@@ -1,3 +1,5 @@
+require "set"
+
module CodeRay
module Encoders
@@ -10,7 +12,8 @@
#
# require 'coderay'
# puts CodeRay.scan('Some /code/', :ruby).html #-> a HTML page
- # puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span) #-> Some /code/
+ # puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span)
+ # #-> Some /code/
# puts CodeRay.scan('Some /code/', :ruby).span #-> the same
#
# puts CodeRay.scan('Some code', :ruby).html(
@@ -55,7 +58,8 @@
#
# === :hint
# Include some information into the output using the title attribute.
- # Can be :info (show token type on mouse-over), :info_long (with full path) or :debug (via inspect).
+ # Can be :info (show token type on mouse-over), :info_long (with full path)
+ # or :debug (via inspect).
#
# Default: false
class HTML < Encoder
@@ -82,7 +86,7 @@
:hint => false,
}
- helper :classes, :output, :css
+ helper :output, :css
attr_reader :css
@@ -115,11 +119,14 @@
end
}
+ TRANSPARENT_TOKEN_KINDS = [
+ :delimiter, :modifier, :content, :escape, :inline_delimiter,
+ ].to_set
+
# Generate a hint about the given +classes+ in a +hint+ style.
#
# +hint+ may be :info, :info_long or :debug.
def self.token_path_to_hint hint, classes
- return '' unless hint
title =
case hint
when :info
@@ -129,7 +136,7 @@
when :debug
classes.inspect
end
- " title=\"#{title}\""
+ title ? " title=\"#{title}\"" : ''
end
def setup options
@@ -143,42 +150,45 @@
hint = options[:hint]
if hint and not [:debug, :info, :info_long].include? hint
- raise ArgumentError, "Unknown value %p for :hint; expected :info, :debug, false or nil." % hint
+ raise ArgumentError, "Unknown value %p for :hint; \
+ expected :info, :debug, false, or nil." % hint
end
case options[:css]
when :class
@css_style = Hash.new do |h, k|
- if k.is_a? Array
- type = k.first
- else
- type = k
- end
- c = ClassOfKind[type]
+ c = CodeRay::Tokens::ClassOfKind[k.first]
if c == :NO_HIGHLIGHT and not hint
- h[k] = false
+ h[k.dup] = false
else
- title = HTML.token_path_to_hint hint, (k[1..-1] << k.first)
- h[k] = '' % [title, c]
+ title = if hint
+ HTML.token_path_to_hint(hint, k[1..-1] << k.first)
+ else
+ ''
+ end
+ if c == :NO_HIGHLIGHT
+ h[k.dup] = '' % [title]
+ else
+ h[k.dup] = '' % [title, c]
+ end
end
end
when :style
@css_style = Hash.new do |h, k|
- if k.is_a? Array
+ if k.is_a? ::Array
styles = k.dup
else
styles = [k]
end
type = styles.first
- classes = styles.map { |c| ClassOfKind[c] }
+ classes = styles.map { |c| Tokens::ClassOfKind[c] }
if classes.first == :NO_HIGHLIGHT and not hint
h[k] = false
else
- styles.shift if [:delimiter, :modifier, :content, :escape].include? styles.first
+ styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
title = HTML.token_path_to_hint hint, styles
- classes.delete 'il'
style = @css[*classes]
h[k] =
if style
@@ -198,7 +208,9 @@
def finish options
not_needed = @opened.shift
@out << '' * @opened.size
- warn '%d tokens still open: %p' % [@opened.size, @opened] unless @opened.empty?
+ unless @opened.empty?
+ warn '%d tokens still open: %p' % [@opened.size, @opened]
+ end
@out.extend Output
@out.css = @css
@@ -229,8 +241,9 @@
if @opened.empty?
# nothing to close
else
- if @opened.size == 1 or @opened.last != type
- raise 'Malformed token stream: Trying to close a token (%p) that is not open. Open are: %p.' % [type, @opened[1..-1]] if $DEBUG
+ if $DEBUG and (@opened.size == 1 or @opened.last != type)
+ raise 'Malformed token stream: Trying to close a token (%p) \
+ that is not open. Open are: %p.' % [type, @opened[1..-1]]
end
@out << ''
@opened.pop
Index: lib/coderay/encoders/text.rb
===================================================================
--- lib/coderay/encoders/text.rb (revision 200)
+++ lib/coderay/encoders/text.rb (revision 250)
@@ -14,13 +14,12 @@
protected
def setup options
- super
+ @out = ''
@sep = options[:separator]
end
def token text, kind
- return unless text.respond_to? :to_str
- @out << text + @sep
+ @out << text + @sep if text.is_a? ::String
end
def finish options
Index: lib/coderay/encoders/debug.rb
===================================================================
--- lib/coderay/encoders/debug.rb (revision 200)
+++ lib/coderay/encoders/debug.rb (revision 250)
@@ -19,19 +19,14 @@
protected
def text_token text, kind
- @out <<
- if kind == :space
- text
- else
- text = text.gsub(/[)\\]/, '\\\\\0')
- "#{kind}(#{text})"
- end
+ if kind == :space
+ text
+ else
+ text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \
+ "#{kind}(#{text})"
+ end
end
- def block_token action, kind
- @out << super
- end
-
def open_token kind
"#{kind}<"
end
Index: lib/coderay/encoders/statistic.rb
===================================================================
--- lib/coderay/encoders/statistic.rb (revision 200)
+++ lib/coderay/encoders/statistic.rb (revision 250)
@@ -28,19 +28,15 @@
@type_stats[kind].count += 1
@type_stats[kind].size += text.size
@type_stats['TOTAL'].size += text.size
+ @type_stats['TOTAL'].count += 1
end
# TODO Hierarchy handling
def block_token action, kind
- #@content_type = kind
+ @type_stats['TOTAL'].count += 1
@type_stats['open/close'].count += 1
end
- def token text, kind
- super
- @type_stats['TOTAL'].count += 1
- end
-
STATS = <<-STATS
Code Statistics
Index: lib/coderay/encoders/_map.rb
===================================================================
--- lib/coderay/encoders/_map.rb (revision 200)
+++ lib/coderay/encoders/_map.rb (revision 250)
@@ -2,7 +2,8 @@
module Encoders
map :stats => :statistic,
- :plain => :text
+ :plain => :text,
+ :tex => :latex
end
end
Index: lib/coderay/helpers/filetype.rb
===================================================================
--- lib/coderay/helpers/filetype.rb (revision 200)
+++ lib/coderay/helpers/filetype.rb (revision 250)
@@ -1,180 +0,0 @@
-# =FileType
-#
-# A simple filetype recognizer
-#
-# Author: murphy (mail to murphy cYcnus de)
-#
-# Version: 0.1 (2005.september.1)
-#
-# == Documentation
-#
-# # determine the type of the given
-# lang = FileType[ARGV.first]
-#
-# # return :plaintext if the file type is unknown
-# lang = FileType.fetch ARGV.first, :plaintext
-#
-# # try the shebang line, too
-# lang = FileType.fetch ARGV.first, :plaintext, true
-module FileType
-
- UnknownFileType = Class.new Exception
-
- class << self
-
- # Try to determine the file type of the file.
- #
- # +filename+ is a relative or absolute path to a file.
- #
- # The file itself is only accessed when +read_shebang+ is set to true.
- # That means you can get filetypes from files that don't exist.
- def [] filename, read_shebang = false
- name = File.basename filename
- ext = File.extname name
- ext.sub!(/^\./, '') # delete the leading dot
-
- type =
- TypeFromExt[ext] ||
- TypeFromExt[ext.downcase] ||
- TypeFromName[name] ||
- TypeFromName[name.downcase]
- type ||= shebang(filename) if read_shebang
-
- type
- end
-
- def shebang filename
- begin
- File.open filename, 'r' do |f|
- first_line = f.gets
- first_line[TypeFromShebang]
- end
- rescue IOError
- nil
- end
- end
-
- # This works like Hash#fetch.
- #
- # If the filetype cannot be found, the +default+ value
- # is returned.
- def fetch filename, default = nil, read_shebang = false
- if default and block_given?
- warn 'block supersedes default value argument'
- end
-
- unless type = self[filename, read_shebang]
- return yield if block_given?
- return default if default
- raise UnknownFileType, 'Could not determine type of %p.' % filename
- end
- type
- end
-
- end
-
- TypeFromExt = {
- 'rb' => :ruby,
- 'rbw' => :ruby,
- 'rake' => :ruby,
- 'cpp' => :c,
- 'c' => :c,
- 'h' => :c,
- 'xml' => :xml,
- 'htm' => :html,
- 'html' => :html,
- 'xhtml' => :xhtml,
- 'rhtml' => :rhtml,
- 'yaml' => :yaml,
- 'yml' => :yaml,
- }
-
- TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
-
- TypeFromName = {
- 'Rakefile' => :ruby,
- 'Rantfile' => :ruby,
- }
-
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- eval DATA.read, nil, $0, __LINE__+4
-end
-
-__END__
-
-require 'test/unit'
-
-class TC_FileType < Test::Unit::TestCase
-
- def test_fetch
- assert_raise FileType::UnknownFileType do
- FileType.fetch ''
- end
-
- assert_throws :not_found do
- FileType.fetch '.' do
- throw :not_found
- end
- end
-
- assert_equal :default, FileType.fetch('c', :default)
-
- stderr, fake_stderr = $stderr, Object.new
- $err = ''
- def fake_stderr.write x
- $err << x
- end
- $stderr = fake_stderr
- FileType.fetch('c', :default) { }
- assert_equal "block supersedes default value argument\n", $err
- $stderr = stderr
- end
-
- def test_ruby
- assert_equal :ruby, FileType['test.rb']
- assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
- assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
- assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
- assert_equal :ruby, FileType['./lib/tasks\repository.rake']
- assert_not_equal :ruby, FileType['test_rb']
- assert_not_equal :ruby, FileType['Makefile']
- assert_not_equal :ruby, FileType['set.rb/set']
- assert_not_equal :ruby, FileType['~/projects/blabla/rb']
- end
-
- def test_c
- assert_equal :c, FileType['test.c']
- assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
- assert_not_equal :c, FileType['test_c']
- assert_not_equal :c, FileType['Makefile']
- assert_not_equal :c, FileType['set.h/set']
- assert_not_equal :c, FileType['~/projects/blabla/c']
- end
-
- def test_html
- assert_equal :html, FileType['test.htm']
- assert_equal :xhtml, FileType['test.xhtml']
- assert_equal :xhtml, FileType['test.html.xhtml']
- assert_equal :rhtml, FileType['_form.rhtml']
- end
-
- def test_yaml
- assert_equal :yaml, FileType['test.yml']
- assert_equal :yaml, FileType['test.yaml']
- assert_equal :yaml, FileType['my.html.yaml']
- assert_not_equal :yaml, FileType['YAML']
- end
-
- def test_shebang
- dir = './test'
- if File.directory? dir
- Dir.chdir dir do
- assert_equal :c, FileType['test.c']
- end
- end
- end
-
-end
Index: lib/coderay/helpers/plugin.rb
===================================================================
--- lib/coderay/helpers/plugin.rb (revision 200)
+++ lib/coderay/helpers/plugin.rb (revision 250)
@@ -1,3 +1,5 @@
+module CodeRay
+
# = PluginHost
#
# $Id$
@@ -20,7 +22,7 @@
#
# Generators[:fancy] #-> FancyGenerator
# # or
-# require_plugin 'Generators/fancy'
+# CodeRay.require_plugin 'Generators/fancy'
module PluginHost
# Raised if Encoders::[] fails because:
@@ -310,17 +312,18 @@
end
-
# Convenience method for plugin loading.
# The syntax used is:
#
-# require_plugin '/'
+# CodeRay.require_plugin '/'
#
# Returns the loaded plugin.
-def require_plugin path
+def self.require_plugin path
host_id, plugin_id = path.split '/', 2
host = PluginHost.host_by_id(host_id)
raise PluginHost::HostNotFound,
"No host for #{host_id.inspect} found." unless host
host.load plugin_id
end
+
+end
\ No newline at end of file
Index: lib/coderay/helpers/file_type.rb
===================================================================
--- lib/coderay/helpers/file_type.rb (revision 0)
+++ lib/coderay/helpers/file_type.rb (revision 250)
@@ -0,0 +1,210 @@
+#!/usr/bin/env ruby
+module CodeRay
+
+# = FileType
+#
+# A simple filetype recognizer.
+#
+# Copyright (c) 2006 by murphy (Kornelius Kalnbach)
+#
+# License:: LGPL / ask the author
+# Version:: 0.1 (2005-09-01)
+#
+# == Documentation
+#
+# # determine the type of the given
+# lang = FileType[ARGV.first]
+#
+# # return :plaintext if the file type is unknown
+# lang = FileType.fetch ARGV.first, :plaintext
+#
+# # try the shebang line, too
+# lang = FileType.fetch ARGV.first, :plaintext, true
+module FileType
+
+ UnknownFileType = Class.new Exception
+
+ class << self
+
+ # Try to determine the file type of the file.
+ #
+ # +filename+ is a relative or absolute path to a file.
+ #
+ # The file itself is only accessed when +read_shebang+ is set to true.
+ # That means you can get filetypes from files that don't exist.
+ def [] filename, read_shebang = false
+ name = File.basename filename
+ ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot
+ ext2 = filename[/\.(.*)/, 1] # from first dot
+
+ type =
+ TypeFromExt[ext.downcase] ||
+ (TypeFromExt[ext2.downcase] if ext2) ||
+ TypeFromName[name] ||
+ TypeFromName[name.downcase]
+ type ||= shebang(filename) if read_shebang
+
+ type
+ end
+
+ def shebang filename
+ begin
+ File.open filename, 'r' do |f|
+ if first_line = f.gets
+ if type = first_line[TypeFromShebang]
+ type.to_sym
+ end
+ end
+ end
+ rescue IOError
+ nil
+ end
+ end
+
+ # This works like Hash#fetch.
+ #
+ # If the filetype cannot be found, the +default+ value
+ # is returned.
+ def fetch filename, default = nil, read_shebang = false
+ if default and block_given?
+ warn 'block supersedes default value argument'
+ end
+
+ unless type = self[filename, read_shebang]
+ return yield if block_given?
+ return default if default
+ raise UnknownFileType, 'Could not determine type of %p.' % filename
+ end
+ type
+ end
+
+ end
+
+ TypeFromExt = {
+ 'rb' => :ruby,
+ 'rbw' => :ruby,
+ 'rake' => :ruby,
+ 'mab' => :ruby,
+ 'cpp' => :c,
+ 'c' => :c,
+ 'h' => :c,
+ 'xml' => :xml,
+ 'htm' => :html,
+ 'html' => :html,
+ 'xhtml' => :xhtml,
+ 'raydebug' => :debug,
+ 'rhtml' => :rhtml,
+ 'html.erb' => :rhtml,
+ 'ss' => :scheme,
+ 'sch' => :scheme,
+ 'yaml' => :yaml,
+ 'yml' => :yaml,
+ }
+
+ TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
+
+ TypeFromName = {
+ 'Rakefile' => :ruby,
+ 'Rantfile' => :ruby,
+ }
+
+end
+
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ eval DATA.read, nil, $0, __LINE__+4
+end
+
+__END__
+require 'test/unit'
+
+class TC_FileType < Test::Unit::TestCase
+
+ include CodeRay
+
+ def test_fetch
+ assert_raise FileType::UnknownFileType do
+ FileType.fetch ''
+ end
+
+ assert_throws :not_found do
+ FileType.fetch '.' do
+ throw :not_found
+ end
+ end
+
+ assert_equal :default, FileType.fetch('c', :default)
+
+ stderr, fake_stderr = $stderr, Object.new
+ $err = ''
+ def fake_stderr.write x
+ $err << x
+ end
+ $stderr = fake_stderr
+ FileType.fetch('c', :default) { }
+ assert_equal "block supersedes default value argument\n", $err
+ $stderr = stderr
+ end
+
+ def test_ruby
+ assert_equal :ruby, FileType['test.rb']
+ assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
+ assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
+ assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
+ assert_equal :ruby, FileType['./lib/tasks\repository.rake']
+ assert_not_equal :ruby, FileType['test_rb']
+ assert_not_equal :ruby, FileType['Makefile']
+ assert_not_equal :ruby, FileType['set.rb/set']
+ assert_not_equal :ruby, FileType['~/projects/blabla/rb']
+ end
+
+ def test_c
+ assert_equal :c, FileType['test.c']
+ assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
+ assert_not_equal :c, FileType['test_c']
+ assert_not_equal :c, FileType['Makefile']
+ assert_not_equal :c, FileType['set.h/set']
+ assert_not_equal :c, FileType['~/projects/blabla/c']
+ end
+
+ def test_html
+ assert_equal :html, FileType['test.htm']
+ assert_equal :xhtml, FileType['test.xhtml']
+ assert_equal :xhtml, FileType['test.html.xhtml']
+ assert_equal :rhtml, FileType['_form.rhtml']
+ assert_equal :rhtml, FileType['_form.html.erb']
+ end
+
+ def test_yaml
+ assert_equal :yaml, FileType['test.yml']
+ assert_equal :yaml, FileType['test.yaml']
+ assert_equal :yaml, FileType['my.html.yaml']
+ assert_not_equal :yaml, FileType['YAML']
+ end
+
+ def test_no_shebang
+ dir = './test'
+ if File.directory? dir
+ Dir.chdir dir do
+ assert_equal :c, FileType['test.c']
+ end
+ end
+ end
+
+ def test_shebang_empty_file
+ require 'tmpdir'
+ tmpfile = File.join(Dir.tmpdir, 'bla')
+ File.open(tmpfile, 'w') { } # touch
+ assert_equal nil, FileType[tmpfile]
+ end
+
+ def test_shebang
+ require 'tmpdir'
+ tmpfile = File.join(Dir.tmpdir, 'bla')
+ File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
+ assert_equal :ruby, FileType[tmpfile, true]
+ end
+
+end
Property changes on: lib/coderay/helpers/file_type.rb
___________________________________________________________________
Added: svn:keywords
+ Id Rev
Index: lib/coderay/helpers/gzip_simple.rb
===================================================================
--- lib/coderay/helpers/gzip_simple.rb (revision 200)
+++ lib/coderay/helpers/gzip_simple.rb (revision 250)
@@ -46,6 +46,7 @@
end
end
+
# String extensions to use the GZip module.
#
# The methods gzip and gunzip provide an even more simple
Index: lib/coderay/helpers/word_list.rb
===================================================================
--- lib/coderay/helpers/word_list.rb (revision 200)
+++ lib/coderay/helpers/word_list.rb (revision 250)
@@ -1,15 +1,19 @@
+module CodeRay
+
# = WordList
+#
+# A Hash subclass designed for mapping word lists to token types.
+#
+# Copyright (c) 2006 by murphy (Kornelius Kalnbach)
#
-# Copyright (c) 2006 by murphy (Kornelius Kalnbach)
-#
# License:: LGPL / ask the author
-# Version:: 1.0 (2006-Feb-3)
+# Version:: 1.1 (2006-Oct-19)
#
# A WordList is a Hash with some additional features.
# It is intended to be used for keyword recognition.
#
# WordList is highly optimized to be used in Scanners,
-# typically to decide whether a given ident is a keyword.
+# typically to decide whether a given ident is a special token.
#
# For case insensitive words use CaseIgnoringWordList.
#
@@ -47,25 +51,30 @@
# ...
class WordList < Hash
- # Create a WordList for the given +words+.
- #
- # This WordList responds to [] with +true+, if the word is
- # in +words+, and with +false+ otherwise.
- def self.for words
- new.add words
- end
-
# Creates a new WordList with +default+ as default value.
- def initialize default = false, &block
- super default, &block
+ #
+ # You can activate +caching+ to store the results for every [] request.
+ #
+ # With caching, methods like +include?+ or +delete+ may no longer behave
+ # as you expect. Therefore, it is recommended to use the [] method only.
+ def initialize default = false, caching = false, &block
+ if block
+ raise ArgumentError, 'Can\'t combine block with caching.' if caching
+ super(&block)
+ else
+ if caching
+ super() do |h, k|
+ h[k] = h.fetch k, default
+ end
+ else
+ super default
+ end
+ end
end
- # Checks if a word is included.
- def include? word
- has_key? word
- end
-
# Add words to the list and associate them with +kind+.
+ #
+ # Returns +self+, so you can concat add calls.
def add words, kind = true
words.each do |word|
self[word] = kind
@@ -78,24 +87,30 @@
# A CaseIgnoringWordList is like a WordList, only that
# keys are compared case-insensitively.
+#
+# Ignoring the text case is realized by sending the +downcase+ message to
+# all keys.
+#
+# Caching usually makes a CaseIgnoringWordList faster, but it has to be
+# activated explicitely.
class CaseIgnoringWordList < WordList
- # Creates a new WordList with +default+ as default value.
- #
- # Text case is ignored.
- def initialize default = false, &block
- block ||= proc do |h, k|
- h[k] = h.fetch k.downcase, default
+ # Creates a new case-insensitive WordList with +default+ as default value.
+ #
+ # You can activate caching to store the results for every [] request.
+ def initialize default = false, caching = false
+ if caching
+ super(default, false) do |h, k|
+ h[k] = h.fetch k.downcase, default
+ end
+ else
+ def self.[] key # :nodoc:
+ super(key.downcase)
+ end
end
- super default
end
- # Checks if a word is included.
- def include? word
- has_key? word.downcase
- end
-
- # Add words to the list and associate them with +kind+.
+ # Add +words+ to the list and associate them with +kind+.
def add words, kind = true
words.each do |word|
self[word.downcase] = kind
@@ -104,3 +119,5 @@
end
end
+
+end
\ No newline at end of file
Index: lib/coderay/styles/cycnus.rb
===================================================================
--- lib/coderay/styles/cycnus.rb (revision 200)
+++ lib/coderay/styles/cycnus.rb (revision 250)
@@ -42,12 +42,14 @@
MAIN
TOKEN_COLORS = <<-'TOKENS'
+.debug { color:white ! important; background:blue ! important; }
+
.af { color:#00C }
.an { color:#007 }
.av { color:#700 }
.aw { color:#C00 }
.bi { color:#509; font-weight:bold }
-.c { color:#888 }
+.c { color:#666; }
.ch { color:#04D }
.ch .k { color:#04D }
@@ -83,7 +85,7 @@
.la { color:#970; font-weight:bold }
.lv { color:#963 }
.oc { color:#40E; font-weight:bold }
-.on { color:#000; font-weight:bold }
+.of { color:#000; font-weight:bold }
.op { }
.pc { color:#038; font-weight:bold }
.pd { color:#369; font-weight:bold }
Index: lib/coderay/styles/murphy.rb
===================================================================
--- lib/coderay/styles/murphy.rb (revision 200)
+++ lib/coderay/styles/murphy.rb (revision 250)
@@ -47,7 +47,7 @@
.av { color:#700; }
.aw { color:#C00; }
.bi { color:#509; font-weight:bold; }
-.c { color:#666; }
+.c { color:#555; background-color: black; }
.ch { color:#88F; }
.ch .k { color:#04D; }
@@ -77,7 +77,7 @@
.la { color:#970; font-weight:bold; }
.lv { color:#963; }
.oc { color:#40E; font-weight:bold; }
-.on { color:#000; font-weight:bold; }
+.of { color:#000; font-weight:bold; }
.op { }
.pc { color:#08f; font-weight:bold; }
.pd { color:#369; font-weight:bold; }
Index: lib/coderay/tokens.rb
===================================================================
--- lib/coderay/tokens.rb (revision 200)
+++ lib/coderay/tokens.rb (revision 250)
@@ -115,7 +115,7 @@
# tokens.each_text_token { |text, kind| text.replace html_escape(text) }
def each_text_token
each do |text, kind|
- next unless text.respond_to? :to_str
+ next unless text.is_a? ::String
yield text, kind
end
end
@@ -252,7 +252,7 @@
#
# You can configure the level of compression,
# but the default value 7 should be what you want
- # in most cases as it is a good comprimise between
+ # in most cases as it is a good compromise between
# speed and compression rate.
#
# See GZip module.
@@ -267,9 +267,20 @@
# Should be equal to the input size before
# scanning.
def text_size
- map { |t, k| t }.join.size
+ size = 0
+ each_text_token do |t, k|
+ size + t.size
+ end
+ size
end
+ # The total size of the tokens.
+ # Should be equal to the input size before
+ # scanning.
+ def text
+ map { |t, k| t if t.is_a? ::String }.join
+ end
+
# Include this module to give an object an #undump
# method.
#
@@ -342,7 +353,7 @@
#
# Returns self.
def << token
- @callback.call token
+ @callback.call(*token)
@size += 1
self
end
@@ -365,4 +376,8 @@
end
+
+ # Token name abbreviations
+ require 'coderay/token_classes'
+
end
Index: lib/coderay/duo.rb
===================================================================
--- lib/coderay/duo.rb (revision 200)
+++ lib/coderay/duo.rb (revision 250)
@@ -4,26 +4,84 @@
#
# $Id: scanner.rb 123 2006-03-21 14:46:34Z murphy $
#
- # TODO: Doc.
+ # A Duo is a convenient way to use CodeRay. You just create a Duo,
+ # giving it a lang (language of the input code) and a format (desired
+ # output format), and call Duo#highlight with the code.
+ #
+ # Duo makes it easy to re-use both scanner and encoder for a repetitive
+ # task. It also provides a very easy interface syntax:
+ #
+ # require 'coderay'
+ # CodeRay::Duo[:python, :div].highlight 'import this'
+ #
+ # Until you want to do uncommon things with CodeRay, I recommend to use
+ # this method, since it takes care of everything.
class Duo
- attr_accessor :scanner, :encoder
-
- def initialize lang, format, options = {}
- @scanner = CodeRay.scanner lang, CodeRay.get_scanner_options(options)
- @encoder = CodeRay.encoder format, options
+ attr_accessor :lang, :format, :options
+
+ # Create a new Duo, holding a lang and a format to highlight code.
+ #
+ # simple:
+ # CodeRay::Duo[:ruby, :page].highlight 'bla 42'
+ #
+ # streaming:
+ # CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
+ #
+ # with options:
+ # CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
+ #
+ # alternative syntax without options:
+ # CodeRay::Duo[:ruby => :statistic].encode 'class << self; end'
+ #
+ # alternative syntax with options:
+ # CodeRay::Duo[{ :ruby => :statistic }, :do => :something].encode 'abc'
+ #
+ # The options are forwarded to scanner and encoder
+ # (see CodeRay.get_scanner_options).
+ def initialize lang = nil, format = nil, options = {}
+ if format == nil and lang.is_a? Hash and lang.size == 1
+ @lang = lang.keys.first
+ @format = lang[@lang]
+ else
+ @lang = lang
+ @format = format
+ end
+ @options = options
end
class << self
+ # To allow calls like Duo[:ruby, :html].highlight.
alias [] new
end
- def encode code
- @scanner.string = code
- @encoder.encode_tokens(scanner.tokenize)
+ # The scanner of the duo. Only created once.
+ def scanner
+ @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
end
+
+ # The encoder of the duo. Only created once.
+ def encoder
+ @encoder ||= CodeRay.encoder @format, @options
+ end
+
+ # Tokenize and highlight the code using +scanner+ and +encoder+.
+ #
+ # If the :stream option is set, the Duo will go into streaming mode,
+ # saving memory for the cost of time.
+ def encode code, options = { :stream => false }
+ stream = options.delete :stream
+ options = @options.merge options
+ if stream
+ encoder.encode_stream(code, @lang, options)
+ else
+ scanner.code = code
+ encoder.encode_tokens(scanner.tokenize, options)
+ end
+ end
alias highlight encode
end
end
+
Index: lib/coderay/scanner.rb
===================================================================
--- lib/coderay/scanner.rb (revision 200)
+++ lib/coderay/scanner.rb (revision 250)
@@ -66,8 +66,18 @@
end
def normify code
- code = code.to_s.to_unix
+ code = code.to_s
+ code.force_encoding 'binary' if code.respond_to? :force_encoding
+ code.to_unix
end
+
+ def file_extension extension = nil
+ if extension
+ @file_extension = extension.to_s
+ else
+ @file_extension ||= plugin_id.to_s
+ end
+ end
end
@@ -117,9 +127,6 @@
setup
end
- # More mnemonic accessor name for the input string.
- alias code string
-
def reset
super
reset_instance
@@ -131,6 +138,10 @@
reset_instance
end
+ # More mnemonic accessor name for the input string.
+ alias code string
+ alias code= string=
+
# Scans the code and returns all tokens in a Tokens object.
def tokenize new_string=nil, options = {}
options = @options.merge(options)
@@ -148,6 +159,11 @@
def tokens
@cached_tokens ||= tokenize
end
+
+ # Whether the scanner is in streaming mode.
+ def streaming?
+ !!@options[:stream]
+ end
# Traverses the tokens.
def each &block
@@ -195,7 +211,7 @@
raise ScanError, <<-EOE % [
-***ERROR in %s: %s
+***ERROR in %s: %s (after %d tokens)
tokens:
%s
@@ -211,13 +227,14 @@
***ERROR***
EOE
- File.basename(caller[0]),
- msg,
- tokens.last(10).map { |t| t.inspect }.join("\n"),
- line, pos,
- matched, state, bol?, eos?,
- string[pos-ambit,ambit],
- string[pos,ambit],
+ File.basename(caller[0]),
+ msg,
+ tokens.size,
+ tokens.last(10).map { |t| t.inspect }.join("\n"),
+ line, pos,
+ matched, state, bol?, eos?,
+ string[pos-ambit,ambit],
+ string[pos,ambit],
]
end
Index: lib/coderay/for_redcloth.rb
===================================================================
--- lib/coderay/for_redcloth.rb (revision 0)
+++ lib/coderay/for_redcloth.rb (revision 250)
@@ -0,0 +1,72 @@
+module CodeRay # :nodoc:
+
+ # A little hack to enable CodeRay highlighting in RedCloth.
+ #
+ # Usage:
+ # require 'coderay'
+ # require 'coderay/for_redcloth'
+ # RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
+ #
+ # Make sure you have RedCloth 4.0.3 activated, for example by calling
+ # require 'rubygems'
+ # before RedCloth is loaded and before calling CodeRay.for_redcloth.
+ module ForRedCloth
+
+ def self.install
+ gem 'RedCloth', '>= 4.0.3' rescue nil
+ require 'redcloth'
+ raise 'CodeRay.for_redcloth needs RedCloth 4.0.3 or later.' unless RedCloth::VERSION.to_s >= '4.0.3'
+ RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
+ RedCloth::Formatters::HTML.module_eval do
+ def unescape(html)
+ replacements = {
+ '&' => '&',
+ '"' => '"',
+ '>' => '>',
+ '<' => '<',
+ }
+ html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] }
+ end
+ undef_method :code, :bc_open, :bc_close, :escape_pre
+ def code(opts) # :nodoc:
+ opts[:block] = true
+ if opts[:lang] && !filter_coderay
+ require 'coderay'
+ @in_bc ||= nil
+ format = @in_bc ? :div : :span
+ highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
+ highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
+ highlighted_code = unescape(highlighted_code) unless @in_bc
+ highlighted_code
+ else
+ "#{opts[:text]}"
+ end
+ end
+ def bc_open(opts) # :nodoc:
+ opts[:block] = true
+ @in_bc = opts
+ opts[:lang] ? '' : ""
+ end
+ def bc_close(opts) # :nodoc:
+ @in_bc = nil
+ opts[:lang] ? '' : "\n"
+ end
+ def escape_pre(text)
+ if @in_bc ||= nil
+ text
+ else
+ html_esc(text, :html_escape_preformatted)
+ end
+ end
+ end
+ end
+
+ module TextileDoc # :nodoc:
+ attr_accessor :filter_coderay
+ end
+
+ end
+
+end
+
+CodeRay::ForRedCloth.install
\ No newline at end of file
Index: lib/coderay/scanners/ruby/patterns.rb
===================================================================
--- lib/coderay/scanners/ruby/patterns.rb (revision 200)
+++ lib/coderay/scanners/ruby/patterns.rb (revision 250)
@@ -14,19 +14,14 @@
DEF_KEYWORDS = %w[ def ]
UNDEF_KEYWORDS = %w[ undef ]
+ ALIAS_KEYWORDS = %w[ alias ]
MODULE_KEYWORDS = %w[class module]
DEF_NEW_STATE = WordList.new(:initial).
add(DEF_KEYWORDS, :def_expected).
add(UNDEF_KEYWORDS, :undef_expected).
+ add(ALIAS_KEYWORDS, :alias_expected).
add(MODULE_KEYWORDS, :module_expected)
- IDENTS_ALLOWING_REGEXP = %w[
- and or not while until unless if then elsif when sub sub! gsub gsub!
- scan slice slice! split
- ]
- REGEXP_ALLOWED = WordList.new(false).
- add(IDENTS_ALLOWING_REGEXP, :set)
-
PREDEFINED_CONSTANTS = %w[
nil true false self
DATA ARGV ARGF __FILE__ __LINE__
@@ -41,19 +36,20 @@
METHOD_NAME = / #{IDENT} [?!]? /ox
METHOD_NAME_OPERATOR = /
\*\*? # multiplication and power
- | [-+]@? # plus, minus
- | [\/%&|^`~] # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
+ | [-+~]@? # plus, minus, tilde with and without @
+ | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system`
| \[\]=? # array getter and setter
| << | >> # append or shift left, shift right
| <=?>? | >=? # comparison, rocket operator
- | ===? # simple equality and case equality
+ | ===? | =~ # simple equality, case equality, match
+ | ![~=@]? # negation with and without @, not-equal and not-match
/ox
METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
INSTANCE_VARIABLE = / @ #{IDENT} /ox
CLASS_VARIABLE = / @@ #{IDENT} /ox
OBJECT_VARIABLE = / @@? #{IDENT} /ox
GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
- PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
+ PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox
VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
QUOTE_TO_TYPE = {
@@ -73,7 +69,7 @@
EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
- NUMERIC = / [-+]? (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
+ NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
SYMBOL = /
:
@@ -83,6 +79,7 @@
| ['"]
)
/ox
+ METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox
# TODO investigste \M, \c and \C escape sequences
# (?: M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-)? (?: \\ (?: [0-7]{3} | x[0-9A-Fa-f]{2} | . ) )
@@ -111,7 +108,7 @@
(?:
( [A-Za-z_0-9]+ ) # $2 = delim
|
- ( ["'`] ) # $3 = quote, type
+ ( ["'`\/] ) # $3 = quote, type
( [^\n]*? ) \3 # $4 = delim
)
/mx
@@ -129,15 +126,14 @@
/mx
# Checks for a valid value to follow. This enables
- # fancy_allowed in method calls.
+ # value_expected in method calls without parentheses.
VALUE_FOLLOWS = /
- \s+
+ (?>[ \t\f\v]+)
(?:
[%\/][^\s=]
- |
- <<-?\S
- |
- #{CHARACTER}
+ | <<-?\S
+ | [-+] \d
+ | #{CHARACTER}
)
/x
Index: lib/coderay/scanners/ruby.rb
===================================================================
--- lib/coderay/scanners/ruby.rb (revision 200)
+++ lib/coderay/scanners/ruby.rb (revision 250)
@@ -18,6 +18,7 @@
include Streamable
register_for :ruby
+ file_extension 'rb'
helper :patterns
@@ -90,15 +91,15 @@
end
when '#'
- case peek(1)[0]
- when ?{
+ case peek(1)
+ when '{'
inline_block_stack << [state, depth, heredocs]
value_expected = true
state = :initial
depth = 1
tokens << [:open, :inline]
tokens << [match + getch, :inline_delimiter]
- when ?$, ?@
+ when '$', '@'
tokens << [match, :escape]
last_state = state # scan one token as normal code, then return here
state = :initial
@@ -121,36 +122,37 @@
# }}}
else
# {{{
- if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
- ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
- case m = match[0]
- when ?\s, ?\t, ?\f
- match << scan(/\s*/) unless eos? or heredocs
- kind = :space
- when ?\n, ?\\
- kind = :space
- if m == ?\n
- value_expected = true # FIXME not quite true
- state = :initial if state == :undef_comma_expected
- end
- if heredocs
- unscan # heredoc scanning needs \n at start
- state = heredocs.shift
- tokens << [:open, state.type]
- heredocs = nil if heredocs.empty?
- next
- else
- match << scan(/\s*/) unless eos?
- end
- when ?#, ?=, ?_
- kind = :comment
- value_expected = true
+ if match = scan(/[ \t\f]+/)
+ kind = :space
+ match << scan(/\s*/) unless eos? || heredocs
+ value_expected = true if match.index(?\n) # FIXME not quite true
+ tokens << [match, kind]
+ next
+
+ elsif match = scan(/\\?\n/)
+ kind = :space
+ if match == "\n"
+ value_expected = true # FIXME not quite true
+ state = :initial if state == :undef_comma_expected
+ end
+ if heredocs
+ unscan # heredoc scanning needs \n at start
+ state = heredocs.shift
+ tokens << [:open, state.type]
+ heredocs = nil if heredocs.empty?
+ next
else
- raise_inspect 'else-case _ reached, because case %p was
- not handled' % [matched[0].chr], tokens
+ match << scan(/\s*/) unless eos?
end
tokens << [match, kind]
next
+
+ elsif match = scan(/\#.*/) or
+ ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
+ kind = :comment
+ value_expected = true
+ tokens << [match, kind]
+ next
elsif state == :initial
@@ -167,19 +169,19 @@
end
end
## experimental!
- value_expected = :set if
- patterns::REGEXP_ALLOWED[match] or check(/#{patterns::VALUE_FOLLOWS}/o)
+ value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o)
kind = :ident
value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
# OPERATORS #
- elsif not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)
+ # TODO: match (), [], {} as one single operator
+ elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
value_expected = :set
end
- last_token_dot = :set if match == '.' or match == '::'
+ last_token_dot = :set if self[1]
kind = :operator
unless inline_block_stack.empty?
case match
@@ -210,8 +212,9 @@
interpreted = true
state = patterns::StringState.new :regexp, interpreted, match
- elsif match = scan(/#{patterns::NUMERIC}/o)
- kind = if self[1] then :float else :integer end
+ # elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
+ elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
+ kind = self[1] ? :float : :integer
elsif match = scan(/#{patterns::SYMBOL}/o)
case delim = match[1]
@@ -285,6 +288,18 @@
next
end
+ elsif state == :module_expected
+ if match = scan(/<\?~_\^]/i
+ #IDENTIFIER_SUBSEQUENT = /#{IDENTIFIER_INITIAL}|\d|\.|\+|-/
+ #IDENTIFIER = /#{IDENTIFIER_INITIAL}#{IDENTIFIER_SUBSEQUENT}*|\+|-|\.{3}/
+ IDENTIFIER = /[a-zA-Z!@$%&*\/:<=>?~_^][\w!@$%&*\/:<=>?~^.+\-]*|[+-]|\.\.\./
+ DIGIT = /\d/
+ DIGIT10 = DIGIT
+ DIGIT16 = /[0-9a-f]/i
+ DIGIT8 = /[0-7]/
+ DIGIT2 = /[01]/
+ RADIX16 = /\#x/i
+ RADIX8 = /\#o/i
+ RADIX2 = /\#b/i
+ RADIX10 = /\#d/i
+ EXACTNESS = /#i|#e/i
+ SIGN = /[\+-]?/
+ EXP_MARK = /[esfdl]/i
+ EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/
+ SUFFIX = /#{EXP}?/
+ PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/
+ PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/
+ PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/
+ PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/
+ UINT10 = /#{DIGIT10}+#*/
+ UINT16 = /#{DIGIT16}+#*/
+ UINT8 = /#{DIGIT8}+#*/
+ UINT2 = /#{DIGIT2}+#*/
+ DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/
+ UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/
+ UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/
+ UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/
+ UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/
+ REAL10 = /#{SIGN}#{UREAL10}/
+ REAL16 = /#{SIGN}#{UREAL16}/
+ REAL8 = /#{SIGN}#{UREAL8}/
+ REAL2 = /#{SIGN}#{UREAL2}/
+ IMAG10 = /i|#{UREAL10}i/
+ IMAG16 = /i|#{UREAL16}i/
+ IMAG8 = /i|#{UREAL8}i/
+ IMAG2 = /i|#{UREAL2}i/
+ COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/
+ COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/
+ COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/
+ COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/
+ NUM10 = /#{PREFIX10}?#{COMPLEX10}/
+ NUM16 = /#{PREFIX16}#{COMPLEX16}/
+ NUM8 = /#{PREFIX8}#{COMPLEX8}/
+ NUM2 = /#{PREFIX2}#{COMPLEX2}/
+ NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/
+
+ private
+ def scan_tokens tokens,options
+
+ state = :initial
+ ident_kind = IDENT_KIND
+
+ until eos?
+ kind = match = nil
+
+ case state
+ when :initial
+ if scan(/ \s+ | \\\n /x)
+ kind = :space
+ elsif scan(/['\(\[\)\]]|#\(/)
+ kind = :operator_fat
+ elsif scan(/;.*/)
+ kind = :comment
+ elsif scan(/#\\(?:newline|space|.?)/)
+ kind = :char
+ elsif scan(/#[ft]/)
+ kind = :pre_constant
+ elsif scan(/#{IDENTIFIER}/o)
+ kind = ident_kind[matched]
+ elsif scan(/\./)
+ kind = :operator
+ elsif scan(/"/)
+ tokens << [:open, :string]
+ state = :string
+ tokens << ['"', :delimiter]
+ next
+ elsif scan(/#{NUM}/o) and not matched.empty?
+ kind = :integer
+ elsif getch
+ kind = :error
+ end
+
+ when :string
+ if scan(/[^"\\]+/) or scan(/\\.?/)
+ kind = :content
+ elsif scan(/"/)
+ tokens << ['"', :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ next
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1),
+ tokens, state
+ end
+
+ else
+ raise "else case reached"
+ end
+
+ match ||= matched
+ if $DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens, state unless match
+
+ tokens << [match, kind]
+
+ end # until eos
+
+ if state == :string
+ tokens << [:close, :string]
+ end
+
+ tokens
+
+ end #scan_tokens
+ end #class
+ end #module scanners
+end #module coderay
\ No newline at end of file
Index: lib/coderay/scanners/delphi.rb
===================================================================
--- lib/coderay/scanners/delphi.rb (revision 200)
+++ lib/coderay/scanners/delphi.rb (revision 250)
@@ -29,13 +29,18 @@
'virtual', 'write', 'writeonly'
]
- IDENT_KIND = CaseIgnoringWordList.new(:ident).
+ IDENT_KIND = CaseIgnoringWordList.new(:ident, caching=true).
add(RESERVED_WORDS, :reserved).
add(DIRECTIVES, :directive)
+
+ NAME_FOLLOWS = CaseIgnoringWordList.new(false, caching=true).
+ add(%w(procedure function .))
+ private
def scan_tokens tokens, options
state = :initial
+ last_token = ''
until eos?
@@ -45,19 +50,29 @@
if state == :initial
if scan(/ \s+ /x)
- kind = :space
+ tokens << [matched, :space]
+ next
elsif scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
- kind = :preprocessor
+ tokens << [matched, :preprocessor]
+ next
elsif scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
- kind = :comment
+ tokens << [matched, :comment]
+ next
- elsif scan(/ [-+*\/=<>:;,.@\^|\(\)\[\]]+ /x)
+ elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
kind = :operator
+
+ elsif match = scan(/\./)
+ kind = :operator
+ if last_token == 'end'
+ tokens << [match, kind]
+ next
+ end
elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
- kind = IDENT_KIND[match]
+ kind = NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /x)
tokens << [:open, :char]
@@ -101,6 +116,7 @@
state = :initial
next
elsif scan(/\n/)
+ tokens << [:close, :string]
kind = :error
state = :initial
else
@@ -119,6 +135,7 @@
end
raise_inspect 'Empty token', tokens unless match
+ last_token = match
tokens << [match, kind]
end
Index: lib/coderay/scanners/debug.rb
===================================================================
--- lib/coderay/scanners/debug.rb (revision 0)
+++ lib/coderay/scanners/debug.rb (revision 250)
@@ -0,0 +1,60 @@
+module CodeRay
+module Scanners
+
+ # = Debug Scanner
+ class Debug < Scanner
+
+ include Streamable
+ register_for :debug
+
+ protected
+ def scan_tokens tokens, options
+
+ opened_tokens = []
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ if scan(/\s+/)
+ tokens << [matched, :space]
+ next
+
+ elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
+ kind = self[1].to_sym
+ match = self[2].gsub(/\\(.)/, '\1')
+
+ elsif scan(/ (\w+) < /x)
+ kind = self[1].to_sym
+ opened_tokens << kind
+ match = :open
+
+ elsif scan(/ > /x)
+ kind = opened_tokens.pop
+ match = :close
+
+ else
+ kind = :error
+ getch
+
+ end
+
+ match ||= matched
+ if $DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
Index: lib/coderay/scanners/rhtml.rb
===================================================================
--- lib/coderay/scanners/rhtml.rb (revision 200)
+++ lib/coderay/scanners/rhtml.rb (revision 250)
@@ -51,10 +51,10 @@
start_tag = match[/\A<%[-=]?/]
end_tag = match[/-?%?>?\z/]
tokens << [:open, :inline]
- tokens << [start_tag, :delimiter]
+ tokens << [start_tag, :inline_delimiter]
code = match[start_tag.size .. -1 - end_tag.size]
@ruby_scanner.tokenize code
- tokens << [end_tag, :delimiter] unless end_tag.empty?
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
tokens << [:close, :inline]
else
Index: lib/coderay/scanners/nitro_xhtml.rb
===================================================================
--- lib/coderay/scanners/nitro_xhtml.rb (revision 200)
+++ lib/coderay/scanners/nitro_xhtml.rb (revision 250)
@@ -95,20 +95,20 @@
delimiter = CLOSING_PAREN[start_tag[1,1]]
end_tag = match[-1,1] == delimiter ? delimiter : ''
tokens << [:open, :inline]
- tokens << [start_tag, :delimiter]
+ tokens << [start_tag, :inline_delimiter]
code = match[start_tag.size .. -1 - end_tag.size]
@ruby_scanner.tokenize code
- tokens << [end_tag, :delimiter] unless end_tag.empty?
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
tokens << [:close, :inline]
elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
start_tag = '' ? '?>' : ''
tokens << [:open, :inline]
- tokens << [start_tag, :delimiter]
+ tokens << [start_tag, :inline_delimiter]
code = match[start_tag.size .. -(end_tag.size)-1]
@ruby_scanner.tokenize code
- tokens << [end_tag, :delimiter] unless end_tag.empty?
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
tokens << [:close, :inline]
elsif entity = scan(/#{NITRO_ENTITY}/o)
Index: lib/coderay/scanners/plaintext.rb
===================================================================
--- lib/coderay/scanners/plaintext.rb (revision 200)
+++ lib/coderay/scanners/plaintext.rb (revision 250)
@@ -4,6 +4,8 @@
class Plaintext < Scanner
register_for :plaintext, :plain
+
+ include Streamable
def scan_tokens tokens, options
text = (scan_until(/\z/) || '')
Index: lib/coderay.rb
===================================================================
--- lib/coderay.rb (revision 200)
+++ lib/coderay.rb (revision 250)
@@ -24,8 +24,8 @@
#
# == Usage
#
-# Remember you need RubyGems to use CodeRay. Run Ruby with -rubygems option
-# if required.
+# Remember you need RubyGems to use CodeRay, unless you have it in your load path. Run Ruby with
+# -rubygems option if required.
#
# === Highlight Ruby code in a string as html
#
@@ -44,19 +44,15 @@
#
# You can include this div in your page. The used CSS styles can be printed with
#
-# % ruby -rcoderay -e "print CodeRay::Encoders[:html]::CSS"
+# % coderay_stylesheet
#
# === Highlight without typing too much
-#
+#
# If you are one of the hasty (or lazy, or extremely curious) people, just run this file:
-#
-# % ruby -rubygems coderay.rb
#
-# If the output was to fast for you, try
+# % ruby -rubygems /path/to/coderay/coderay.rb > example.html
#
-# % ruby -rubygems coderay.rb > example.html
-#
-# and look at the file it created.
+# and look at the file it created in your browser.
#
# = CodeRay Module
#
@@ -111,7 +107,7 @@
#
# CodeRay.scan_stream:: Scan in stream mode.
#
-# == All-in-One Encoding
+# == All-in-One Encoding
#
# CodeRay.encode:: Highlight a string with a given input and output format.
#
@@ -121,11 +117,16 @@
# for this Encoder must only be done once.
#
# CodeRay.encoder:: Create an Encoder instance with format and options.
+# CodeRay.scanner:: Create an Scanner instance for lang, with '' as default code.
#
-# There is no CodeRay.scanner method because Scanners are bound to an input string
-# on creation; you can't re-use them with another string.
+# To make use of CodeRay.scanner, use CodeRay::Scanner::code=.
#
-# The scanning methods provide more flexibility; we recommend to use these.
+# The scanning methods provide more flexibility; we recommend to use these.
+#
+# == Reusing Scanners and Encoders
+#
+# If you want to re-use scanners and encoders (because that is faster), see
+# CodeRay::Duo for the most convenient (and recommended) interface.
module CodeRay
# Version: Major.Minor.Teeny[.Revision]
@@ -133,7 +134,7 @@
# Minor: odd for beta, even for stable
# Teeny: development state
# Revision: Subversion Revision number (generated on rake)
- VERSION = '0.7.4'
+ VERSION = '0.7.9'
require 'coderay/tokens'
require 'coderay/scanner'
@@ -170,7 +171,7 @@
def scan_file filename, lang = :auto, options = {}, &block
file = IO.read filename
if lang == :auto
- require 'coderay/helpers/filetype'
+ require 'coderay/helpers/file_type'
lang = FileType.fetch filename, :plaintext, true
end
scan file, lang, options = {}, &block
@@ -314,6 +315,7 @@
# Run a test script.
if $0 == __FILE__
$stderr.print 'Press key to print demo.'; gets
- code = File.read($0)[/module CodeRay.*/m]
+ # Just use this file as an example of Ruby code.
+ code = File.read(__FILE__)[/module CodeRay.*/m]
print CodeRay.scan(code, :ruby).html
end
Property changes on: lib
___________________________________________________________________
Added: svn:externals
+ term http://term-ansicolor.rubyforge.org/svn/trunk/lib/term/