From 3adc7d19fc8529a95e56852c2cb18c544219ad9b Mon Sep 17 00:00:00 2001 From: murphy Date: Thu, 29 Sep 2005 04:14:38 +0000 Subject: After merge with Plugin branch. --- lib/coderay/encoder.rb | 97 ++------- lib/coderay/helpers/plugin.rb | 287 ++++++++++++++++++++++++++ lib/coderay/scanner.rb | 457 ++++++++++++++++-------------------------- 3 files changed, 472 insertions(+), 369 deletions(-) create mode 100644 lib/coderay/helpers/plugin.rb (limited to 'lib/coderay') diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb index c5295be..74e1582 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoder.rb @@ -8,65 +8,8 @@ module CodeRay # mechanism and the [] method that returns the Encoder class # belonging to the given format. module Encoders - - # Raised if Encoders::[] fails because: - # * a file could not be found - # * the requested Encoder is not registered - EncoderNotFound = Class.new Exception - - def Encoders.create_encoders_hash - Hash.new do |h, lang| - path = Encoders.path_to lang - lang = lang.to_sym - begin - require path - rescue LoadError - raise EncoderNotFound, "#{path} not found." - else - # Encoder should have registered by now - unless h[lang] - raise EncoderNotFound, - "No Encoder for #{lang} found in #{path}." - end - end - h[lang] - end - end - - # Loaded Encoders are saved here. - ENCODERS = create_encoders_hash - - class << self - - # Every Encoder class must register itself for one or more - # +formats+ by calling register_for, which calls this method. - # - # See CodeRay::Encoder.register_for. - def register encoder_class, *formats - for format in formats - ENCODERS[format.to_sym] = encoder_class - end - end - - # Returns the Encoder for +lang+. - # - # Example: - # require 'coderay' - # yaml_encoder = CodeRay::Encoders[:yaml] - def [] lang - ENCODERS[lang] - end - - # Alias for +[]+. - alias load [] - - # Returns the path to the encoder for format. - def path_to plugin - File.join 'coderay', 'encoders', "#{plugin}.rb" - end - - end - + extend PluginHost + plugin_path 'coderay/encoders' # = Encoder # @@ -81,24 +24,13 @@ module CodeRay # If you want the highlighted code in a div or a span instead, # use its subclasses Div and Span. class Encoder + extend Plugin + plugin_host Encoders attr_reader :token_stream class << self - # Register this class for the given langs. - # - # Example: - # class MyEncoder < CodeRay::Encoders:Encoder - # register_for :myenc - # ... - # end - # - # See Encoder.register. - def register_for *args - Encoders.register self, *args - end - # Returns if the Encoder can be used in streaming mode. def streamable? is_a? Streamable @@ -131,12 +63,12 @@ module CodeRay # - encode_tokens expects a +tokens+ object instead # - encode_stream is like encode, but uses streaming mode. # - # Each method has an optional +options+ parameter. These are added - # to the options you passed at creation. + # Each method has an optional +options+ parameter. These are + # added to the options you passed at creation. def initialize options = {} @options = self.class::DEFAULT_OPTIONS.merge options - raise "I am only the basic Encoder class. I can't encode anything. :(\n"\ - "Use my subclasses." if self.class == Encoder + raise "I am only the basic Encoder class. I can't encode "\ + "anything. :( Use my subclasses." if self.class == Encoder end # Encode a Tokens object. @@ -147,8 +79,8 @@ module CodeRay finish options end - # Encode the given +code+ after tokenizing it using the Scanner for - # +lang+. + # Encode the given +code+ after tokenizing it using the Scanner + # for +lang+. def encode code, lang, options = {} options = @options.merge options scanner_options = CodeRay.get_scanner_options(options) @@ -160,8 +92,8 @@ module CodeRay # more clear to you. alias highlight encode - # Encode the given +code+ using the Scanner for +lang+ in streaming - # mode. + # Encode the given +code+ using the Scanner for +lang+ in + # streaming mode. def encode_stream code, lang, options = {} raise NotStreamableError, self unless kind_of? Streamable options = @options.merge options @@ -177,7 +109,7 @@ module CodeRay method(:token).to_proc end - protected + protected # Called with merged options before encoding starts. # Sets @out to an empty string. @@ -193,7 +125,8 @@ module CodeRay # Raises a NotImplementedError exception if it is not overwritten # in subclass. def token text, kind - raise NotImplementedError, "#{self.class}#token not implemented." + raise NotImplementedError, + "#{self.class}#token not implemented." end # Called with merged options after encoding starts. diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb new file mode 100644 index 0000000..b0bb49e --- /dev/null +++ b/lib/coderay/helpers/plugin.rb @@ -0,0 +1,287 @@ + +# = PluginHost +# +# $Id$ +# +# A simple subclass plugin system. +# +# Example: +# class Generators < PluginHost +# plugin_path 'app/generators' +# end +# +# class Generator +# extend Plugin +# PLUGIN_HOST = Generators +# end +# +# class FancyGenerator < Generator +# register_for :fancy +# end +# +# Generators[:fancy] #-> FancyGenerator +# # or +# require_plugin 'Generators/fancy' +module PluginHost + + # Raised if Encoders::[] fails because: + # * a file could not be found + # * the requested Encoder is not registered + PluginNotFound = Class.new Exception + + PLUGIN_HOSTS = [] + PLUGIN_HOSTS_BY_ID = {} # dummy hash + + class << self + + def extended mod + PLUGIN_HOSTS << mod + end + + def included mod + warn "#{name} should not be included. Use extend." + end + + # Find the PluginHost for host_id. + def host_by_id host_id + unless PLUGIN_HOSTS_BY_ID.default_proc + ph = Hash.new do |h, _host_id| + for host in PLUGIN_HOSTS + h[host.host_id] = host + end + h.fetch _host_id, nil + end + PLUGIN_HOSTS_BY_ID.replace ph + end + PLUGIN_HOSTS_BY_ID[host_id] + end + + end + + def plugin_host_id host_id + if host_id.is_a? String + raise ArgumentError, + "String or Symbol expected, but #{lang.class} given." + end + + end + + # The path where the plugins can be found. + def plugin_path *args + unless args.empty? + @plugin_path = File.join(*args) + end + @plugin_path + end + + # The host's ID. + # + # If PLUGIN_HOST_ID is not set, it is simply the class name. + def host_id + if self.const_defined? :PLUGIN_HOST_ID + self::PLUGIN_HOST_ID + else + name + end + end + + def create_plugin_hash + @plugin_hash = + Hash.new do |h, plugin_id| + id = validate_id(plugin_id) + path = path_to id + begin + puts 'Loading plugin: ' + path if $DEBUG + require path + rescue LoadError + raise PluginNotFound, "#{path} not found." + else + # Plugin should have registered by now + unless h.has_key? id + raise PluginNotFound, + "No #{self.name} plugin for #{id} found in #{path}." + end + end + h[id] + end + end + + def plugin_hash + @plugin_hash ||= create_plugin_hash + end + + + # Every plugin must register itself for one or more + # +ids+ by calling register_for, which calls this method. + # + # See Plugin#register_for. + def register plugin, *ids + for id in ids + unless id.is_a? Symbol + raise ArgumentError, + "id must be a Symbol, but it was a #{id.class}" + end + plugin_hash[validate_id(id)] = plugin + end + end + + + # Returns an array of all .rb files in the plugin path. + # + # The extension .rb is not included. + def all_plugin_names + Dir[path_to('*')].map do |file| + File.basename file, '.rb' + end + end + + # Loads all plugins using all_plugin_names and load. + def load_all + for plugin in all_plugin_names + load_plugin plugin + end + end + + + # Returns the Plugin for +id+. + # + # Example: + # yaml_plugin = MyPluginHost[:yaml] + def [] id, *args, &blk + plugin_hash.[] validate_id(id), *args, &blk + end + + # Alias for +[]+. + alias load_plugin [] + + # Returns the Plugin for +id+. + # Use it like Hash#fetch. + # + # Example: + # yaml_plugin = MyPluginHost[:yaml, :default] + def fetch id, *args, &blk + plugin_hash.fetch validate_id(id), *args, &blk + end + + # Returns the path to the encoder for format. + def path_to plugin_id + File.join plugin_path, "#{plugin_id}.rb" + end + + # Converts +id+ to a downcase Symbol if it is a String, + # or returns +id+ if it already is a Symbol. + # + # Raises +ArgumentError+ for all other objects, or if the + # given String includes non-alphanumeric characters (\W). + def validate_id id + if id.is_a? Symbol + id + elsif id.is_a? String + if id[/\w+/] == id + id.downcase.to_sym + else + raise ArgumentError, "Invalid id: '#{id}' given." + end + else + raise ArgumentError, + "String or Symbol expected, but #{id.class} given." + end + end + + #end + + +end + + +# = Plugin +# +# Plugins have to include this module. +# +# IMPORTANT: use extend for this module. +# +# Example: see PluginHost. +module Plugin + + def included mod + warn "#{name} should not be included. Use extend." + end + + # Register this class for the given langs. + # Example: + # class MyPlugin < PluginHost::BaseClass + # register_for :my_id + # ... + # end + # + # See PluginHost.register. + def register_for *ids + plugin_host.register self, *ids + end + + # The host for this Plugin class. + def plugin_host host = nil + if host and not host.is_a? PluginHost + raise ArgumentError, + "PluginHost expected, but #{host.class} given." + end + self.const_set :PLUGIN_HOST, host if host + self::PLUGIN_HOST + end + +end + + +# Convenience method for plugin loading. +# The syntax used is: +# +# require_plugin '/' +# +# Returns the loaded plugin. +def require_plugin path + host, plugin_id = path.split '/', 2 + PluginHost.host_by_id(host).load_plugin plugin_id +end + + +if $0 == __FILE__ + $VERBOSE = $DEBUG = true + eval DATA.read, nil, $0, __LINE__+4 +end + +__END__ + +require 'test/unit' + +class TC_PLUGINS < Test::Unit::TestCase + + class Generators + extend PluginHost + plugin_path '.' + end + + class Generator + extend Plugin + plugin_host Generators + end + + class FancyGenerator < Generator + register_for :plugin_host + end + + def test_plugin + assert_nothing_raised do + Generators[:plugin_host] + end + assert_equal FancyGenerator, Generators[:plugin_host] + end + + def test_require + assert_nothing_raised do + require_plugin('TC_PLUGINS::Generators/plugin_host') + end + assert_equal FancyGenerator, + require_plugin('TC_PLUGINS::Generators/plugin_host') + end + +end diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 58a92f7..cf4f3c6 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -1,283 +1,166 @@ module CodeRay - - # This module holds class Scanner and its subclasses. - # For example, the Ruby scanner is named CodeRay::Scanners::Ruby - # can be found in coderay/scanners/ruby. - # - # Scanner also provides methods and constants for the register - # mechanism and the [] method that returns the Scanner class - # belonging to the given lang. - module Scanners - - # Raised if Scanners[] fails because: - # * a file could not be found - # * the requested Scanner is not registered - ScannerNotFound = Class.new(Exception) - - # Loaded Scanners are saved here. - SCANNERS = Hash.new { |h, lang| - raise ScannerNotFound, "No scanner for #{lang} found." - } - - class << self - - # Registers a scanner class by setting SCANNERS[lang]. - # - # Typically used in Scanners, for example in the Ruby scanner: - # - # register_for :ruby - def register scanner_class, *langs - for lang in langs - unless lang.is_a? Symbol - raise ArgumentError, - "lang must be a Symbol, but it was a #{lang.class}" - end - SCANNERS[lang] = scanner_class - end - end - - # Loads the scanner class for +lang+ and returns it. - # - # Example: - # - # Scanners[:xml].new - # - # +lang+ is converted using +normalize+ and must be - # * a String containing only alphanumeric characters (\w+) - # * a Symbol - # - # Strings are converted to lowercase symbols (so +'C'+ and +'c'+ - # load the same scanner, namely the one registered for +:c+.) - # - # If the scanner isn't registered yet, it is searched. - # CodeRay expects that the scanner class is defined in - # - # /coderay/scanners/.rb - # - # (See path_to.) - # - # If the file isn't found, a ScannerNotFound exception is raised - # - # The scanner should register itself using +register+. If the - # scanner is still not found (because has not registered or - # registered under another - # lang), a ScannerNotFound exception is raised. - def [] lang - lang = normalize lang - - SCANNERS.fetch lang do - scanner_file = path_to lang - - begin - require scanner_file - rescue LoadError - raise ScannerNotFound, "File #{scanner_file} not found." - end - - SCANNERS.fetch lang do - raise ScannerNotFound, <<-ERR -No scanner for #{lang} found in #{scanner_file}. -Known scanners: #{SCANNERS} - ERR - end - end - end - - # Alias for +[]+. - alias load [] - - # Calculates the path where a scanner for +lang+ - # is expected to be. This is: - # - # /coderay/scanners/.rb - def path_to lang - File.join 'coderay', 'scanners', "#{lang}.rb" - end - - # Returns an array of all filenames in the scanners/ folder. - # The extension +.rb+ is not included. - def languages - scanners = File.join File.dirname(__FILE__), 'scanners', '*.rb' - Dir[scanners].map do |file| - File.basename file, '.rb' - end - end - - # Loads all scanners that +languages+ finds using +load+. - def load_all - for lang in languages - load lang - end - end - - # Converts +lang+ to a downcase Symbol if it is a String, - # or returns +lang+ if it already is a Symbol. - # - # Raises +ArgumentError+ for all other objects, or if the - # given String includes non-alphanumeric characters (\W). - def normalize lang - if lang.is_a? Symbol - lang - elsif lang.is_a? String - if lang[/\w+/] == lang - lang[/\w+/].downcase.to_sym - else - raise ArgumentError, "Invalid lang: '#{lang}' given." - end - elsif lang.nil? - :plaintext - else - raise ArgumentError, - "String or Symbol expected, but #{lang.class} given." - end - end - - end - - - require 'strscan' - # = Scanner - # - # The base class for all Scanners. - # - # It is a subclass of Ruby's great +StringScanner+, which - # makes it easy to access the scanning methods inside. - # - # It is also +Enumerable+, so you can use it like an Array of Tokens: - # - # require 'coderay' - # - # c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;" - # - # for text, kind in c_scanner - # puts text if kind == :operator - # end - # - # # prints: (*==)++; - # - # OK, this is a very simple example :) - # You can also use +map+, +any?+, +find+ and even +sort_by+, - # if you want. - class Scanner < StringScanner - - # Raised if a Scanner fails while scanning - ScanError = Class.new(Exception) - - require 'coderay/helpers/scanner_helper' - - # The default options for all scanner classes. - # - # Define @default_options for subclasses. - DEFAULT_OPTIONS = { :stream => false } - - class << self - # Register the scanner class for all - # +langs+. - # - # See Scanners.register. - def register_for *langs - Scanners.register self, *langs - end - - # Returns if the Scanner can be used in streaming mode. - def streamable? - is_a? Streamable - end - - end + + require 'coderay/helpers/plugin' + + # = Scanners + # + # $Id$ + # + # This module holds the Scanner class and its subclasses. + # For example, the Ruby scanner is named CodeRay::Scanners::Ruby + # can be found in coderay/scanners/ruby. + # + # Scanner also provides methods and constants for the register + # mechanism and the [] method that returns the Scanner class + # belonging to the given lang. + # + # See PluginHost. + module Scanners + extend PluginHost + plugin_path 'coderay/scanners' + + require 'strscan' + + # = Scanner + # + # The base class for all Scanners. + # + # It is a subclass of Ruby's great +StringScanner+, which + # makes it easy to access the scanning methods inside. + # + # It is also +Enumerable+, so you can use it like an Array of + # Tokens: + # + # require 'coderay' + # + # c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;" + # + # for text, kind in c_scanner + # puts text if kind == :operator + # end + # + # # prints: (*==)++; + # + # OK, this is a very simple example :) + # You can also use +map+, +any?+, +find+ and even +sort_by+, + # if you want. + class Scanner < StringScanner + extend Plugin + plugin_host Scanners + + # Raised if a Scanner fails while scanning + ScanError = Class.new(Exception) + + require 'coderay/helpers/scanner_helper' + + # The default options for all scanner classes. + # + # Define @default_options for subclasses. + DEFAULT_OPTIONS = { :stream => false } + + class << self + + # Returns if the Scanner can be used in streaming mode. + def streamable? + is_a? Streamable + end + + end =begin - ## Excluded for speed reasons; protected seems to make methods slow. - - # Save the StringScanner methods from being called. - # This would not be useful for highlighting. -strscan_public_methods = - StringScanner.instance_methods - StringScanner.ancestors[1].instance_methods -protected(*strscan_public_methods) +## Excluded for speed reasons; protected seems to make methods slow. + +# Save the StringScanner methods from being called. +# This would not be useful for highlighting. + strscan_public_methods = + StringScanner.instance_methods - + StringScanner.ancestors[1].instance_methods + protected(*strscan_public_methods) =end - # Creates a new Scanner. - # - # * +code+ is the input String and is handled by the superclass - # StringScanner. - # * +options+ is a Hash with Symbols as keys. - # It is merged with the default options of the class (you can - # overwrite default options here.) - # * +block+ is the callback for streamed highlighting. - # - # If you set :stream to +true+ in the options, the Scanner uses a - # TokenStream with the +block+ as callback to handle the tokens. - # - # Else, a Tokens object is used. - def initialize code, options = {}, &block - @options = self.class::DEFAULT_OPTIONS.merge options - raise "I am only the basic Scanner class. I can't scan anything. :(\n" + - "Use my subclasses." if self.class == Scanner - - # I love this hack. It seems to silence - # all dos/unix/mac newline problems. - super code.gsub(/\r\n?/, "\n") - - if @options[:stream] - warn "warning in CodeRay::Scanner.new: :stream is set, "\ - "but no block was given" unless block_given? - raise NotStreamableError, self unless kind_of? Streamable - @tokens = TokenStream.new(&block) - else - warn "warning in CodeRay::Scanner.new: Block given, "\ - "but :stream is #{@options[:stream]}" if block_given? - @tokens = Tokens.new - end - end - - # More mnemonic accessor name for the input string. - alias code string - - # Scans the code and returns all tokens in a Tokens object. - def tokenize options = {} - options = @options.merge({}) #options - if @options[:stream] # :stream must have been set already - reset ## what is this for? - scan_tokens @tokens, options - @tokens - else - @cached_tokens ||= scan_tokens @tokens, options - end - end - - # you can also see this as a read-only attribute - alias tokens tokenize - - # Traverses the tokens. - def each &block - raise ArgumentError, - 'Cannot traverse TokenStream.' if @options[:stream] - tokens.each(&block) - end - include Enumerable - - # The current line position of the scanner. - # - # Beware, this is implemented inefficiently. It should be used - # for debugging only. - def line - string[0..pos].count("\n") + 1 - end - - protected - - # This is the central method, and commonly the only one a subclass - # implements. - # - # Subclasses must implement this method; it must return +tokens+ - # and must only use Tokens#<< for storing scanned tokens! - def scan_tokens tokens, options - raise NotImplementedError, "#{self.class}#scan_tokens not implemented." - end - - # Scanner error with additional status information - def raise_inspect msg, tokens, ambit = 30 - raise ScanError, <<-EOE % [ + # Create a new Scanner. + # + # * +code+ is the input String and is handled by the superclass + # StringScanner. + # * +options+ is a Hash with Symbols as keys. + # It is merged with the default options of the class (you can + # overwrite default options here.) + # * +block+ is the callback for streamed highlighting. + # + # If you set :stream to +true+ in the options, the Scanner uses a + # TokenStream with the +block+ as callback to handle the tokens. + # + # Else, a Tokens object is used. + def initialize code, options = {}, &block + @options = self.class::DEFAULT_OPTIONS.merge options + raise "I am only the basic Scanner class. I can't scan "\ + "anything. :( Use my subclasses." if self.class == Scanner + + # I love this hack. It seems to silence + # all dos/unix/mac newline problems. + super code.gsub(/\r\n?/, "\n") + + if @options[:stream] + warn "warning in CodeRay::Scanner.new: :stream is set, "\ + "but no block was given" unless block_given? + raise NotStreamableError, self unless kind_of? Streamable + @tokens = TokenStream.new(&block) + else + warn "warning in CodeRay::Scanner.new: Block given, "\ + "but :stream is #{@options[:stream]}" if block_given? + @tokens = Tokens.new + end + end + + # More mnemonic accessor name for the input string. + alias code string + + # Scans the code and returns all tokens in a Tokens object. + def tokenize options = {} + options = @options.merge({}) #options + if @options[:stream] # :stream must have been set already + reset ## what is this for? + scan_tokens @tokens, options + @tokens + else + @cached_tokens ||= scan_tokens @tokens, options + end + end + + # You can also see tokenize as a read-only attribute + alias tokens tokenize + + # Traverses the tokens. + def each &block + raise ArgumentError, + 'Cannot traverse TokenStream.' if @options[:stream] + tokens.each(&block) + end + include Enumerable + + # The current line position of the scanner. + # + # Beware, this is implemented inefficiently. It should be used + # for debugging only. + def line + string[0..pos].count("\n") + 1 + end + + protected + + # This is the central method, and commonly the only one a + # subclass implements. + # + # Subclasses must implement this method; it must return +tokens+ + # and must only use Tokens#<< for storing scanned tokens! + def scan_tokens tokens, options + raise NotImplementedError, + "#{self.class}#scan_tokens not implemented." + end + + # Scanner error with additional status information + def raise_inspect msg, tokens, ambit = 30 + raise ScanError, <<-EOE % [ ***ERROR in %s: %s @@ -295,20 +178,20 @@ surrounding code: ***ERROR*** - EOE - File.basename(caller[0]), - msg, - tokens.last(10).map { |t| t.inspect }.join("\n"), - line, pos, - matched, bol?, eos?, - string[pos-ambit,ambit], - string[pos,ambit], - ] - end + EOE + File.basename(caller[0]), + msg, + tokens.last(10).map { |t| t.inspect }.join("\n"), + line, pos, + matched, bol?, eos?, + string[pos-ambit,ambit], + string[pos,ambit], + ] + end - end + end - end + end end # vim:sw=2:ts=2:noet:tw=78 -- cgit v1.2.1