diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/coderay/encoder.rb | 2 | ||||
| -rw-r--r-- | lib/coderay/scanners/java/builtin_types.rb | 2 | ||||
| -rw-r--r-- | lib/coderay/tokens.rb | 91 | ||||
| -rw-r--r-- | lib/coderay/tokens_proxy.rb | 29 |
4 files changed, 26 insertions, 98 deletions
diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb index cc331d1..d2d6c7e 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoder.rb @@ -34,7 +34,7 @@ module CodeRay # downcase class name instead. def const_missing sym if sym == :FILE_EXTENSION - (@plugin_id || name[/\w+$/].downcase).to_s + (defined?(@plugin_id) && @plugin_id || name[/\w+$/].downcase).to_s else super end diff --git a/lib/coderay/scanners/java/builtin_types.rb b/lib/coderay/scanners/java/builtin_types.rb index 8087edd..d1b8b73 100644 --- a/lib/coderay/scanners/java/builtin_types.rb +++ b/lib/coderay/scanners/java/builtin_types.rb @@ -3,6 +3,7 @@ module Scanners module Java::BuiltinTypes # :nodoc: + #:nocov: List = %w[ AbstractAction AbstractBorder AbstractButton AbstractCellEditor AbstractCollection AbstractColorChooserPanel AbstractDocument AbstractExecutorService AbstractInterruptibleChannel @@ -412,6 +413,7 @@ module Scanners XPathFactoryConfigurationException XPathFunction XPathFunctionException XPathFunctionResolver XPathVariableResolver ZipEntry ZipException ZipFile ZipInputStream ZipOutputStream ZoneView ] + #:nocov: end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index ee28a4e..045cf4a 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -83,97 +83,6 @@ module CodeRay super end - # Returns the tokens compressed by joining consecutive - # tokens of the same kind. - # - # This can not be undone, but should yield the same output - # in most Encoders. It basically makes the output smaller. - # - # Combined with dump, it saves space for the cost of time. - # - # If the scanner is written carefully, this is not required - - # for example, consecutive //-comment lines could already be - # joined in one comment token by the Scanner. - def optimize - raise NotImplementedError, 'Tokens#optimize needs to be rewritten.' - # last_kind = last_text = nil - # new = self.class.new - # for text, kind in self - # if text.is_a? String - # if kind == last_kind - # last_text << text - # else - # new << [last_text, last_kind] if last_kind - # last_text = text - # last_kind = kind - # end - # else - # new << [last_text, last_kind] if last_kind - # last_kind = last_text = nil - # new << [text, kind] - # end - # end - # new << [last_text, last_kind] if last_kind - # new - end - - # Compact the object itself; see optimize. - def optimize! - replace optimize - end - - # Ensure that all begin_group tokens have a correspondent end_group. - # - # TODO: Test this! - def fix - raise NotImplementedError, 'Tokens#fix needs to be rewritten.' - # tokens = self.class.new - # # Check token nesting using a stack of kinds. - # opened = [] - # for type, kind in self - # case type - # when :begin_group - # opened.push [:begin_group, kind] - # when :begin_line - # opened.push [:end_line, kind] - # when :end_group, :end_line - # expected = opened.pop - # if [type, kind] != expected - # # Unexpected end; decide what to do based on the kind: - # # - token was never opened: delete the end (just skip it) - # next unless opened.rindex expected - # # - token was opened earlier: also close tokens in between - # tokens << token until (token = opened.pop) == expected - # end - # end - # tokens << [type, kind] - # end - # # Close remaining opened tokens - # tokens << token while token = opened.pop - # tokens - end - - def fix! - replace fix - end - - # TODO: Scanner#split_into_lines - # - # Makes sure that: - # - newlines are single tokens - # (which means all other token are single-line) - # - there are no open tokens at the end the line - # - # This makes it simple for encoders that work line-oriented, - # like HTML with list-style numeration. - def split_into_lines - raise NotImplementedError - end - - def split_into_lines! - replace split_into_lines - end - # Split the tokens into parts of the given +sizes+. # # The result will be an Array of Tokens objects. The parts have diff --git a/lib/coderay/tokens_proxy.rb b/lib/coderay/tokens_proxy.rb index 2a20435..31ff39b 100644 --- a/lib/coderay/tokens_proxy.rb +++ b/lib/coderay/tokens_proxy.rb @@ -1,7 +1,23 @@ module CodeRay - class TokensProxy < Struct.new :input, :lang, :options, :block + # The result of a scan operation is a TokensProxy, but should act like Tokens. + # + # This proxy makes it possible to use the classic CodeRay.scan.encode API + # while still providing the benefits of direct streaming. + class TokensProxy + attr_accessor :input, :lang, :options, :block + + # Create a new TokensProxy with the arguments of CodeRay.scan. + def initialize input, lang, options = {}, block = nil + @input = input + @lang = lang + @options = options + @block = block + end + + # Call CodeRay.encode if +encoder+ is a Symbol; + # otherwise, convert the receiver to tokens and call encoder.encode_tokens. def encode encoder, options = {} if encoder.respond_to? :to_sym CodeRay.encode(input, lang, encoder, options) @@ -10,29 +26,30 @@ module CodeRay end end + # Tries to call encode; + # delegates to tokens otherwise. def method_missing method, *args, &blk - encode method, *args + encode method.to_sym, *args rescue PluginHost::PluginNotFound tokens.send(method, *args, &blk) end + # The (cached) result of the tokenized input; a Tokens instance. def tokens @tokens ||= scanner.tokenize(input) end + # A (cached) scanner instance to use for the scan task. def scanner @scanner ||= CodeRay.scanner(lang, options, &block) end + # Overwrite Struct#each. def each *args, &blk tokens.each(*args, &blk) self end - def count - tokens.count - end - end end |