diff options
Diffstat (limited to 'lib/coderay')
| -rw-r--r-- | lib/coderay/encoder.rb | 2 | ||||
| -rw-r--r-- | lib/coderay/tokens.rb | 22 |
2 files changed, 16 insertions, 8 deletions
diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb index b6a22f0..53febc2 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoder.rb @@ -165,7 +165,7 @@ module CodeRay raise NotStreamableError, self unless kind_of? Streamable
options = @options.merge options
setup options
- scanner_options = options.fetch :scanner_options, {}
+ scanner_options = CodeRay.get_scanner_options options
@token_stream = CodeRay.scan_stream code, lang, scanner_options, &self
finish options
end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 71ad33a..988008e 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -151,9 +151,15 @@ module CodeRay # This can not be undone, but should yield the same output
# in most Encoders. It basically makes the output smaller.
#
- # Combined with dump, it saves database space.
+ # Combined with dump, it saves space for the cost
+ # calculating time.
+ #
+ # If the scanner is written carefully, this is not required -
+ # for example, consecutive //-comment lines can already be
+ # joined in one token by the Scanner.
def optimize
- last_kind, last_text = nil, nil
+ print ' Tokens#optimize: before: %d - ' % size if $DEBUG
+ last_kind = last_text = nil
new = self.class.new
each do |text, kind|
if text.is_a? String
@@ -166,15 +172,17 @@ module CodeRay end
else
new << [last_text, last_kind] if last_kind
- last_kind, last_text = nil, nil
+ last_kind = last_text = nil
new << [text, kind]
end
end
new << [last_text, last_kind] if last_kind
+ print 'after: %d (%d saved = %2.0f%%)' %
+ [new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG
new
end
- # Compact the object itself; see compact.
+ # Compact the object itself; see optimize.
def optimize!
replace optimize
end
@@ -290,9 +298,9 @@ module CodeRay raise NotImplementedError, 'A TokenStream cannot be dumped.'
end
- # A TokenStream cannot be compacted. Use Tokens.
- def compact
- raise NotImplementedError, 'A TokenStream cannot be compacted.'
+ # A TokenStream cannot be optimized. Use Tokens.
+ def optimize
+ raise NotImplementedError, 'A TokenStream cannot be optimized.'
end
end
|