diff options
| -rw-r--r-- | Changes.textile | 5 | ||||
| -rw-r--r-- | lib/coderay/tokens.rb | 115 |
2 files changed, 77 insertions, 43 deletions
diff --git a/Changes.textile b/Changes.textile index feefb2d..c2ab2de 100644 --- a/Changes.textile +++ b/Changes.textile @@ -16,6 +16,7 @@ Changes related to the new tokens handling include: * *REWRITE* of all Scanners, Encoders, Filters, and Tokens. * *RENAMED* @:open@ and @:close@ actions to @:begin_group@ and @:end_group@. * *RENAMED* @open_token@ and @close_token@ methods to @begin_group@ and @end_group@. +* *NEW* method @:tokens@ allows to add several tokens to the stream. * *CHANGED* The above name changes also affect the JSON, XML, and YAML encoders. CodeRay 1.0 output will be incompatible with earlier versions. * *REMOVED* @TokenStream@ and the @Streamable@ API and all related features like @NotStreamableError@ are now obsolete and have been removed. @@ -27,7 +28,9 @@ h3. General changes h3. @Tokens@ -* *REMOVED* method @stream?@. +* *REMOVED* methods @#stream?@, @#each_text_token@. +* *REMOVED* @#text@ and @#text_size@ methods. Use the @Text@ encoder instead. +* *REMOVED* special implementation of @#each@ taking a filter parameter. Use @TokenKindFilter@ instead. * *NEW* methods @encode_with@, @count@, @begin_group@, @end_group@, @begin_line@, and @end_line@. h3. *RENAMED*: @Tokens::AbbreviationForKind@ diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 50b6834..bf3ff59 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -52,32 +52,6 @@ module CodeRay # The Scanner instance that created the tokens. attr_accessor :scanner - # Iterates over all tokens. - # - # If a filter is given, only tokens of that kind are yielded. - def each kind_filter = nil, &block - unless kind_filter - super(&block) - else - super() do |text, kind| - next unless kind == kind_filter - yield text, kind - end - end - end - - # Iterates over all text tokens. - # Token actions are left out. - # - # Example: - # tokens.each_text_token { |text, kind| text.replace html_escape(text) } - def each_text_token - each do |text, kind| - next unless text.is_a? ::String - yield text, kind - end - end - # Encode the tokens using encoder. # # encoder can be @@ -129,6 +103,7 @@ module CodeRay # for example, consecutive //-comment lines could already be # joined in one comment token by the Scanner. def optimize + raise NotImplementedError, 'Tokens#optimize needs to be rewritten.' last_kind = last_text = nil new = self.class.new for text, kind in self @@ -159,6 +134,7 @@ module CodeRay # # TODO: Test this! def fix + raise NotImplementedError, 'Tokens#fix needs to be rewritten.' tokens = self.class.new # Check token nesting using a stack of kinds. opened = [] @@ -205,6 +181,76 @@ module CodeRay def split_into_lines! replace split_into_lines end + + # Split the tokens into parts of the given +sizes+. + # + # The result will be an Array of Tokens objects. The parts have + # the text size specified by the parameter. In addition, each + # part closes all opened tokens. This is useful to insert tokens + # betweem them. + # + # This method is used by @Scanner#tokenize@ when called with an Array + # of source strings. The Diff encoder uses it for inline highlighting. + def split_into_parts *sizes + parts = [] + opened = [] + content = nil + part = Tokens.new + part_size = 0 + size = sizes.first + i = 0 + for item in self + case content + when nil + content = item + when String + if size && part_size + content.size > size # token must be cut + if part_size < size # some part of the token goes into this part + content = content.dup # content may no be safe to change + part << content.slice!(0, size - part_size) << item + end + # close all open groups and lines... + closing = opened.reverse.flatten.map do |content_or_kind| + case content_or_kind + when :begin_group + :end_group + when :begin_line + :end_line + else + content_or_kind + end + end + parts << part.concat(closing) + part = Tokens.new + # ...and open them again. + part.concat opened.flatten + part_size = 0 + size = sizes[i += 1] + redo unless content.empty? + else + part << content << item + part_size += content.size + end + content = nil + when Symbol + case content + when :begin_group, :begin_line + opened << [content, item] + when :end_group, :end_line + opened.pop + else + raise 'Unknown token action: %p, kind = %p' % [content, item] + end + part << content << item + content = nil + else + raise 'else case reached' + end + end + parts << part + parts << Tokens.new while parts.size < sizes.size + parts + end # Dumps the object into a String that can be saved # in files or databases. @@ -233,22 +279,6 @@ module CodeRay size / 2 end - # The total size of the tokens. - # Should be equal to the input size before - # scanning. - def text_size - size = 0 - each_text_token do |t, k| - size + t.size - end - size - end - - # Return all text tokens joined into a single string. - def text - map { |t, k| t if t.is_a? ::String }.join - end - # Include this module to give an object an #undump # method. # @@ -276,6 +306,7 @@ module CodeRay def end_group kind; push :end_group, kind end def begin_line kind; push :begin_line, kind end def end_line kind; push :end_line, kind end + alias tokens concat end |