From dc57601571af8024700991b6a80129285a980e9e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:38 +0200 Subject: add Lint encoder; do we still need DebugLint? --- lib/coderay/encoders/debug_lint.rb | 12 ++++---- lib/coderay/encoders/lint.rb | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 lib/coderay/encoders/lint.rb (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 17a0795..2c14186 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -1,6 +1,8 @@ module CodeRay module Encoders + load :lint + # = Debug Lint Encoder # # Debug encoder with additional checks for: @@ -15,12 +17,8 @@ module Encoders register_for :debug_lint - InvalidTokenStream = Class.new StandardError - EmptyToken = Class.new InvalidTokenStream - IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token' if text.empty? super end @@ -30,7 +28,7 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end @@ -41,7 +39,7 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb new file mode 100644 index 0000000..4601e90 --- /dev/null +++ b/lib/coderay/encoders/lint.rb @@ -0,0 +1,57 @@ +module CodeRay +module Encoders + + # = Lint Encoder + # + # Checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::DebugLint + class Lint < Debug + + register_for :lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + end + + def begin_group kind + @opened << kind + end + + def end_group kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + def begin_line kind + @opened << kind + end + + def end_line kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + protected + + def setup options + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? + end + + end + +end +end -- cgit v1.2.1 From 5c23a731ca55729fc65630eca3b37a5b1a71e5b1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 20 Jul 2013 18:12:04 +0200 Subject: mark possibly problematic spots with FIXME --- lib/coderay/encoders/html.rb | 2 ++ lib/coderay/encoders/html/css.rb | 2 ++ 2 files changed, 4 insertions(+) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 20f2409..6dd231a 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -142,6 +142,7 @@ module Encoders HTML_ESCAPE = make_html_escape_hash HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ + # FIXME: cache attack TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } end @@ -284,6 +285,7 @@ module Encoders end def make_span_for_kinds method, hint + # FIXME: cache attack Hash.new do |h, kinds| h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin css_class = css_class_for_kinds(kinds) diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index 164d7f8..de98f0e 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -21,6 +21,7 @@ module Encoders end def get_style_for_css_classes css_classes + # FIXME: cache attack cl = @styles[css_classes.first] return '' unless cl style = '' @@ -52,6 +53,7 @@ module Encoders for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop + # FIXME: cache attack @styles[cl] ||= Hash.new @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end -- cgit v1.2.1 From ea107396fdd72cdbbaf4820d09a87bd879ba7e6c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 16:43:19 +0200 Subject: check token kinds in Lint encoders --- lib/coderay/encoders/debug_lint.rb | 3 ++- lib/coderay/encoders/lint.rb | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 2c14186..a4eba2c 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -18,7 +18,8 @@ module Encoders register_for :debug_lint def text_token text, kind - raise Lint::EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb index 4601e90..88c8bd1 100644 --- a/lib/coderay/encoders/lint.rb +++ b/lib/coderay/encoders/lint.rb @@ -17,10 +17,12 @@ module Encoders InvalidTokenStream = Class.new StandardError EmptyToken = Class.new InvalidTokenStream + UnknownTokenKind = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind end def begin_group kind -- cgit v1.2.1 From 60afd6857c8d0f1c3f9f2d6ca45f01b216d6b4b5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 16:49:40 +0200 Subject: no attack vector found --- lib/coderay/encoders/html.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 6dd231a..ee2d91a 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -142,7 +142,6 @@ module Encoders HTML_ESCAPE = make_html_escape_hash HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ - # FIXME: cache attack TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } end -- cgit v1.2.1 From 5cd749771379b9832ab1b37936bd98fb7cc80a34 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 17:04:09 +0200 Subject: don't dup @span_for_kinds hash keys --- lib/coderay/encoders/html.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ee2d91a..ad5fafc 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -286,7 +286,7 @@ module Encoders def make_span_for_kinds method, hint # FIXME: cache attack Hash.new do |h, kinds| - h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin + h[kinds] = begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint -- cgit v1.2.1 From ee30738b0b0615715321aa4f1ed8c7e4025cb411 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 17:04:23 +0200 Subject: rename local variable --- lib/coderay/encoders/html.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ad5fafc..81a6ffa 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -310,8 +310,8 @@ module Encoders def break_lines text, style reopen = '' - @opened.each_with_index do |k, index| - reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '') + @opened.each_with_index do |kind, index| + reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '') end text.gsub("\n", "#{'' * @opened.size}#{'' if style}\n#{reopen}#{style}") end -- cgit v1.2.1 From 05f5a0e270ce2cde4ff242634033c902c58f13ea Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:45:33 +0200 Subject: no cache attacks possible, static input (CSS) --- lib/coderay/encoders/html/css.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index de98f0e..164d7f8 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -21,7 +21,6 @@ module Encoders end def get_style_for_css_classes css_classes - # FIXME: cache attack cl = @styles[css_classes.first] return '' unless cl style = '' @@ -53,7 +52,6 @@ module Encoders for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop - # FIXME: cache attack @styles[cl] ||= Hash.new @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end -- cgit v1.2.1 From ee992427810a1cf88b53c12ccf7fda91a30ab33e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:58:07 +0200 Subject: limit HTML encoder span_for_kinds cache size --- lib/coderay/encoders/html.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 81a6ffa..ffde5d2 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -284,9 +284,8 @@ module Encoders end def make_span_for_kinds method, hint - # FIXME: cache attack Hash.new do |h, kinds| - h[kinds] = begin + begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint @@ -298,6 +297,9 @@ module Encoders "" end end + end.tap do |span| + h.clear if h.size >= 100 + h[kinds] = span end end end -- cgit v1.2.1 From a31b36683834f39c1581add498cce0b016f20fb5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 30 Aug 2013 16:22:19 +0200 Subject: fix coderay -HTML option --- lib/coderay/encoders/html.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ffde5d2..d2ebb5a 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -197,13 +197,15 @@ module Encoders @last_opened = nil end - @out.extend Output - @out.css = @css - if options[:line_numbers] - Numbering.number! @out, options[:line_numbers], options + if @out.respond_to? :to_str + @out.extend Output + @out.css = @css + if options[:line_numbers] + Numbering.number! @out, options[:line_numbers], options + end + @out.wrap! options[:wrap] + @out.apply_title! options[:title] end - @out.wrap! options[:wrap] - @out.apply_title! options[:title] if defined?(@real_out) && @real_out @real_out << @out -- cgit v1.2.1 From da39961195a297293bfe274e4f60c607ad21eada Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 17 May 2014 21:16:38 +0200 Subject: HTML envoder keeps \t with tab_width: false Fixes #170 --- lib/coderay/encoders/html.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index d2ebb5a..c7c0c2d 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -180,7 +180,7 @@ module Encoders @break_lines = (options[:break_lines] == true) - @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width]) + @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t") @opened = [] @last_opened = nil -- cgit v1.2.1 From 7f1f2287650c3f3da75ffe6d9e79793dfcc7a67d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 15:39:51 +0100 Subject: document new option to keep tabs --- lib/coderay/encoders/html.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/coderay/encoders') diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index c7c0c2d..942b9c8 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -25,7 +25,8 @@ module Encoders # == Options # # === :tab_width - # Convert \t characters to +n+ spaces (a number.) + # Convert \t characters to +n+ spaces (a number or false.) + # false will keep tab characters untouched. # # Default: 8 # -- cgit v1.2.1