From 65eeaca6c18bea379b966559084e47d6fb45c75c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 12 Jun 2013 16:19:41 +0200 Subject: add python binary string --- lib/coderay/scanners/python.rb | 4 ++-- lib/coderay/styles/alpha.rb | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index a9492ab..09c8b6e 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -157,12 +157,12 @@ module Scanners encoder.text_token match, :operator elsif match = scan(/(u?r?|b)?("""|"|'''|')/i) + modifiers = self[1] string_delimiter = self[2] - string_type = docstring_coming ? :docstring : :string + string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string) docstring_coming = false if docstring_coming encoder.begin_group string_type string_raw = false - modifiers = self[1] unless modifiers.empty? string_raw = !!modifiers.index(?r) encoder.text_token modifiers, :modifier diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index f57e4a1..2ad088a 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -56,7 +56,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .annotation { color:#007 } .attribute-name { color:#b48 } .attribute-value { color:#700 } -.binary { color:#509 } +.binary { color:#549 } +.binary .delimiter { color:#325 } +.binary .char { color:#325 } .char .content { color:#D20 } .char .delimiter { color:#710 } .char { color:#D20 } -- cgit v1.2.1 From 3cc51316b3836c030b7da079f5177a79405700a7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:05:06 +0200 Subject: add taskpaper file extension --- lib/coderay/helpers/file_type.rb | 97 ++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 48 deletions(-) (limited to 'lib') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index a5d83ff..6144e8c 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -77,54 +77,55 @@ module CodeRay end TypeFromExt = { - 'c' => :c, - 'cfc' => :xml, - 'cfm' => :xml, - 'clj' => :clojure, - 'css' => :css, - 'diff' => :diff, - 'dpr' => :delphi, - 'erb' => :erb, - 'gemspec' => :ruby, - 'groovy' => :groovy, - 'gvy' => :groovy, - 'h' => :c, - 'haml' => :haml, - 'htm' => :html, - 'html' => :html, - 'html.erb' => :erb, - 'java' => :java, - 'js' => :java_script, - 'json' => :json, - 'mab' => :ruby, - 'pas' => :delphi, - 'patch' => :diff, - 'phtml' => :php, - 'php' => :php, - 'php3' => :php, - 'php4' => :php, - 'php5' => :php, - 'prawn' => :ruby, - 'py' => :python, - 'py3' => :python, - 'pyw' => :python, - 'rake' => :ruby, - 'raydebug' => :raydebug, - 'rb' => :ruby, - 'rbw' => :ruby, - 'rhtml' => :erb, - 'rjs' => :ruby, - 'rpdf' => :ruby, - 'ru' => :ruby, - 'rxml' => :ruby, - 'sass' => :sass, - 'sql' => :sql, - 'tmproj' => :xml, - 'xaml' => :xml, - 'xhtml' => :html, - 'xml' => :xml, - 'yaml' => :yaml, - 'yml' => :yaml, + 'c' => :c, + 'cfc' => :xml, + 'cfm' => :xml, + 'clj' => :clojure, + 'css' => :css, + 'diff' => :diff, + 'dpr' => :delphi, + 'erb' => :erb, + 'gemspec' => :ruby, + 'groovy' => :groovy, + 'gvy' => :groovy, + 'h' => :c, + 'haml' => :haml, + 'htm' => :html, + 'html' => :html, + 'html.erb' => :erb, + 'java' => :java, + 'js' => :java_script, + 'json' => :json, + 'mab' => :ruby, + 'pas' => :delphi, + 'patch' => :diff, + 'phtml' => :php, + 'php' => :php, + 'php3' => :php, + 'php4' => :php, + 'php5' => :php, + 'prawn' => :ruby, + 'py' => :python, + 'py3' => :python, + 'pyw' => :python, + 'rake' => :ruby, + 'raydebug' => :raydebug, + 'rb' => :ruby, + 'rbw' => :ruby, + 'rhtml' => :erb, + 'rjs' => :ruby, + 'rpdf' => :ruby, + 'ru' => :ruby, + 'rxml' => :ruby, + 'sass' => :sass, + 'sql' => :sql, + 'taskpaper' => :taskpaper, + 'tmproj' => :xml, + 'xaml' => :xml, + 'xhtml' => :html, + 'xml' => :xml, + 'yaml' => :yaml, + 'yml' => :yaml, } for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp -- cgit v1.2.1 From 4a85580f76421bb0afb92501119c4dfcfa6540ef Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:12:14 +0200 Subject: tweak Terminal encoder, cleanup token kinds --- lib/coderay/encoders/terminal.rb | 179 ++++++++++++++++++++++----------------- lib/coderay/styles/alpha.rb | 29 +++---- lib/coderay/token_kinds.rb | 136 +++++++++++++++-------------- 3 files changed, 181 insertions(+), 163 deletions(-) (limited to 'lib') diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb index 9894b91..c7ae014 100644 --- a/lib/coderay/encoders/terminal.rb +++ b/lib/coderay/encoders/terminal.rb @@ -19,105 +19,135 @@ module CodeRay register_for :terminal TOKEN_COLORS = { - :annotation => "\e[35m", - :attribute_name => "\e[33m", + :debug => "\e[1;37;44m", + + :annotation => "\e[34m", + :attribute_name => "\e[35m", :attribute_value => "\e[31m", - :binary => "\e[1;35m", + :binary => { + :self => "\e[31m", + :char => "\e[1;31m", + :delimiter => "\e[1;31m", + }, :char => { - :self => "\e[36m", :delimiter => "\e[1;34m" + :self => "\e[35m", + :delimiter => "\e[1;35m" }, - :class => "\e[1;35m", + :class => "\e[1;35;4m", :class_variable => "\e[36m", :color => "\e[32m", - :comment => "\e[37m", - :complex => "\e[1;34m", - :constant => "\e[1;34m\e[4m", - :decoration => "\e[35m", - :definition => "\e[1;32m", - :directive => "\e[32m\e[4m", - :doc => "\e[46m", - :doctype => "\e[1;30m", - :docstring => "\e[31m\e[4m", - :entity => "\e[33m", - :error => "\e[1;33m\e[41m", + :comment => { + :self => "\e[1;30m", + :char => "\e[37m", + :delimiter => "\e[37m", + }, + :constant => "\e[1;34;4m", + :decorator => "\e[35m", + :definition => "\e[1;33m", + :directive => "\e[33m", + :docstring => "\e[31m", + :doctype => "\e[1;34m", + :done => "\e[1;30;2m", + :entity => "\e[31m", + :error => "\e[1;37;41m", :exception => "\e[1;31m", :float => "\e[1;35m", :function => "\e[1;34m", - :global_variable => "\e[42m", + :global_variable => "\e[1;32m", :hex => "\e[1;36m", - :include => "\e[33m", + :id => "\e[1;34m", + :include => "\e[31m", :integer => "\e[1;34m", - :key => "\e[35m", - :label => "\e[1;15m", + :imaginary => "\e[1;34m", + :important => "\e[1;31m", + :key => { + :self => "\e[35m", + :char => "\e[1;35m", + :delimiter => "\e[1;35m", + }, + :keyword => "\e[32m", + :label => "\e[1;33m", :local_variable => "\e[33m", - :octal => "\e[1;35m", - :operator_name => "\e[1;29m", + :namespace => "\e[1;35m", + :octal => "\e[1;34m", + :predefined => "\e[36m", :predefined_constant => "\e[1;36m", - :predefined_type => "\e[1;30m", - :predefined => "\e[4m\e[1;34m", - :preprocessor => "\e[36m", + :predefined_type => "\e[1;32m", + :preprocessor => "\e[1;36m", :pseudo_class => "\e[1;34m", :regexp => { - :self => "\e[31m", - :content => "\e[31m", - :delimiter => "\e[1;29m", + :self => "\e[35m", + :delimiter => "\e[1;35m", :modifier => "\e[35m", + :char => "\e[1;35m", }, - :reserved => "\e[1;31m", + :reserved => "\e[32m", :shell => { - :self => "\e[42m", - :content => "\e[1;29m", - :delimiter => "\e[37m", + :self => "\e[33m", + :char => "\e[1;33m", + :delimiter => "\e[1;33m", + :escape => "\e[1;33m", }, :string => { - :self => "\e[32m", - :modifier => "\e[1;32m", - :escape => "\e[1;36m", - :delimiter => "\e[1;32m", - :char => "\e[1;36m", + :self => "\e[31m", + :modifier => "\e[1;31m", + :char => "\e[1;35m", + :delimiter => "\e[1;31m", + :escape => "\e[1;31m", + }, + :symbol => { + :self => "\e[33m", + :delimiter => "\e[1;33m", }, - :symbol => "\e[1;32m", - :tag => "\e[1;34m", + :tag => "\e[32m", :type => "\e[1;34m", :value => "\e[36m", - :variable => "\e[1;34m", + :variable => "\e[34m", - :insert => "\e[42m", - :delete => "\e[41m", - :change => "\e[44m", - :head => "\e[45m" + :insert => { + :self => "\e[42m", + :insert => "\e[1;32;42m", + :eyecatcher => "\e[102m", + }, + :delete => { + :self => "\e[41m", + :delete => "\e[1;31;41m", + :eyecatcher => "\e[101m", + }, + :change => { + :self => "\e[44m", + :change => "\e[37;44m", + }, + :head => { + :self => "\e[45m", + :filename => "\e[37;45m" + }, } + TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved] TOKEN_COLORS[:method] = TOKEN_COLORS[:function] - TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex] - TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] = - TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] + TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] protected def setup(options) super @opened = [] - @subcolors = nil + @color_scopes = [TOKEN_COLORS] end public def text_token text, kind - if color = (@subcolors || TOKEN_COLORS)[kind] - if Hash === color - if color[:self] - color = color[:self] - else - @out << text - return - end - end + if color = @color_scopes.last[kind] + color = color[:self] if color.is_a? Hash @out << color - @out << text.gsub("\n", "\e[0m\n" + color) + @out << (text.index("\n") ? text.gsub("\n", "\e[0m\n" + color) : text) @out << "\e[0m" - @out << @subcolors[:self] if @subcolors + if outer_color = @color_scopes.last[:self] + @out << outer_color + end else @out << text end @@ -130,40 +160,33 @@ module CodeRay alias begin_line begin_group def end_group kind - if @opened.empty? - # nothing to close - else - @opened.pop + if @opened.pop + @color_scopes.pop @out << "\e[0m" - @out << open_token(@opened.last) + if outer_color = @color_scopes.last[:self] + @out << outer_color + end end end def end_line kind - if @opened.empty? - # nothing to close - else - @opened.pop - # whole lines to be highlighted, - # eg. added/modified/deleted lines in a diff - @out << (@line_filler ||= "\t" * 100 + "\e[0m") - @out << open_token(@opened.last) - end + @out << (@line_filler ||= "\t" * 100) + end_group kind end private def open_token kind - if color = TOKEN_COLORS[kind] - if Hash === color - @subcolors = color + if color = @color_scopes.last[kind] + if color.is_a? Hash + @color_scopes << color color[:self] else - @subcolors = {} + @color_scopes << @color_scopes.last color end else - @subcolors = nil + @color_scopes << @color_scopes.last '' end end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 2ad088a..9a94091 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -57,26 +57,25 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .attribute-name { color:#b48 } .attribute-value { color:#700 } .binary { color:#549 } -.binary .delimiter { color:#325 } .binary .char { color:#325 } +.binary .delimiter { color:#325 } +.char { color:#D20 } .char .content { color:#D20 } .char .delimiter { color:#710 } -.char { color:#D20 } .class { color:#B06; font-weight:bold } .class-variable { color:#369 } .color { color:#0A0 } .comment { color:#777 } .comment .char { color:#444 } .comment .delimiter { color:#444 } -.complex { color:#A08 } .constant { color:#036; font-weight:bold } .decorator { color:#B0B } .definition { color:#099; font-weight:bold } .delimiter { color:black } .directive { color:#088; font-weight:bold } -.doc { color:#970 } -.doc-string { color:#D42; font-weight:bold } +.docstring { color:#D42; } .doctype { color:#34b } +.done { text-decoration: line-through; color: gray } .entity { color:#800; font-weight:bold } .error { color:#F00; background-color:#FAA } .escape { color:#666 } @@ -87,16 +86,16 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .global-variable { color:#d70 } .hex { color:#02b } .id { color:#33D; font-weight:bold } -.imaginary { color:#f00 } .include { color:#B44; font-weight:bold } .inline { background-color: hsla(0,0%,0%,0.07); color: black } .inline-delimiter { font-weight: bold; color: #666 } .instance-variable { color:#33B } .integer { color:#00D } +.imaginary { color:#f00 } .important { color:#D00 } +.key { color: #606 } .key .char { color: #60f } .key .delimiter { color: #404 } -.key { color: #606 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } .local-variable { color:#963 } @@ -108,30 +107,30 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .predefined-type { color:#0a5; font-weight:bold } .preprocessor { color:#579 } .pseudo-class { color:#00C; font-weight:bold } +.regexp { background-color:hsla(300,100%,50%,0.06); } .regexp .content { color:#808 } .regexp .delimiter { color:#404 } .regexp .modifier { color:#C2C } -.regexp { background-color:hsla(300,100%,50%,0.06); } .reserved { color:#080; font-weight:bold } +.shell { background-color:hsla(120,100%,50%,0.06); } .shell .content { color:#2B2 } .shell .delimiter { color:#161 } -.shell { background-color:hsla(120,100%,50%,0.06); } +.string { background-color:hsla(0,100%,50%,0.05); } .string .char { color: #b0b } .string .content { color: #D20 } .string .delimiter { color: #710 } .string .modifier { color: #E40 } -.string { background-color:hsla(0,100%,50%,0.05); } +.symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.symbol { color:#A60 } .tag { color:#070 } .type { color:#339; font-weight:bold } -.value { color: #088; } -.variable { color:#037 } +.value { color: #088 } +.variable { color:#037 } .insert { background: hsla(120,100%,50%,0.12) } .delete { background: hsla(0,100%,50%,0.12) } -.change { color: #bbf; background: #007; } +.change { color: #bbf; background: #007 } .head { color: #f8f; background: #505 } .head .filename { color: white; } @@ -142,8 +141,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .delete .delete { color: #c00; background:transparent; font-weight:bold } .change .change { color: #88f } .head .head { color: #f4f } - -.done { text-decoration: line-through; color: gray } TOKENS end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index de3a0d0..42ea427 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -10,79 +10,77 @@ module CodeRay TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity TokenKinds.update( # :nodoc: - :annotation => 'annotation', - :attribute_name => 'attribute-name', - :attribute_value => 'attribute-value', - :binary => 'bin', - :char => 'char', - :class => 'class', - :class_variable => 'class-variable', - :color => 'color', - :comment => 'comment', - :complex => 'complex', - :constant => 'constant', - :content => 'content', - :debug => 'debug', - :decorator => 'decorator', - :definition => 'definition', - :delimiter => 'delimiter', - :directive => 'directive', - :doc => 'doc', - :doctype => 'doctype', - :docstring => 'doc-string', - :done => 'done', - :entity => 'entity', - :error => 'error', - :escape => 'escape', - :exception => 'exception', - :filename => 'filename', - :float => 'float', - :function => 'function', - :global_variable => 'global-variable', - :hex => 'hex', - :id => 'id', - :imaginary => 'imaginary', - :important => 'important', - :include => 'include', - :inline => 'inline', - :inline_delimiter => 'inline-delimiter', - :instance_variable => 'instance-variable', - :integer => 'integer', - :key => 'key', - :keyword => 'keyword', - :label => 'label', - :local_variable => 'local-variable', - :modifier => 'modifier', - :namespace => 'namespace', - :octal => 'octal', - :predefined => 'predefined', - :predefined_constant => 'predefined-constant', - :predefined_type => 'predefined-type', - :preprocessor => 'preprocessor', - :pseudo_class => 'pseudo-class', - :regexp => 'regexp', - :reserved => 'reserved', - :shell => 'shell', - :string => 'string', - :symbol => 'symbol', - :tag => 'tag', - :type => 'type', - :value => 'value', - :variable => 'variable', + :debug => 'debug', # highlight for debugging (white on blue background) - :change => 'change', - :delete => 'delete', - :head => 'head', - :insert => 'insert', + :annotation => 'annotation', # Groovy, Java + :attribute_name => 'attribute-name', # HTML, CSS + :attribute_value => 'attribute-value', # HTML + :binary => 'binary', # Python, Ruby + :char => 'char', # most scanners, also inside of strings + :class => 'class', # lots of scanners, for different purposes also in CSS + :class_variable => 'class-variable', # Ruby, YAML + :color => 'color', # CSS + :comment => 'comment', # most scanners + :constant => 'constant', # PHP, Ruby + :content => 'content', # inside of strings, most scanners + :decorator => 'decorator', # Python + :definition => 'definition', # CSS + :delimiter => 'delimiter', # inside strings, comments and other types + :directive => 'directive', # lots of scanners + :doctype => 'doctype', # Goorvy, HTML, Ruby, YAML + :docstring => 'docstring', # Python + :done => 'done', # Taskpaper + :entity => 'entity', # HTML + :error => 'error', # invalid token, most scanners + :escape => 'escape', # Ruby (string inline variables like #$foo, #@bar) + :exception => 'exception', # Java, PHP, Python + :filename => 'filename', # Diff + :float => 'float', # most scanners + :function => 'function', # CSS, JavaScript, PHP + :global_variable => 'global-variable', # Ruby, YAML + :hex => 'hex', # hexadecimal number; lots of scanners + :id => 'id', # CSS + :imaginary => 'imaginary', # Python + :important => 'important', # CSS, Taskpaper + :include => 'include', # C, Groovy, Java, Python, Sass + :inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners + :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why? + :instance_variable => 'instance-variable', # Ruby + :integer => 'integer', # most scanners + :key => 'key', # lots of scanners, used together with :value + :keyword => 'keyword', # reserved word that's actually implemented; most scanners + :label => 'label', # C, PHP + :local_variable => 'local-variable', # local and magic variables; some scanners + :modifier => 'modifier', # used inside on strings; lots of scanners + :namespace => 'namespace', # Clojure, Java, Taskpaper + :octal => 'octal', # lots of scanners + :predefined => 'predefined', # predefined function: lots of scanners + :predefined_constant => 'predefined-constant',# lots of scanners + :predefined_type => 'predefined-type', # C, Java, PHP + :preprocessor => 'preprocessor', # C, Delphi, HTML + :pseudo_class => 'pseudo-class', # CSS + :regexp => 'regexp', # Groovy, JavaScript, Ruby + :reserved => 'reserved', # most scanners + :shell => 'shell', # Ruby + :string => 'string', # most scanners + :symbol => 'symbol', # Clojure, Ruby, YAML + :tag => 'tag', # CSS, HTML + :type => 'type', # CSS, Java, SQL, YAML + :value => 'value', # used together with :key; CSS, JSON, YAML + :variable => 'variable', # Sass, SQL, YAML - :eyecatcher => 'eyecatcher', + :change => 'change', # Diff + :delete => 'delete', # Diff + :head => 'head', # Diff, YAML + :insert => 'insert', # Diff + :eyecatcher => 'eyecatcher', # Diff - :ident => false, - :operator => false, + :ident => false, # almost all scanners + :operator => false, # almost all scanners - :space => false, - :plain => false + :space => false, # almost all scanners + :plain => false # almost all scanners ) - TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:method] = TokenKinds[:function] end -- cgit v1.2.1 From df21150ed089f1e3d1d35badde23b2d905a421cd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:35:39 +0200 Subject: speedup/tweak debug encoder --- lib/coderay/encoders/debug.rb | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index c03d3fb..94d52af 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -25,34 +25,37 @@ module Encoders def text_token text, kind raise 'empty token' if $CODERAY_DEBUG && text.empty? + if kind == :space @out << text else - # TODO: Escape ( - text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/) - @out << kind.to_s << '(' << text << ')' + text = text.gsub('\\', '\\\\\\\\') if text.index('\\') + text = text.gsub(')', '\\\\)') if text.index(')') + @out << "#{kind}(#{text})" end end def begin_group kind - @opened << kind - @out << kind.to_s << '<' + @opened << kind if $CODERAY_DEBUG + + @out << "#{kind}<" end def end_group kind - if @opened.last != kind - puts @out - raise "we are inside #{@opened.inspect}, not #{kind}" - end - @opened.pop + raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind + @out << '>' end def begin_line kind - @out << kind.to_s << '[' + @opened << kind if $CODERAY_DEBUG + + @out << "#{kind}[" end def end_line kind + raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind + @out << ']' end -- cgit v1.2.1 From 88b812274a808f69675445a9265dab1d4e41cc44 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:46:11 +0200 Subject: refactor check functionality to DebugLint encoder --- lib/coderay/encoders/debug.rb | 15 ----------- lib/coderay/encoders/debug_lint.rb | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 lib/coderay/encoders/debug_lint.rb (limited to 'lib') diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 94d52af..61520a1 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -18,14 +18,7 @@ module Encoders FILE_EXTENSION = 'raydebug' - def initialize options = {} - super - @opened = [] - end - def text_token text, kind - raise 'empty token' if $CODERAY_DEBUG && text.empty? - if kind == :space @out << text else @@ -36,26 +29,18 @@ module Encoders end def begin_group kind - @opened << kind if $CODERAY_DEBUG - @out << "#{kind}<" end def end_group kind - raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind - @out << '>' end def begin_line kind - @opened << kind if $CODERAY_DEBUG - @out << "#{kind}[" end def end_line kind - raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind - @out << ']' end diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb new file mode 100644 index 0000000..eeb2a92 --- /dev/null +++ b/lib/coderay/encoders/debug_lint.rb @@ -0,0 +1,55 @@ +module CodeRay +module Encoders + + # = Debug Lint Encoder + # + # Debug encoder with additional checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::Debug + class DebugLint < Debug + + register_for :debug_lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def initialize options = {} + super + @opened = [] + end + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + super + end + + def begin_group kind + @opened << kind + super + end + + def end_group kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + super + end + + def begin_line kind + @opened << kind + super + end + + def end_line kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + super + end + + end + +end +end -- cgit v1.2.1 From e423275749744fb2cf768087a07b4c839eaf4734 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 05:33:14 +0200 Subject: remove dump/undump functionality --- lib/coderay/encoders/debug.rb | 1 - lib/coderay/helpers/gzip.rb | 41 ------------------------------------ lib/coderay/tokens.rb | 48 +------------------------------------------ 3 files changed, 1 insertion(+), 89 deletions(-) delete mode 100644 lib/coderay/helpers/gzip.rb (limited to 'lib') diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 61520a1..f4db330 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -9,7 +9,6 @@ module Encoders # # You cannot fully restore the tokens information from the # output, because consecutive :space tokens are merged. - # Use Tokens#dump for caching purposes. # # See also: Scanners::Debug class Debug < Encoder diff --git a/lib/coderay/helpers/gzip.rb b/lib/coderay/helpers/gzip.rb deleted file mode 100644 index 245014a..0000000 --- a/lib/coderay/helpers/gzip.rb +++ /dev/null @@ -1,41 +0,0 @@ -module CodeRay - - # A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.) - module GZip - - require 'zlib' - - # The default zipping level. 7 zips good and fast. - DEFAULT_GZIP_LEVEL = 7 - - # Unzips the given string +s+. - # - # Example: - # require 'gzip_simple' - # print GZip.gunzip(File.read('adresses.gz')) - def GZip.gunzip s - Zlib::Inflate.inflate s - end - - # Zips the given string +s+. - # - # Example: - # require 'gzip_simple' - # File.open('adresses.gz', 'w') do |file - # file.write GZip.gzip('Mum: 0123 456 789', 9) - # end - # - # If you provide a +level+, you can control how strong - # the string is compressed: - # - 0: no compression, only convert to gzip format - # - 1: compress fast - # - 7: compress more, but still fast (default) - # - 8: compress more, slower - # - 9: compress best, very slow - def GZip.gzip s, level = DEFAULT_GZIP_LEVEL - Zlib::Deflate.new(level).deflate s, Zlib::FINISH - end - - end - -end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 6957d69..9142b76 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -1,8 +1,5 @@ module CodeRay - # GZip library for writing and reading token dumps. - autoload :GZip, coderay_path('helpers', 'gzip') - # = Tokens TODO: Rewrite! # # The Tokens class represents a list of tokens returnd from @@ -45,8 +42,7 @@ module CodeRay # See how small it is? ;) # # Tokens gives you the power to handle pre-scanned code very easily: - # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string - # that you put in your DB. + # You can convert it to a webpage, a YAML file, or a .raydebug representation. # # It also allows you to generate tokens directly (without using a scanner), # to load them from a file, and still use any Encoder that CodeRay provides. @@ -157,53 +153,11 @@ module CodeRay parts end - # Dumps the object into a String that can be saved - # in files or databases. - # - # The dump is created with Marshal.dump; - # In addition, it is gzipped using GZip.gzip. - # - # The returned String object includes Undumping - # so it has an #undump method. See Tokens.load. - # - # You can configure the level of compression, - # but the default value 7 should be what you want - # in most cases as it is a good compromise between - # speed and compression rate. - # - # See GZip module. - def dump gzip_level = 7 - dump = Marshal.dump self - dump = GZip.gzip dump, gzip_level - dump.extend Undumping - end - # Return the actual number of tokens. def count size / 2 end - # Include this module to give an object an #undump - # method. - # - # The string returned by Tokens.dump includes Undumping. - module Undumping - # Calls Tokens.load with itself. - def undump - Tokens.load self - end - end - - # Undump the object using Marshal.load, then - # unzip it using GZip.gunzip. - # - # The result is commonly a Tokens object, but - # this is not guaranteed. - def Tokens.load dump - dump = GZip.gunzip dump - @dump = Marshal.load dump - end - alias text_token push def begin_group kind; push :begin_group, kind end def end_group kind; push :end_group, kind end -- cgit v1.2.1 From cb41b00d5673312f4982e914883f9cea95f2ccae Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:12 +0200 Subject: cleanup TODOs, FIXMEs --- lib/coderay.rb | 1 - lib/coderay/encoders/html/numbering.rb | 2 +- lib/coderay/encoders/statistic.rb | 1 - lib/coderay/token_kinds.rb | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/coderay.rb b/lib/coderay.rb index 88c7cc2..24ae5a2 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -166,7 +166,6 @@ module CodeRay # # See also demo/demo_simple. def scan code, lang, options = {}, &block - # FIXME: return a proxy for direct-stream encoding TokensProxy.new code, lang, options, block end diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb index 332145b..5908bf0 100644 --- a/lib/coderay/encoders/html/numbering.rb +++ b/lib/coderay/encoders/html/numbering.rb @@ -75,7 +75,7 @@ module Encoders line_number = start output.gsub!(/^.*$\n?/) do |line| line_number_text = bolding.call line_number - indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x) + indent = ' ' * (max_width - line_number.to_s.size) line_number += 1 "#{indent}#{line_number_text}#{line}" end diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb index 2315d9e..b2f8b83 100644 --- a/lib/coderay/encoders/statistic.rb +++ b/lib/coderay/encoders/statistic.rb @@ -67,7 +67,6 @@ Token Types (%d): @type_stats['TOTAL'].count += 1 end - # TODO Hierarchy handling def begin_group kind block_token ':begin_group', kind end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 42ea427..9154658 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -44,7 +44,7 @@ module CodeRay :important => 'important', # CSS, Taskpaper :include => 'include', # C, Groovy, Java, Python, Sass :inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners - :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why? + :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why use inline_delimiter? :instance_variable => 'instance-variable', # Ruby :integer => 'integer', # most scanners :key => 'key', # lots of scanners, used together with :value -- cgit v1.2.1 From 462302814dc56c33f26f43516d5ddfc345dcd2a3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:30 +0200 Subject: cleanup Tokens documentation --- lib/coderay/tokens.rb | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 6957d69..ad59b7f 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,25 +3,23 @@ module CodeRay # GZip library for writing and reading token dumps. autoload :GZip, coderay_path('helpers', 'gzip') - # = Tokens TODO: Rewrite! + # The Tokens class represents a list of tokens returned from + # a Scanner. It's actually just an Array with a few helper methods. # - # The Tokens class represents a list of tokens returnd from - # a Scanner. - # - # A token is not a special object, just a two-element Array + # A token itself is not a special object, just a two-element Array # consisting of # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # - # A token looks like this: + # It looks like this: # # ['# It looks like this', :comment] # ['3.1415926', :float] # ['$^', :error] # # Some scanners also yield sub-tokens, represented by special - # token actions, namely begin_group and end_group. + # token actions, for example :begin_group and :end_group. # # The Ruby scanner, for example, splits "a string" into: # @@ -33,23 +31,17 @@ module CodeRay # [:end_group, :string] # ] # - # Tokens is the interface between Scanners and Encoders: - # The input is split and saved into a Tokens object. The Encoder - # then builds the output from this object. - # - # Thus, the syntax below becomes clear: - # - # CodeRay.scan('price = 2.59', :ruby).html - # # the Tokens object is here -------^ + # Tokens can be used to save the output of a Scanners in a simple + # Ruby object that can be send to an Encoder later: # - # See how small it is? ;) + # tokens = CodeRay.scan('price = 2.59', :ruby).tokens + # tokens.encode(:html) + # tokens.html + # CodeRay.encoder(:html).encode_tokens(tokens) # # Tokens gives you the power to handle pre-scanned code very easily: - # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string - # that you put in your DB. - # - # It also allows you to generate tokens directly (without using a scanner), - # to load them from a file, and still use any Encoder that CodeRay provides. + # You can serialize it to a JSON string and store it in a database, pass it + # around to encode it more than once, send it to other algorithms... class Tokens < Array # The Scanner instance that created the tokens. @@ -58,8 +50,7 @@ module CodeRay # Encode the tokens using encoder. # # encoder can be - # * a symbol like :html oder :statistic - # * an Encoder class + # * a plugin name like :html oder 'statistic' # * an Encoder object # # options are passed to the encoder. -- cgit v1.2.1 From dc9528456008216c265cfb6613cfa459c3c9ffad Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:48 +0200 Subject: speedup HTML encoder numbering for Ruby 1.8.7+ --- lib/coderay/encoders/html/numbering.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb index 5908bf0..a1b9c04 100644 --- a/lib/coderay/encoders/html/numbering.rb +++ b/lib/coderay/encoders/html/numbering.rb @@ -26,7 +26,7 @@ module Encoders "#{line}" end else - proc { |line| line.to_s } # :to_s.to_proc in Ruby 1.8.7+ + :to_s.to_proc end bold_every = options[:bold_every] -- cgit v1.2.1 From fc21c91d7a89d98785e494ffd41e9f3adbdfed81 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 19 Jun 2013 11:50:04 +0200 Subject: use different key/value heuristic in JSON scanner --- lib/coderay/scanners/json.rb | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 4e0f462..3754a9b 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -14,15 +14,17 @@ module Scanners ESCAPE = / [bfnrt\\"\/] /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: + KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /x protected + def setup + @state = :initial + end + # See http://json.org/ for a definition of the JSON lexic/grammar. def scan_tokens encoder, options - - state = :initial - stack = [] - key_expected = false + state = options[:state] || @state until eos? @@ -32,18 +34,11 @@ module Scanners if match = scan(/ \s+ /x) encoder.text_token match, :space elsif match = scan(/"/) - state = key_expected ? :key : :string + state = check(/#{KEY}/o) ? :key : :string encoder.begin_group state encoder.text_token match, :delimiter elsif match = scan(/ [:,\[{\]}] /x) encoder.text_token match, :operator - case match - when ':' then key_expected = false - when ',' then key_expected = true if stack.last == :object - when '{' then stack << :object; key_expected = true - when '[' then stack << :array - when '}', ']' then stack.pop # no error recovery, but works for valid JSON - end elsif match = scan(/ true | false | null /x) encoder.text_token match, :value elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x) @@ -82,6 +77,10 @@ module Scanners end end + if options[:keep_state] + @state = state + end + if [:string, :key].include? state encoder.end_group state end -- cgit v1.2.1 From 5d1812b9d0e1074eb9ec7b24d6ce0f4f7f80619a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:42:30 +0200 Subject: tags should be bold --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 9a94091..f4d07f1 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -123,7 +123,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.tag { color:#070 } +.tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } .variable { color:#037 } -- cgit v1.2.1 From 2abfc49bdc9a9f4e86c90aa968c302ca76c20812 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:43:06 +0200 Subject: use JSON to highlight .template files (AWS CF) --- lib/coderay/helpers/file_type.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6144e8c..19f27ac 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -120,6 +120,7 @@ module CodeRay 'sass' => :sass, 'sql' => :sql, 'taskpaper' => :taskpaper, + 'template' => :json, # AWS CloudFormation template 'tmproj' => :xml, 'xaml' => :xml, 'xhtml' => :html, -- cgit v1.2.1