From e1e6a6af97ef710cfc02a1380180b772f1c34672 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:23:54 -0600 Subject: additional Ruby files types Ruby seems to have no shortage of these specially named files. --- lib/coderay/helpers/file_type.rb | 41 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d4fa92..6d928d6 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -116,7 +116,7 @@ module CodeRay 'rhtml' => :erb, 'rjs' => :ruby, 'rpdf' => :ruby, - 'ru' => :ruby, + 'ru' => :ruby, # config.ru 'rxml' => :ruby, 'sass' => :sass, 'sql' => :sql, @@ -132,16 +132,19 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, + 'Guardfile' => :ruby, + 'Vagrantfile' => :ruby, + 'Appraisals' => :ruby } - + end - + end -- cgit v1.2.1 From ae1969b1809365c87a9cb485b671364c502236a0 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 13:00:17 -0600 Subject: revert trimming spaces --- lib/coderay/helpers/file_type.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d928d6..a0f9fd6 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -132,9 +132,9 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, @@ -144,7 +144,7 @@ module CodeRay 'Vagrantfile' => :ruby, 'Appraisals' => :ruby } - + end - + end -- cgit v1.2.1 From e75fecef1998836a4e209db4974b9cb9ec470e95 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 03:50:23 +0200 Subject: don't use -w flag in executable tests --- test/executable/suite.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index ac0ff1d..997405c 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun - "#{RUBY_COMMAND}--ng -w -I%s %s" + "#{RUBY_COMMAND}--ng -I%s %s" else - "#{RUBY_COMMAND} -w -I%s %s" + "#{RUBY_COMMAND} -I%s %s" end % [ROOT_DIR + 'lib', EXECUTABLE] def coderay args, options = {} -- cgit v1.2.1 From fb8c0dbfd065e19419b30213efb8176a1e968945 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:34 +0200 Subject: cleanup benchmark script (finally!) --- .gitignore | 1 - bench/bench.rb | 158 +++++++++------------------------------------- rake_tasks/benchmark.rake | 3 +- 3 files changed, 32 insertions(+), 130 deletions(-) diff --git a/.gitignore b/.gitignore index dd001c8..deed1a2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,4 @@ Gemfile.lock test/executable/source.rb.html test/executable/source.rb.json test/scanners -bench/test.div.html old-stuff diff --git a/bench/bench.rb b/bench/bench.rb index 1958c73..92f9d07 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -1,142 +1,46 @@ -# The most ugly test script I've ever written! -# Shame on me! - -require 'pathname' -require 'profile' if ARGV.include? '-p' - -MYDIR = File.dirname(__FILE__) -LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s -$:.unshift MYDIR, LIBDIR +require 'benchmark' +$: << File.expand_path('../../lib', __FILE__) require 'coderay' -@size = ARGV.fetch(2, 100).to_i * 1000 - -lang = ARGV.fetch(0) do - puts <<-HELP -Usage: - ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream] - - SIZE defaults to 100 kB (= 100,000 bytes). - SIZE = 0 means the whole input. - --p generates a profile (slow! use with SIZE = 1) --o shows the output -stream enabled streaming mode - -Sorry for the strange interface. I will improve it in the next release. - HELP +if ARGV.include? '-h' + puts DATA.read exit end -format = ARGV.fetch(1, 'html').downcase - -$stream = ARGV.include? 'stream' -$optimize = ARGV.include? 'opt' -$style = ARGV.include? 'style' - -require 'benchmark' -require 'fileutils' +lang = ARGV.fetch(0, 'ruby') +data = nil +File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read } +raise 'Example file is empty.' if data.empty? -if format == 'comp' - format = 'page' - begin - require 'syntax' - require 'syntax/convertors/html.rb' - rescue LoadError - puts 'Syntax no found!! (Try % gem install syntax)' - end -end +format = ARGV.fetch(1, 'html').downcase +encoder = CodeRay.encoder(format) -def here fn = nil - return MYDIR unless fn - File.join here, fn +size = ARGV.fetch(2, 1000).to_i * 1000 +unless size.zero? + data += data until data.size >= size + data = data[0, size] end +size = data.size +puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round] -n = ARGV.find { |a| a[/^N/] } -N = if n then n[/\d+/].to_i else 1 end -$filename = ARGV.include?('strange') ? 'strange' : 'example' - -Benchmark.bm(20) do |bm| -N.times do - - data = nil - File.open(here("#$filename." + lang), 'rb') { |f| data = f.read } - raise 'Example file is empty.' if data.empty? - unless @size.zero? - data += data until data.size >= @size - data = data[0, @size] - end - @size = data.size - - options = { - :tab_width => 2, - # :line_numbers => :inline, - :css => $style ? :style : :class, - } - $hl = CodeRay.encoder(format, options) - time = bm.report('CodeRay') do - if $stream || true - $o = $hl.encode(data, lang, options) - else - tokens = CodeRay.scan(data, lang) - tokens.optimize! if $optimize - $o = tokens.encode($hl) - end - end - $file_created = here('test.' + $hl.file_extension) - File.open($file_created, 'wb') do |f| - # f.write $o - end - - time_real = time.real - - puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000] - puts $o if ARGV.include? '-o' - -end +n = ARGV.fetch(3, 5).to_s[/\d+/].to_i +require 'profile' if ARGV.include? '-p' +n.times do |i| + time = Benchmark.realtime { encoder.encode(data, lang) } + puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0] end -puts "Files created: #$file_created" -STDIN.gets if ARGV.include? 'wait' +STDIN.gets if ARGV.include? '-w' __END__ -.ruby .normal {} -.ruby .comment { color: #005; font-style: italic; } -.ruby .keyword { color: #A00; font-weight: bold; } -.ruby .method { color: #077; } -.ruby .class { color: #074; } -.ruby .module { color: #050; } -.ruby .punct { color: #447; font-weight: bold; } -.ruby .symbol { color: #099; } -.ruby .string { color: #944; background: #FFE; } -.ruby .char { color: #F07; } -.ruby .ident { color: #004; } -.ruby .constant { color: #07F; } -.ruby .regex { color: #B66; background: #FEF; } -.ruby .number { color: #F99; } -.ruby .attribute { color: #7BB; } -.ruby .global { color: #7FB; } -.ruby .expr { color: #227; } -.ruby .escape { color: #277; } +Usage: + ruby bench.rb [lang] [format] [size in kB] [number of runs] -.xml .normal {} -.xml .namespace { color: #B66; font-weight: bold; } -.xml .tag { color: #F88; } -.xml .comment { color: #005; font-style: italic; } -.xml .punct { color: #447; font-weight: bold; } -.xml .string { color: #944; } -.xml .number { color: #F99; } -.xml .attribute { color: #BB7; } + - lang defaults to ruby. + - format defaults to html. + - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input. + - number of runs defaults to 5. -.yaml .normal {} -.yaml .document { font-weight: bold; color: #07F; } -.yaml .type { font-weight: bold; color: #05C; } -.yaml .key { color: #F88; } -.yaml .comment { color: #005; font-style: italic; } -.yaml .punct { color: #447; font-weight: bold; } -.yaml .string { color: #944; } -.yaml .number { color: #F99; } -.yaml .time { color: #F99; } -.yaml .date { color: #F99; } -.yaml .ref { color: #944; } -.yaml .anchor { color: #944; } +-h prints this help +-p generates a profile (slow, use with SIZE = 1) +-w waits after the benchmark (for debugging memory usw) diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake index 040951b..2e38b57 100644 --- a/rake_tasks/benchmark.rake +++ b/rake_tasks/benchmark.rake @@ -1,7 +1,6 @@ desc 'Do a benchmark' task :benchmark do - ruby "-v" - ruby "-wIlib bench/bench.rb ruby div 3000 N5" + ruby 'bench/bench.rb ruby html 3000' end task :bench => :benchmark -- cgit v1.2.1 From dc57601571af8024700991b6a80129285a980e9e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:38 +0200 Subject: add Lint encoder; do we still need DebugLint? --- lib/coderay/encoders/debug_lint.rb | 12 ++++---- lib/coderay/encoders/lint.rb | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 lib/coderay/encoders/lint.rb diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 17a0795..2c14186 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -1,6 +1,8 @@ module CodeRay module Encoders + load :lint + # = Debug Lint Encoder # # Debug encoder with additional checks for: @@ -15,12 +17,8 @@ module Encoders register_for :debug_lint - InvalidTokenStream = Class.new StandardError - EmptyToken = Class.new InvalidTokenStream - IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token' if text.empty? super end @@ -30,7 +28,7 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end @@ -41,7 +39,7 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb new file mode 100644 index 0000000..4601e90 --- /dev/null +++ b/lib/coderay/encoders/lint.rb @@ -0,0 +1,57 @@ +module CodeRay +module Encoders + + # = Lint Encoder + # + # Checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::DebugLint + class Lint < Debug + + register_for :lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + end + + def begin_group kind + @opened << kind + end + + def end_group kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + def begin_line kind + @opened << kind + end + + def end_line kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + protected + + def setup options + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? + end + + end + +end +end -- cgit v1.2.1