1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
require 'rubygems'
require_gem 'rubylexer'
require 'rubylexer.rb'
module CodeRay module Scanners
class RubyLex < Scanner
register_for :rubylex
class FakeFile < String
def initialize(*)
super
@pos = 0
end
attr_accessor :pos
def read x
pos = @pos
@pos += x
self[pos ... @pos]
end
def getc
pos = @pos
@pos += 1
self[pos]||-1
end
def eof?
@pos == size
end
def each_byte
until eof?
yield getc
end
end
def method_missing meth, *args
raise NoMethodError, '%s%s' % [meth, args]
end
end
private
Translate = {
:ignore => :comment,
:varname => :ident,
:number => :integer,
:ws => :space,
:escnl => :space,
:keyword => :reserved,
:methname => :method,
:renderexactlystring => :regexp,
:string => :string,
}
def scan_tokens tokens, options
require 'tempfile'
Tempfile.open('~coderay_tempfile') do |file|
file.binmode
file.write code
file.rewind
lexer = RubyLexer.new 'code', file
loop do
begin
tok = lexer.get1token
rescue => kaboom
err = <<-EOE
ERROR!!!
#{kaboom.inspect}
#{kaboom.backtrace.join("\n")}
EOE
tokens << [err, :error]
Kernel.raise
end
break if tok.is_a? EoiToken
next if tok.is_a? FileAndLineToken
kind = tok.class.name[/(.*?)Token$/,1].downcase.to_sym
kind = Translate.fetch kind, kind
text = tok.ident
case kind
when :hereplaceholder
text = tok.ender
kind = :string
when :herebody, :outlinedherebody
text = tok.ident.ident
kind = :string
end
text = text.inspect unless text.is_a? String
p token if kind == :error
tokens << [text.dup, kind]
end
end
tokens
end
end
end end
|
