1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
module CodeRay
module Scanners
# = Debug Scanner
#
# Parses the output of the Encoders::Debug encoder.
class Debug < Scanner
include Streamable
register_for :debug
file_extension 'raydebug'
title 'CodeRay Token Dump'
protected
def scan_tokens tokens, options
opened_tokens = []
until eos?
kind = nil
match = nil
if scan(/\s+/)
tokens << [matched, :space]
next
elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
kind = self[1].to_sym
match = self[2].gsub(/\\(.)/, '\1')
unless Tokens::AbbreviationForKind.has_key? kind
kind = :error
match = matched
end
elsif scan(/ (\w+) ([<\[]) /x)
kind = self[1].to_sym
opened_tokens << kind
case self[2]
when '<'
match = :open
when '['
match = :begin_line
else
raise
end
elsif !opened_tokens.empty? && scan(/ > /x)
kind = opened_tokens.pop
match = :close
elsif !opened_tokens.empty? && scan(/ \] /x)
kind = opened_tokens.pop
match = :end_line
else
kind = :space
getch
end
match ||= matched
if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
raise_inspect 'Empty token', tokens unless match
tokens << [match, kind]
end
tokens << [:close, opened_tokens.pop] until opened_tokens.empty?
tokens
end
end
end
end
if $0 == __FILE__
$VERBOSE = true
$: << File.join(File.dirname(__FILE__), '..')
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
class DebugScannerTest < Test::Unit::TestCase
def test_creation
assert CodeRay::Scanners::Debug < CodeRay::Scanners::Scanner
debug = nil
assert_nothing_raised do
debug = CodeRay.scanner :debug
end
assert_kind_of CodeRay::Scanners::Scanner, debug
end
TEST_INPUT = <<-'DEBUG'.chomp
integer(10)operator((\\\))string<content(test)>test[
method([])]
DEBUG
TEST_OUTPUT = CodeRay::Tokens[
['10', :integer],
['(\\)', :operator],
[:open, :string],
['test', :content],
[:close, :string],
[:begin_line, :test],
["\n\n \t \n", :space],
["[]", :method],
[:end_line, :test],
]
def test_filtering_text_tokens
assert_equal TEST_OUTPUT, CodeRay::Scanners::Debug.new.tokenize(TEST_INPUT)
assert_equal TEST_OUTPUT, CodeRay.scan(TEST_INPUT, :debug)
end
end
|
