1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
module CodeRay
module Encoders
# Makes a statistic for the given tokens.
#
# Alias: +stats+
class Statistic < Encoder
include Streamable
register_for :stats, :statistic
attr_reader :type_stats, :real_token_count # :nodoc:
TypeStats = Struct.new :count, :size # :nodoc:
protected
def setup options
@type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
@real_token_count = 0
end
def generate tokens, options
@tokens = tokens
super
end
def text_token text, kind
@real_token_count += 1 unless kind == :space
@type_stats[kind].count += 1
@type_stats[kind].size += text.size
@type_stats['TOTAL'].size += text.size
@type_stats['TOTAL'].count += 1
end
# TODO Hierarchy handling
def begin_group kind
block_token 'begin_group'
end
def end_group kind
block_token 'end_group'
end
def begin_line kind
block_token 'begin_line'
end
def end_line kind
block_token 'end_line'
end
def block_token action
@type_stats['TOTAL'].count += 1
@type_stats[action].count += 1
end
STATS = <<-STATS # :nodoc:
Code Statistics
Tokens %8d
Non-Whitespace %8d
Bytes Total %8d
Token Types (%d):
type count ratio size (average)
-------------------------------------------------------------
%s
STATS
# space 12007 33.81 % 1.7
TOKEN_TYPES_ROW = <<-TKR # :nodoc:
%-20s %8d %6.2f %% %5.1f
TKR
def finish options
all = @type_stats['TOTAL']
all_count, all_size = all.count, all.size
@type_stats.each do |type, stat|
stat.size /= stat.count.to_f
end
types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
end.join
STATS % [
all_count, @real_token_count, all_size,
@type_stats.delete_if { |k, v| k.is_a? String }.size,
types_stats
]
end
end
end
end
if $0 == __FILE__
$VERBOSE = true
$: << File.join(File.dirname(__FILE__), '..')
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
class StatisticEncoderTest < Test::Unit::TestCase
def test_creation
assert CodeRay::Encoders::Statistic < CodeRay::Encoders::Encoder
stats = nil
assert_nothing_raised do
stats = CodeRay.encoder :statistic
end
assert_kind_of CodeRay::Encoders::Encoder, stats
end
TEST_INPUT = CodeRay::Tokens[
['10', :integer],
['(\\)', :operator],
[:begin_group, :string],
['test', :content],
[:end_group, :string],
[:begin_line, :test],
["\n", :space],
["\n \t", :space],
[" \n", :space],
["[]", :method],
[:end_line, :test],
].flatten
TEST_OUTPUT = <<-'DEBUG'
Code Statistics
Tokens 11
Non-Whitespace 4
Bytes Total 20
Token Types (5):
type count ratio size (average)
-------------------------------------------------------------
TOTAL 11 100.00 % 1.8
space 3 27.27 % 3.0
begin_group 1 9.09 % 0.0
begin_line 1 9.09 % 0.0
content 1 9.09 % 4.0
end_group 1 9.09 % 0.0
end_line 1 9.09 % 0.0
integer 1 9.09 % 2.0
method 1 9.09 % 2.0
operator 1 9.09 % 3.0
DEBUG
def test_filtering_text_tokens
assert_equal TEST_OUTPUT, CodeRay::Encoders::Statistic.new.encode_tokens(TEST_INPUT)
assert_equal TEST_OUTPUT, TEST_INPUT.statistic
end
end
|
