From 59b31ae8596f9606217b09d4e3f00dcf5aab8475 Mon Sep 17 00:00:00 2001 From: murphy Date: Wed, 22 Apr 2009 02:40:04 +0000 Subject: Improved Python scanner (issue #41). * fixed numeric literals * better Python 3 support * bugfixes, optimizations * added two more test files --- lib/coderay/scanners/python.rb | 64 ++++++--- lib/coderay/styles/cycnus.rb | 2 + lib/coderay/token_classes.rb | 3 + test/scanners/python/literals.expected.raydebug | 16 +++ test/scanners/python/literals.in.py | 16 +++ test/scanners/python/pleac.expected.raydebug | 28 ++-- test/scanners/python/pygments.expected.raydebug | 172 +++++++++++------------ test/scanners/python/python3.expected.raydebug | 19 +++ test/scanners/python/python3.in.py | 19 +++ test/scanners/python/unistring.expected.raydebug | 2 +- 10 files changed, 221 insertions(+), 120 deletions(-) create mode 100644 test/scanners/python/literals.expected.raydebug create mode 100644 test/scanners/python/literals.in.py create mode 100644 test/scanners/python/python3.expected.raydebug create mode 100644 test/scanners/python/python3.in.py diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 685232b..6e86b88 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -15,10 +15,13 @@ module Scanners 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield', - 'exec', 'print', # gone in Python 3 'nonlocal', # new in Python 3 ] + OLD_KEYWORDS = [ + 'exec', 'print', # gone in Python 3 + ] + PREDEFINED_METHODS_AND_TYPES = %w[ __import__ abs all any apply basestring bin bool buffer bytearray bytes callable chr classmethod cmp coerce compile @@ -53,12 +56,13 @@ module Scanners IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). + add(OLD_KEYWORDS, :old_keyword). add(PREDEFINED_METHODS_AND_TYPES, :predefined). add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant). add(PREDEFINED_EXCEPTIONS, :exception) - ESCAPE = / [rbfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x + ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x OPERATOR = / \.\.\. | # ellipsis @@ -66,10 +70,18 @@ module Scanners [,;:()\[\]{}] | # simple delimiters \/\/=? | \*\*=? | # special math [-+*\/%&|^]=? | # ordinary math and binary logic - [~@] | # whatever + ~ | # binary complement <<=? | >>=? | [<>=]=? | != # comparison and assignment /x + STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter| + h[delimiter] = Regexp.union delimiter + end + + STRING_CONTENT_REGEXP = Hash.new do |h, delimiter| + h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x + end + def scan_tokens tokens, options state = :initial @@ -94,10 +106,10 @@ module Scanners tokens << [match, :comment] next - elsif scan(/#{OPERATOR}/ox) + elsif scan(/#{OPERATOR}/o) kind = :operator - elsif match = scan(/(?i:(u?r?))?("""|"|'''|')/) + elsif match = scan(/(u?r?|b)?("""|"|'''|')/i) tokens << [:open, :string] string_delimiter = self[2] string_raw = false @@ -114,22 +126,35 @@ module Scanners scan(/[[:alpha:]_]\w*/x) kind = IDENT_KIND[match] # TODO: handle class, def, from, import - # TODO: handle print, exec used as functions in Python 3 code + # TODO: keyword arguments kind = :ident if last_token_dot + kind = check(/\(/) ? :ident : :keyword if kind == :old_keyword + + elsif scan(/@[a-zA-Z0-9_.]+[lL]?/) + kind = :decorator - elsif scan(/0[xX][0-9A-Fa-f]+/) + elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/) kind = :hex - elsif scan(/(?:0[0-7]+)(?![89.eEfF])/) + elsif scan(/0[bB][01]+[lL]?/) + kind = :bin + + elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) + kind = :float + if scan(/[jJ]/) + match << matched + kind = :imaginary + end + + elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/) kind = :oct - # TODO: Complex numbers - elsif scan(/(?:\d+)(?![.eEfF])/) + elsif match = scan(/\d+([lL])?/) kind = :integer - - # TODO: Floats - elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) - kind = :float + if self[1] == nil && scan(/[jJ]/) + match << matched + kind = :imaginary + end else getch @@ -138,17 +163,18 @@ module Scanners end when :string - # TODO: cache Regexps - if scan(Regexp.union(string_delimiter)) + if scan(STRING_DELIMITER_REGEXP[string_delimiter]) tokens << [matched, :delimiter] tokens << [:close, :string] state = :initial next elsif string_delimiter.size == 3 && scan(/\n/) kind = :content - elsif scan(/ [^\\\n]+? (?= \\ | $ | #{Regexp.escape(string_delimiter)} ) /x) + elsif scan(STRING_CONTENT_REGEXP[string_delimiter]) kind = :content - elsif !string_raw && scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) + elsif !string_raw && scan(/ \\ #{ESCAPE} /ox) + kind = :char + elsif scan(/ \\ #{UNICODE_ESCAPE} /ox) kind = :char elsif scan(/ \\ . /x) kind = :content diff --git a/lib/coderay/styles/cycnus.rb b/lib/coderay/styles/cycnus.rb index f1069b1..6d87db3 100644 --- a/lib/coderay/styles/cycnus.rb +++ b/lib/coderay/styles/cycnus.rb @@ -58,9 +58,11 @@ ol.CodeRay li { white-space: pre } .ch .dl { color:#039 } .cl { color:#B06; font-weight:bold } +.cm { color:#A08; font-weight:bold } .co { color:#036; font-weight:bold } .cr { color:#0A0 } .cv { color:#369 } +.de { color:#B0B; } .df { color:#099; font-weight:bold } .di { color:#088; font-weight:bold } .dl { color:black } diff --git a/lib/coderay/token_classes.rb b/lib/coderay/token_classes.rb index c71705b..ad7e5c8 100755 --- a/lib/coderay/token_classes.rb +++ b/lib/coderay/token_classes.rb @@ -15,8 +15,10 @@ module CodeRay :class_variable => 'cv', :color => 'cr', :comment => 'c', + :complex => 'cm', :constant => 'co', :content => 'k', + :decorator => 'de', :definition => 'df', :delimiter => 'dl', :directive => 'di', @@ -31,6 +33,7 @@ module CodeRay :function => 'fu', :global_variable => 'gv', :hex => 'hx', + :imaginary => 'cm', :important => 'im', :include => 'ic', :inline => 'il', diff --git a/test/scanners/python/literals.expected.raydebug b/test/scanners/python/literals.expected.raydebug new file mode 100644 index 0000000..ca7d22b --- /dev/null +++ b/test/scanners/python/literals.expected.raydebug @@ -0,0 +1,16 @@ +comment(# from http://docs.python.org/reference/lexical_analysis.html#literals) + +ident(re)operator(.)ident(compile)operator(()string comment(# letter or underscore) + string comment(# letter, digit or underscore) + operator(\)) +integer(7) integer(2147483647) oct(0177) +integer(3L) integer(79228162514264337593543950336L) oct(0377L) hex(0x100000000L) + integer(79228162514264337593543950336) hex(0xdeadbeef) + +float(3.14) float(10.) float(.001) float(1e100) float(3.14e-10) float(0e0) + +imaginary(3.14j) imaginary(10.j) imaginary(10j) imaginary(.001j) imaginary(1e100j) imaginary(3.14e-10j) + +integer(2) operator(+) imaginary(1j) + +float(3.5e+54) operator(-) imaginary(2.5e-13j) \ No newline at end of file diff --git a/test/scanners/python/literals.in.py b/test/scanners/python/literals.in.py new file mode 100644 index 0000000..ccf15ac --- /dev/null +++ b/test/scanners/python/literals.in.py @@ -0,0 +1,16 @@ +# from http://docs.python.org/reference/lexical_analysis.html#literals + +re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) +7 2147483647 0177 +3L 79228162514264337593543950336L 0377L 0x100000000L + 79228162514264337593543950336 0xdeadbeef + +3.14 10. .001 1e100 3.14e-10 0e0 + +3.14j 10.j 10j .001j 1e100j 3.14e-10j + +2 + 1j + +3.5e+54 - 2.5e-13j \ No newline at end of file diff --git a/test/scanners/python/pleac.expected.raydebug b/test/scanners/python/pleac.expected.raydebug index fc88237..d4af49e 100644 --- a/test/scanners/python/pleac.expected.raydebug +++ b/test/scanners/python/pleac.expected.raydebug @@ -278,7 +278,7 @@ keyword(for) ident(ln) keyword(in) ident(fileinput)operator(.)ident(input)operat keyword(for) ident(c) keyword(in) ident(ln)operator(:) ident(sys)operator(.)ident(stdout)operator(.)ident(write)operator(()ident(c)operator(\)) ident(sys)operator(.)ident(stdout)operator(.)ident(flush)operator(()operator(\)) - ident(select)operator(.)ident(select)operator(()operator([)operator(])operator(,)operator([)operator(])operator(,)operator([)operator(])operator(,) float(0)float(.005) operator(*) ident(DELAY)operator(\)) + ident(select)operator(.)ident(select)operator(()operator([)operator(])operator(,)operator([)operator(])operator(,)operator([)operator(])operator(,) float(0.005) operator(*) ident(DELAY)operator(\)) comment(#-----------------------------) comment(# @@PLEAC@@_1.6) @@ -827,7 +827,7 @@ comment(# @@PLEAC@@_2.3) comment(#-----------------------------) ident(rounded) operator(=) predefined(round)operator(()ident(num)operator(\)) comment(# rounds to integer) comment(#-----------------------------) -ident(a) operator(=) float(0)float(.255) +ident(a) operator(=) float(0.255) ident(b) operator(=) string operator(%) ident(a) keyword(print) string operator(%) operator(()ident(a)operator(,) ident(b)operator(\)) keyword(print) string operator(%) operator(()ident(a)operator(,) ident(a)operator(\)) @@ -839,7 +839,7 @@ comment(#-----------------------------) keyword(from) ident(math) keyword(import) ident(floor)operator(,) ident(ceil) keyword(print) string -ident(a) operator(=) operator([)float(3)float(.3)operator(,) float(3)float(.5)operator(,) float(3)float(.7)operator(,) operator(-)float(3)float(.3)operator(]) +ident(a) operator(=) operator([)float(3.3)operator(,) float(3.5)operator(,) float(3.7)operator(,) operator(-)float(3.3)operator(]) keyword(for) ident(n) keyword(in) ident(a)operator(:) keyword(print) string operator(%) operator(()ident(n)operator(,) predefined(int)operator(()ident(n)operator(\))operator(,) ident(floor)operator(()ident(n)operator(\))operator(,) ident(ceil)operator(()ident(n)operator(\))operator(\)) comment(#=> number int floor ceil) @@ -1018,8 +1018,8 @@ comment(#-----------------------------) comment(# @@PLEAC@@_2.15) comment(#-----------------------------) -ident(a) operator(=) integer(3)operator(+)integer(5)ident(j) -ident(b) operator(=) integer(2)operator(-)integer(2)ident(j) +ident(a) operator(=) integer(3)operator(+)imaginary(5j) +ident(b) operator(=) integer(2)operator(-)imaginary(2j) ident(c) operator(=) ident(a) operator(*) ident(b) keyword(print) stringoperator(,) ident(c) comment(#=> c = (16+4j\)) @@ -1028,7 +1028,7 @@ keyword(print) ident(c)operator(.)ident(real)operator(,) ident(c)operator(.)iden comment(#=> 16.0 4.0 (16-4j\)) comment(#-----------------------------) keyword(import) ident(cmath) -keyword(print) ident(cmath)operator(.)ident(sqrt)operator(()integer(3)operator(+)integer(4)ident(j)operator(\)) +keyword(print) ident(cmath)operator(.)ident(sqrt)operator(()integer(3)operator(+)imaginary(4j)operator(\)) comment(#=> (2+1j\)) comment(#-----------------------------) @@ -1426,7 +1426,7 @@ comment(# @@PLEAC@@_3.10) comment(#----------------------------- ) comment(# Short Sleeps) -ident(seconds) operator(=) float(3)float(.1) +ident(seconds) operator(=) float(3.1) ident(time)operator(.)ident(sleep)operator(()ident(seconds)operator(\)) keyword(print) string @@ -2118,7 +2118,7 @@ keyword(for) ident(term) keyword(in) ident(mylist)operator(:) ident(matching)operator(.)ident(append)operator(()ident(term)operator(\)) comment(#-----------------------------) ident(bigs) operator(=) operator([)ident(num) keyword(for) ident(num) keyword(in) ident(nums) keyword(if) ident(num) operator(>) integer(1000000)operator(]) -ident(pigs) operator(=) operator([)ident(user) keyword(for) operator(()ident(user)operator(,) ident(val)operator(\)) keyword(in) ident(users)operator(.)ident(items)operator(()operator(\)) keyword(if) ident(val) operator(>) float(1)ident(e7)operator(]) +ident(pigs) operator(=) operator([)ident(user) keyword(for) operator(()ident(user)operator(,) ident(val)operator(\)) keyword(in) ident(users)operator(.)ident(items)operator(()operator(\)) keyword(if) ident(val) operator(>) float(1e7)operator(]) comment(#-----------------------------) keyword(import) ident(os) ident(matching) operator(=) operator([)ident(line) keyword(for) ident(line) keyword(in) ident(os)operator(.)ident(popen)operator(()stringoperator(\)) @@ -5818,7 +5818,7 @@ keyword(print) ident(hypotenuse)operator(()operator(*)ident(a)operator(\)) comment(#-----------------------------) ident(both) operator(=) ident(men) operator(+) ident(women) comment(#-----------------------------) -ident(nums) operator(=) operator([)float(1)float(.4)operator(,) float(3)float(.5)operator(,) float(6)float(.7)operator(]) +ident(nums) operator(=) operator([)float(1.4)operator(,) float(3.5)operator(,) float(6.7)operator(]) comment(# Provided for demonstration purposes only. Use:) comment(# ints = [int(num\) for num in nums] ) keyword(def) ident(int_all)operator(()ident(nums)operator(\))operator(:) @@ -5828,7 +5828,7 @@ keyword(def) ident(int_all)operator(()ident(nums)operator(\))operator(:) keyword(return) ident(retlist) ident(ints) operator(=) ident(int_all)operator(()ident(nums)operator(\)) comment(# nums unchanged) comment(#-----------------------------) -ident(nums) operator(=) operator([)float(1)float(.4)operator(,) float(3)float(.5)operator(,) float(6)float(.7)operator(]) +ident(nums) operator(=) operator([)float(1.4)operator(,) float(3.5)operator(,) float(6.7)operator(]) keyword(def) ident(trunc_em)operator(()ident(nums)operator(\))operator(:) keyword(for) ident(i)operator(,)ident(elem) keyword(in) predefined(enumerate)operator(()ident(nums)operator(\))operator(:) @@ -6669,7 +6669,7 @@ ident(rp) operator(=) ident(employees)operator(.)ident(get)operator(()integer(13 keyword(if) operator(()ident(rp)operator(\))operator(:) keyword(print) string operator(%) ident(rp)operator([)stringoperator(]) -ident(byname)operator([)stringoperator(])operator([)stringoperator(]) operator(*=) float(1)float(.035) +ident(byname)operator([)stringoperator(])operator([)stringoperator(]) operator(*=) float(1.035) ident(peons) operator(=) operator([)ident(r) keyword(for) ident(r) keyword(in) ident(employees)operator(.)ident(values)operator(()operator(\)) keyword(if) ident(r)operator([)stringoperator(]) operator(==) stringoperator(]) ident(tsevens) operator(=) operator([)ident(r) keyword(for) ident(r) keyword(in) ident(employees)operator(.)ident(values)operator(()operator(\)) keyword(if) ident(r)operator([)stringoperator(]) operator(==) integer(27)operator(]) @@ -8270,7 +8270,7 @@ keyword(class) ident(FixNum)operator(:) keyword(def) ident(__div__)operator(()pre_constant(self)operator(,) ident(other)operator(\))operator(:) comment(# Force to use floating point, since 2/3 in Python is 0) comment(# Don't use float(\) since that will convert strings) - keyword(return) ident(FixNum)operator(()operator(()pre_constant(self)operator(.)ident(value)operator(+)float(0)float(.0)operator(\)) operator(/) ident(other)operator(.)ident(value)operator(,) + keyword(return) ident(FixNum)operator(()operator(()pre_constant(self)operator(.)ident(value)operator(+)float(0.0)operator(\)) operator(/) ident(other)operator(.)ident(value)operator(,) predefined(max)operator(()pre_constant(self)operator(.)ident(places)operator(,) ident(other)operator(.)ident(places)operator(\))operator(\)) keyword(def) ident(__str__)operator(()pre_constant(self)operator(\))operator(:) @@ -8916,7 +8916,7 @@ keyword(except) ident(KeyboardError)operator(:) comment(#----------------------------------------) comment(# @@PLEAC@@_15.7) -keyword(print) stringoperator(;) +keyword(print) stringoperator(;) comment(#----------------------------------------) comment(# @@INCOMPLETE@@) @@ -9453,7 +9453,7 @@ keyword(import) ident(sys) keyword(import) ident(signal) keyword(def) ident(ding)operator(()ident(signum)operator(,) ident(frame)operator(\))operator(:) - keyword(print) string + keyword(print) string keyword(return) ident(signal)operator(.)ident(signal)operator(()ident(signal)operator(.)ident(SIGINT)operator(,) ident(ding)operator(\)) diff --git a/test/scanners/python/pygments.expected.raydebug b/test/scanners/python/pygments.expected.raydebug index 9a1a9d2..7084bd7 100644 --- a/test/scanners/python/pygments.expected.raydebug +++ b/test/scanners/python/pygments.expected.raydebug @@ -3742,7 +3742,7 @@ keyword(class) ident(LatexFormatter)operator(()ident(Formatter)operator(\))opera keyword(def) ident(rgbcolor)operator(()ident(col)operator(\))operator(:) keyword(if) ident(col)operator(:) - keyword(return) stringoperator(.)ident(join)operator(()operator([)string operator(%)operator(()predefined(int)operator(()ident(col)operator([)ident(i)operator(]) operator(+) ident(col)operator([)ident(i) operator(+) integer(1)operator(])operator(,) integer(16)operator(\)) operator(/) integer(25)float(5)float(.0)operator(\)) + keyword(return) stringoperator(.)ident(join)operator(()operator([)string operator(%)operator(()predefined(int)operator(()ident(col)operator([)ident(i)operator(]) operator(+) ident(col)operator([)ident(i) operator(+) integer(1)operator(])operator(,) integer(16)operator(\)) operator(/) float(255.0)operator(\)) keyword(for) ident(i) keyword(in) operator(()integer(0)operator(,) integer(2)operator(,) integer(4)operator(\))operator(])operator(\)) keyword(else)operator(:) keyword(return) string @@ -4604,7 +4604,7 @@ ident(__all__) operator(=) operator([)stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(]) -ident(_default_analyse) operator(=) predefined(staticmethod)operator(()keyword(lambda) ident(x)operator(:) float(0)float(.0)operator(\)) +ident(_default_analyse) operator(=) predefined(staticmethod)operator(()keyword(lambda) ident(x)operator(:) float(0.0)operator(\)) keyword(class) ident(LexerMeta)operator(()predefined(type)operator(\))operator(:) @@ -5417,7 +5417,7 @@ keyword(def) ident(guess_lexer_for_filename)operator(()ident(_fn)operator(,) ide ident(result) operator(=) operator([)operator(]) keyword(for) ident(lexer) keyword(in) ident(matching_lexers)operator(:) ident(rv) operator(=) ident(lexer)operator(.)ident(analyse_text)operator(()ident(_text)operator(\)) - keyword(if) ident(rv) operator(==) float(1)float(.0)operator(:) + keyword(if) ident(rv) operator(==) float(1.0)operator(:) keyword(return) ident(lexer)operator(()operator(**)ident(options)operator(\)) ident(result)operator(.)ident(append)operator(()operator(()ident(rv)operator(,) ident(lexer)operator(\))operator(\)) ident(result)operator(.)ident(sort)operator(()operator(\)) @@ -5430,10 +5430,10 @@ keyword(def) ident(guess_lexer)operator(()ident(_text)operator(,) operator(**)id string - ident(best_lexer) operator(=) operator([)float(0)float(.0)operator(,) pre_constant(None)operator(]) + ident(best_lexer) operator(=) operator([)float(0.0)operator(,) pre_constant(None)operator(]) keyword(for) ident(lexer) keyword(in) ident(_iter_lexerclasses)operator(()operator(\))operator(:) ident(rv) operator(=) ident(lexer)operator(.)ident(analyse_text)operator(()ident(_text)operator(\)) - keyword(if) ident(rv) operator(==) float(1)float(.0)operator(:) + keyword(if) ident(rv) operator(==) float(1.0)operator(:) keyword(return) ident(lexer)operator(()operator(**)ident(options)operator(\)) keyword(if) ident(rv) operator(>) ident(best_lexer)operator([)integer(0)operator(])operator(:) ident(best_lexer)operator([)operator(:)operator(]) operator(=) operator(()ident(rv)operator(,) ident(lexer)operator(\)) @@ -10535,8 +10535,8 @@ keyword(class) ident(PerlLexer)operator(()ident(RegexLexer)operator(\))operator( keyword(if) ident(shebang_matches)operator(()ident(text)operator(,) stringoperator(\))operator(:) keyword(return) pre_constant(True) keyword(if) string keyword(in) ident(text)operator(:) - keyword(return) float(0)float(.9) - keyword(return) float(0)float(.1) comment(# who knows, might still be perl!) + keyword(return) float(0.9) + keyword(return) float(0.1) comment(# who knows, might still be perl!) keyword(class) ident(LuaLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -13272,10 +13272,10 @@ keyword(class) ident(CSharpAspxLexer)operator(()ident(DelegatingLexer)operator(\ keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.2) + keyword(return) float(0.2) keyword(elif) ident(re)operator(.)ident(search)operator(()string]+language=[")content(\\')content(]C#)delimiter(')>operator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.15) - keyword(return) float(0)float(.001) comment(# TODO really only for when filename matched...) + keyword(return) float(0.15) + keyword(return) float(0.001) comment(# TODO really only for when filename matched...) keyword(class) ident(VbNetAspxLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.2) + keyword(return) float(0.2) keyword(elif) ident(re)operator(.)ident(search)operator(()string]+language=[")content(\\')content(]vb)delimiter(')>operator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.15) + keyword(return) float(0.15) comment(# -*- coding: utf-8 -*-) stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(M)operator(\))operator(:) comment(# comment) - keyword(return) float(0)float(.9) + keyword(return) float(0.9) keyword(elif) ident(re)operator(.)ident(match)operator(()stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(M)operator(\))operator(:) comment(# system cmd) - keyword(return) float(0)float(.9) - keyword(return) float(0)float(.1) + keyword(return) float(0.9) + keyword(return) float(0.1) ident(line_re) operator(=) ident(re)operator(.)ident(compile)operator(()stringoperator(\)) @@ -16697,7 +16697,7 @@ keyword(class) ident(RagelEmbeddedLexer)operator(()ident(RegexLexer)operator(\)) operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) string keyword(in) ident(text) keyword(or) float(0)float(.1) + keyword(return) string keyword(in) ident(text) keyword(or) float(0.1) keyword(class) ident(RagelRubyLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) string keyword(in) ident(text) keyword(and) string)delimiter(')> keyword(in) ident(text)operator(:) - keyword(return) float(0)float(.4) + keyword(return) float(0.4) keyword(class) ident(SmartyLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -17443,15 +17443,15 @@ keyword(class) ident(SmartyLexer)operator(()ident(RegexLexer)operator(\))operato operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.15) + ident(rv) operator(+=) float(0.15) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.15) + ident(rv) operator(+=) float(0.15) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.15) + ident(rv) operator(+=) float(0.15) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.01) + ident(rv) operator(+=) float(0.01) keyword(return) ident(rv) @@ -17527,13 +17527,13 @@ keyword(class) ident(DjangoLexer)operator(()ident(RegexLexer)operator(\))operato operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(return) ident(rv) @@ -18023,12 +18023,12 @@ keyword(class) ident(HtmlGenshiLexer)operator(()ident(DelegatingLexer)operator(\ operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) + ident(rv) operator(+=) float(0.2) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) - keyword(return) ident(rv) operator(+) ident(HtmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(+=) float(0.2) + keyword(return) ident(rv) operator(+) ident(HtmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(class) ident(GenshiLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18048,12 +18048,12 @@ keyword(class) ident(GenshiLexer)operator(()ident(DelegatingLexer)operator(\))op operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) + ident(rv) operator(+=) float(0.2) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) - keyword(return) ident(rv) operator(+) ident(XmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(+=) float(0.2) + keyword(return) ident(rv) operator(+) ident(XmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(class) ident(JavascriptGenshiLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18075,7 +18075,7 @@ keyword(class) ident(JavascriptGenshiLexer)operator(()ident(DelegatingLexer)oper operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(CssGenshiLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18093,7 +18093,7 @@ keyword(class) ident(CssGenshiLexer)operator(()ident(DelegatingLexer)operator(\) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(RhtmlLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18114,10 +18114,10 @@ keyword(class) ident(RhtmlLexer)operator(()ident(DelegatingLexer)operator(\))ope predefined(super)operator(()ident(RhtmlLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(ErbLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) comment(# one more than the XmlErbLexer returns) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18136,9 +18136,9 @@ keyword(class) ident(XmlErbLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(XmlErbLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(ErbLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18156,7 +18156,7 @@ keyword(class) ident(CssErbLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(CssErbLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(ErbLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptErbLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18177,7 +18177,7 @@ keyword(class) ident(JavascriptErbLexer)operator(()ident(DelegatingLexer)operato operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(HtmlPhpLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18200,9 +18200,9 @@ keyword(class) ident(HtmlPhpLexer)operator(()ident(DelegatingLexer)operator(\))o predefined(super)operator(()ident(HtmlPhpLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(PhpLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18220,9 +18220,9 @@ keyword(class) ident(XmlPhpLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(XmlPhpLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(PhpLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18240,7 +18240,7 @@ keyword(class) ident(CssPhpLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(CssPhpLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(PhpLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptPhpLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18281,9 +18281,9 @@ keyword(class) ident(HtmlSmartyLexer)operator(()ident(DelegatingLexer)operator(\ predefined(super)operator(()ident(HtmlSmartyLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(SmartyLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18302,9 +18302,9 @@ keyword(class) ident(XmlSmartyLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(XmlSmartyLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(SmartyLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18323,7 +18323,7 @@ keyword(class) ident(CssSmartyLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(CssSmartyLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(SmartyLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptSmartyLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18344,7 +18344,7 @@ keyword(class) ident(JavascriptSmartyLexer)operator(()ident(DelegatingLexer)oper operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(HtmlDjangoLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18364,9 +18364,9 @@ keyword(class) ident(HtmlDjangoLexer)operator(()ident(DelegatingLexer)operator(\ predefined(super)operator(()ident(HtmlDjangoLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(DjangoLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18385,9 +18385,9 @@ keyword(class) ident(XmlDjangoLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(XmlDjangoLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(DjangoLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18406,7 +18406,7 @@ keyword(class) ident(CssDjangoLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(CssDjangoLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(DjangoLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptDjangoLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18431,7 +18431,7 @@ keyword(class) ident(JavascriptDjangoLexer)operator(()ident(DelegatingLexer)oper operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JspRootLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -18474,11 +18474,11 @@ keyword(class) ident(JspLexer)operator(()ident(DelegatingLexer)operator(\))opera predefined(super)operator(()ident(JspLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(JspRootLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(JavaLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(JavaLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(if) string keyword(in) ident(text) keyword(and) string)delimiter(')> keyword(in) ident(text)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(return) ident(rv) @@ -18790,7 +18790,7 @@ keyword(class) ident(DiffLexer)operator(()ident(RegexLexer)operator(\))operator( keyword(if) ident(text)operator([)operator(:)integer(5)operator(]) operator(==) stringoperator(:) keyword(return) pre_constant(True) keyword(if) ident(text)operator([)operator(:)integer(4)operator(]) operator(==) stringoperator(:) - keyword(return) float(0)float(.9) + keyword(return) float(0.9) ident(DPATCH_KEYWORDS) operator(=) operator([)stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) @@ -19042,7 +19042,7 @@ keyword(class) ident(GroffLexer)operator(()ident(RegexLexer)operator(\))operator keyword(if) ident(text)operator([)operator(:)integer(4)operator(]) operator(==) stringoperator(:) keyword(return) pre_constant(True) keyword(if) ident(text)operator([)integer(1)operator(:)integer(3)operator(])operator(.)ident(isalnum)operator(()operator(\)) keyword(and) ident(text)operator([)integer(3)operator(])operator(.)ident(isspace)operator(()operator(\))operator(:) - keyword(return) float(0)float(.9) + keyword(return) float(0.9) keyword(class) ident(ApacheConfLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -19270,14 +19270,14 @@ keyword(class) ident(RstLexer)operator(()ident(RegexLexer)operator(\))operator(: keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(text)operator([)operator(:)integer(2)operator(]) operator(==) string keyword(and) ident(text)operator([)integer(2)operator(:)integer(3)operator(]) operator(!=) stringoperator(:) - keyword(return) float(0)float(.3) + keyword(return) float(0.3) ident(p1) operator(=) ident(text)operator(.)ident(find)operator(()stringoperator(\)) ident(p2) operator(=) ident(text)operator(.)ident(find)operator(()stringoperator(,) ident(p1) operator(+) integer(1)operator(\)) keyword(if) operator(()ident(p2) operator(>) operator(-)integer(1) keyword(and) comment(# has two lines) ident(p1) operator(*) integer(2) operator(+) integer(1) operator(==) ident(p2) keyword(and) comment(# they are the same length) ident(text)operator([)ident(p1)operator(+)integer(1)operator(]) keyword(in) string keyword(and) comment(# the next line both starts and ends with) ident(text)operator([)ident(p1)operator(+)integer(1)operator(]) operator(==) ident(text)operator([)ident(p2)operator(-)integer(1)operator(])operator(\))operator(:) comment(# ...a sufficiently high header) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) keyword(class) ident(VimLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20215,7 +20215,7 @@ keyword(class) ident(ActionScriptLexer)operator(()ident(RegexLexer)operator(\))o operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.05) + keyword(return) float(0.05) keyword(class) ident(ActionScript3Lexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20293,8 +20293,8 @@ keyword(class) ident(ActionScript3Lexer)operator(()ident(RegexLexer)operator(\)) operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(if) ident(re)operator(.)ident(match)operator(()stringoperator(,) ident(text)operator(\))operator(:) keyword(return) float(0)float(.3) - keyword(return) float(0)float(.1) + keyword(if) ident(re)operator(.)ident(match)operator(()stringoperator(,) ident(text)operator(\))operator(:) keyword(return) float(0.3) + keyword(return) float(0.1) keyword(class) ident(CssLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20487,7 +20487,7 @@ keyword(class) ident(HtmlLexer)operator(()ident(RegexLexer)operator(\))operator( keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) keyword(class) ident(PhpLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20631,11 +20631,11 @@ keyword(class) ident(PhpLexer)operator(()ident(RegexLexer)operator(\))operator(: keyword(yield) ident(index)operator(,) ident(token)operator(,) ident(value) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.3) + ident(rv) operator(+=) float(0.3) keyword(if) string)delimiter(')> keyword(in) ident(text)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(return) ident(rv) @@ -20684,7 +20684,7 @@ keyword(class) ident(XmlLexer)operator(()ident(RegexLexer)operator(\))operator(: keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) keyword(class) ident(XsltLexer)operator(()ident(XmlLexer)operator(\))operator(:) @@ -20720,7 +20720,7 @@ keyword(class) ident(XsltLexer)operator(()ident(XmlLexer)operator(\))operator(:) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\)) keyword(and) string keyword(in) ident(text)operator(:) - keyword(return) float(0)float(.8) + keyword(return) float(0.8) @@ -22616,8 +22616,8 @@ keyword(def) ident(make_analysator)operator(()ident(f)operator(\))operator(:) keyword(def) ident(text_analyse)operator(()ident(text)operator(\))operator(:) ident(rv) operator(=) ident(f)operator(()ident(text)operator(\)) keyword(if) keyword(not) ident(rv)operator(:) - keyword(return) float(0)float(.0) - keyword(return) predefined(min)operator(()float(1)float(.0)operator(,) predefined(max)operator(()float(0)float(.0)operator(,) predefined(float)operator(()ident(rv)operator(\))operator(\))operator(\)) + keyword(return) float(0.0) + keyword(return) predefined(min)operator(()float(1.0)operator(,) predefined(max)operator(()float(0.0)operator(,) predefined(float)operator(()ident(rv)operator(\))operator(\))operator(\)) ident(text_analyse)operator(.)ident(__doc__) operator(=) ident(f)operator(.)ident(__doc__) keyword(return) predefined(staticmethod)operator(()ident(text_analyse)operator(\)) @@ -22770,7 +22770,7 @@ ident(misspellings) operator(=) operator([)stringoperator(]) comment(# ALLOW-MISSPELLING) -operator(@)ident(checker)operator(()stringoperator(\)) +decorator(@checker)operator(()stringoperator(\)) keyword(def) ident(check_syntax)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) keyword(try)operator(:) predefined(compile)operator(()stringoperator(.)ident(join)operator(()ident(lines)operator(\))operator(,) ident(fn)operator(,) stringoperator(\)) @@ -22778,7 +22778,7 @@ keyword(def) ident(check_syntax)operator(()ident(fn)operator(,) ident(lines)oper keyword(yield) integer(0)operator(,) string operator(%) ident(err) -operator(@)ident(checker)operator(()stringoperator(\)) +decorator(@checker)operator(()stringoperator(\)) keyword(def) ident(check_style_and_encoding)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) ident(encoding) operator(=) string keyword(for) ident(lno)operator(,) ident(line) keyword(in) predefined(enumerate)operator(()ident(lines)operator(\))operator(:) @@ -22802,7 +22802,7 @@ keyword(def) ident(check_style_and_encoding)operator(()ident(fn)operator(,) iden ident(encoding) operator(=) string -operator(@)ident(checker)operator(()stringoperator(,) ident(only_pkg)operator(=)pre_constant(True)operator(\)) +decorator(@checker)operator(()stringoperator(,) ident(only_pkg)operator(=)pre_constant(True)operator(\)) keyword(def) ident(check_fileheader)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) comment(# line number correction) ident(c) operator(=) integer(1) @@ -22866,7 +22866,7 @@ keyword(def) ident(check_fileheader)operator(()ident(fn)operator(,) ident(lines) keyword(yield) integer(0)operator(,) string -operator(@)ident(checker)operator(()stringoperator(,) stringoperator(,) stringoperator(\)) +decorator(@checker)operator(()stringoperator(,) stringoperator(,) stringoperator(\)) keyword(def) ident(check_whitespace_and_spelling)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) keyword(for) ident(lno)operator(,) ident(line) keyword(in) predefined(enumerate)operator(()ident(lines)operator(\))operator(:) keyword(if) string keyword(in) ident(line)operator(:) @@ -22881,7 +22881,7 @@ keyword(def) ident(check_whitespace_and_spelling)operator(()ident(fn)operator(,) ident(bad_tags) operator(=) operator(()string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')> string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')>operator(,) stringoperator(\)) -operator(@)ident(checker)operator(()stringoperator(\)) +decorator(@checker)operator(()stringoperator(\)) keyword(def) ident(check_xhtml)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) keyword(for) ident(lno)operator(,) ident(line) keyword(in) predefined(enumerate)operator(()ident(lines)operator(\))operator(:) keyword(for) ident(bad_tag) keyword(in) ident(bad_tags)operator(:) @@ -24825,7 +24825,7 @@ keyword(class) ident(LexersTest)operator(()ident(unittest)operator(.)ident(TestC ident(a)operator(()predefined(type)operator(()predefined(getattr)operator(()ident(lexer)operator(,) ident(attr)operator(\))operator(\)) keyword(is) predefined(list)operator(,) string operator(%) operator(()ident(lexer)operator(,) ident(attr)operator(\))operator(\)) ident(result) operator(=) ident(lexer)operator(.)ident(analyse_text)operator(()stringoperator(\)) - ident(a)operator(()predefined(isinstance)operator(()ident(result)operator(,) predefined(float)operator(\)) keyword(and) float(0)float(.0) operator(<=) ident(result) operator(<=) float(1)float(.0)operator(\)) + ident(a)operator(()predefined(isinstance)operator(()ident(result)operator(,) predefined(float)operator(\)) keyword(and) float(0.0) operator(<=) ident(result) operator(<=) float(1.0)operator(\)) ident(inst) operator(=) ident(lexer)operator(()ident(opt1)operator(=)stringoperator(,) ident(opt2)operator(=)stringoperator(\)) keyword(if) predefined(issubclass)operator(()ident(lexer)operator(,) ident(RegexLexer)operator(\))operator(:) @@ -25561,9 +25561,9 @@ keyword(class) ident(UtilTest)operator(()ident(unittest)operator(.)ident(TestCas keyword(def) ident(test_analysator)operator(()pre_constant(self)operator(\))operator(:) keyword(class) ident(X)operator(()predefined(object)operator(\))operator(:) keyword(def) ident(analyse)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) ident(analyse) operator(=) ident(util)operator(.)ident(make_analysator)operator(()ident(analyse)operator(\)) - pre_constant(self)operator(.)ident(assertEquals)operator(()ident(X)operator(.)ident(analyse)operator(()stringoperator(\))operator(,) float(0)float(.5)operator(\)) + pre_constant(self)operator(.)ident(assertEquals)operator(()ident(X)operator(.)ident(analyse)operator(()stringoperator(\))operator(,) float(0.5)operator(\)) keyword(def) ident(test_shebang_matches)operator(()pre_constant(self)operator(\))operator(:) pre_constant(self)operator(.)ident(assert_)operator(()ident(util)operator(.)ident(shebang_matches)operator(()stringoperator(,) stringoperator(\))operator(\)) diff --git a/test/scanners/python/python3.expected.raydebug b/test/scanners/python/python3.expected.raydebug new file mode 100644 index 0000000..6a9c056 --- /dev/null +++ b/test/scanners/python/python3.expected.raydebug @@ -0,0 +1,19 @@ +ident(Old)operator(:) keyword(print) stringoperator(,) integer(2)operator(*)integer(2) +ident(New)operator(:) ident(print)operator(()stringoperator(,) integer(2)operator(*)integer(2)operator(\)) + +ident(Old)operator(:) keyword(print) ident(x)operator(,) comment(# Trailing comma suppresses newline) +ident(New)operator(:) ident(print)operator(()ident(x)operator(,) ident(end)operator(=)stringoperator(\)) comment(# Appends a space instead of a newline) + +ident(Old)operator(:) keyword(print) comment(# Prints a newline) +ident(New)operator(:) ident(print)operator(()operator(\)) comment(# You must call the function!) + +ident(Old)operator(:) keyword(print) operator(>>)ident(sys)operator(.)ident(stderr)operator(,) string +ident(New)operator(:) ident(print)operator(()stringoperator(,) ident(file)operator(=)ident(sys)operator(.)ident(stderr)operator(\)) + +ident(Old)operator(:) keyword(print) operator(()ident(x)operator(,) ident(y)operator(\)) comment(# prints repr((x, y\)\)) +ident(New)operator(:) ident(print)operator(()operator(()ident(x)operator(,) ident(y)operator(\))operator(\)) comment(# Not the same as print(x, y\)!) + +ident(print)operator(()stringoperator(,) integer(2)operator(**)integer(32)operator(,) string possibilities!)delimiter(")>operator(,) ident(sep)operator(=)stringoperator(\)) + + +string \ No newline at end of file diff --git a/test/scanners/python/python3.in.py b/test/scanners/python/python3.in.py new file mode 100644 index 0000000..178a798 --- /dev/null +++ b/test/scanners/python/python3.in.py @@ -0,0 +1,19 @@ +Old: print "The answer is", 2*2 +New: print("The answer is", 2*2) + +Old: print x, # Trailing comma suppresses newline +New: print(x, end=" ") # Appends a space instead of a newline + +Old: print # Prints a newline +New: print() # You must call the function! + +Old: print >>sys.stderr, "fatal error" +New: print("fatal error", file=sys.stderr) + +Old: print (x, y) # prints repr((x, y)) +New: print((x, y)) # Not the same as print(x, y)! + +print("There are <", 2**32, "> possibilities!", sep="") + + +b"byte string\99\"" \ No newline at end of file diff --git a/test/scanners/python/unistring.expected.raydebug b/test/scanners/python/unistring.expected.raydebug index 1f21457..aaf2372 100644 --- a/test/scanners/python/unistring.expected.raydebug +++ b/test/scanners/python/unistring.expected.raydebug @@ -22,7 +22,7 @@ ident(Cn) operator(=) string keyword(try)operator(:) - ident(Cs) operator(=) predefined(eval)operator(()ident(u_prefix) operator(+) stringoperator(\)) + ident(Cs) operator(=) predefined(eval)operator(()ident(u_prefix) operator(+) stringoperator(\)) keyword(except) exception(UnicodeDecodeError)operator(:) ident(Cs) operator(=) string comment(# Jython can't handle isolated surrogates) -- cgit v1.2.1