From 59b31ae8596f9606217b09d4e3f00dcf5aab8475 Mon Sep 17 00:00:00 2001 From: murphy Date: Wed, 22 Apr 2009 02:40:04 +0000 Subject: Improved Python scanner (issue #41). * fixed numeric literals * better Python 3 support * bugfixes, optimizations * added two more test files --- test/scanners/python/literals.expected.raydebug | 16 +++ test/scanners/python/literals.in.py | 16 +++ test/scanners/python/pleac.expected.raydebug | 28 ++-- test/scanners/python/pygments.expected.raydebug | 172 +++++++++++------------ test/scanners/python/python3.expected.raydebug | 19 +++ test/scanners/python/python3.in.py | 19 +++ test/scanners/python/unistring.expected.raydebug | 2 +- 7 files changed, 171 insertions(+), 101 deletions(-) create mode 100644 test/scanners/python/literals.expected.raydebug create mode 100644 test/scanners/python/literals.in.py create mode 100644 test/scanners/python/python3.expected.raydebug create mode 100644 test/scanners/python/python3.in.py (limited to 'test') diff --git a/test/scanners/python/literals.expected.raydebug b/test/scanners/python/literals.expected.raydebug new file mode 100644 index 0000000..ca7d22b --- /dev/null +++ b/test/scanners/python/literals.expected.raydebug @@ -0,0 +1,16 @@ +comment(# from http://docs.python.org/reference/lexical_analysis.html#literals) + +ident(re)operator(.)ident(compile)operator(()string comment(# letter or underscore) + string comment(# letter, digit or underscore) + operator(\)) +integer(7) integer(2147483647) oct(0177) +integer(3L) integer(79228162514264337593543950336L) oct(0377L) hex(0x100000000L) + integer(79228162514264337593543950336) hex(0xdeadbeef) + +float(3.14) float(10.) float(.001) float(1e100) float(3.14e-10) float(0e0) + +imaginary(3.14j) imaginary(10.j) imaginary(10j) imaginary(.001j) imaginary(1e100j) imaginary(3.14e-10j) + +integer(2) operator(+) imaginary(1j) + +float(3.5e+54) operator(-) imaginary(2.5e-13j) \ No newline at end of file diff --git a/test/scanners/python/literals.in.py b/test/scanners/python/literals.in.py new file mode 100644 index 0000000..ccf15ac --- /dev/null +++ b/test/scanners/python/literals.in.py @@ -0,0 +1,16 @@ +# from http://docs.python.org/reference/lexical_analysis.html#literals + +re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) +7 2147483647 0177 +3L 79228162514264337593543950336L 0377L 0x100000000L + 79228162514264337593543950336 0xdeadbeef + +3.14 10. .001 1e100 3.14e-10 0e0 + +3.14j 10.j 10j .001j 1e100j 3.14e-10j + +2 + 1j + +3.5e+54 - 2.5e-13j \ No newline at end of file diff --git a/test/scanners/python/pleac.expected.raydebug b/test/scanners/python/pleac.expected.raydebug index fc88237..d4af49e 100644 --- a/test/scanners/python/pleac.expected.raydebug +++ b/test/scanners/python/pleac.expected.raydebug @@ -278,7 +278,7 @@ keyword(for) ident(ln) keyword(in) ident(fileinput)operator(.)ident(input)operat keyword(for) ident(c) keyword(in) ident(ln)operator(:) ident(sys)operator(.)ident(stdout)operator(.)ident(write)operator(()ident(c)operator(\)) ident(sys)operator(.)ident(stdout)operator(.)ident(flush)operator(()operator(\)) - ident(select)operator(.)ident(select)operator(()operator([)operator(])operator(,)operator([)operator(])operator(,)operator([)operator(])operator(,) float(0)float(.005) operator(*) ident(DELAY)operator(\)) + ident(select)operator(.)ident(select)operator(()operator([)operator(])operator(,)operator([)operator(])operator(,)operator([)operator(])operator(,) float(0.005) operator(*) ident(DELAY)operator(\)) comment(#-----------------------------) comment(# @@PLEAC@@_1.6) @@ -827,7 +827,7 @@ comment(# @@PLEAC@@_2.3) comment(#-----------------------------) ident(rounded) operator(=) predefined(round)operator(()ident(num)operator(\)) comment(# rounds to integer) comment(#-----------------------------) -ident(a) operator(=) float(0)float(.255) +ident(a) operator(=) float(0.255) ident(b) operator(=) string operator(%) ident(a) keyword(print) string operator(%) operator(()ident(a)operator(,) ident(b)operator(\)) keyword(print) string operator(%) operator(()ident(a)operator(,) ident(a)operator(\)) @@ -839,7 +839,7 @@ comment(#-----------------------------) keyword(from) ident(math) keyword(import) ident(floor)operator(,) ident(ceil) keyword(print) string -ident(a) operator(=) operator([)float(3)float(.3)operator(,) float(3)float(.5)operator(,) float(3)float(.7)operator(,) operator(-)float(3)float(.3)operator(]) +ident(a) operator(=) operator([)float(3.3)operator(,) float(3.5)operator(,) float(3.7)operator(,) operator(-)float(3.3)operator(]) keyword(for) ident(n) keyword(in) ident(a)operator(:) keyword(print) string operator(%) operator(()ident(n)operator(,) predefined(int)operator(()ident(n)operator(\))operator(,) ident(floor)operator(()ident(n)operator(\))operator(,) ident(ceil)operator(()ident(n)operator(\))operator(\)) comment(#=> number int floor ceil) @@ -1018,8 +1018,8 @@ comment(#-----------------------------) comment(# @@PLEAC@@_2.15) comment(#-----------------------------) -ident(a) operator(=) integer(3)operator(+)integer(5)ident(j) -ident(b) operator(=) integer(2)operator(-)integer(2)ident(j) +ident(a) operator(=) integer(3)operator(+)imaginary(5j) +ident(b) operator(=) integer(2)operator(-)imaginary(2j) ident(c) operator(=) ident(a) operator(*) ident(b) keyword(print) stringoperator(,) ident(c) comment(#=> c = (16+4j\)) @@ -1028,7 +1028,7 @@ keyword(print) ident(c)operator(.)ident(real)operator(,) ident(c)operator(.)iden comment(#=> 16.0 4.0 (16-4j\)) comment(#-----------------------------) keyword(import) ident(cmath) -keyword(print) ident(cmath)operator(.)ident(sqrt)operator(()integer(3)operator(+)integer(4)ident(j)operator(\)) +keyword(print) ident(cmath)operator(.)ident(sqrt)operator(()integer(3)operator(+)imaginary(4j)operator(\)) comment(#=> (2+1j\)) comment(#-----------------------------) @@ -1426,7 +1426,7 @@ comment(# @@PLEAC@@_3.10) comment(#----------------------------- ) comment(# Short Sleeps) -ident(seconds) operator(=) float(3)float(.1) +ident(seconds) operator(=) float(3.1) ident(time)operator(.)ident(sleep)operator(()ident(seconds)operator(\)) keyword(print) string @@ -2118,7 +2118,7 @@ keyword(for) ident(term) keyword(in) ident(mylist)operator(:) ident(matching)operator(.)ident(append)operator(()ident(term)operator(\)) comment(#-----------------------------) ident(bigs) operator(=) operator([)ident(num) keyword(for) ident(num) keyword(in) ident(nums) keyword(if) ident(num) operator(>) integer(1000000)operator(]) -ident(pigs) operator(=) operator([)ident(user) keyword(for) operator(()ident(user)operator(,) ident(val)operator(\)) keyword(in) ident(users)operator(.)ident(items)operator(()operator(\)) keyword(if) ident(val) operator(>) float(1)ident(e7)operator(]) +ident(pigs) operator(=) operator([)ident(user) keyword(for) operator(()ident(user)operator(,) ident(val)operator(\)) keyword(in) ident(users)operator(.)ident(items)operator(()operator(\)) keyword(if) ident(val) operator(>) float(1e7)operator(]) comment(#-----------------------------) keyword(import) ident(os) ident(matching) operator(=) operator([)ident(line) keyword(for) ident(line) keyword(in) ident(os)operator(.)ident(popen)operator(()stringoperator(\)) @@ -5818,7 +5818,7 @@ keyword(print) ident(hypotenuse)operator(()operator(*)ident(a)operator(\)) comment(#-----------------------------) ident(both) operator(=) ident(men) operator(+) ident(women) comment(#-----------------------------) -ident(nums) operator(=) operator([)float(1)float(.4)operator(,) float(3)float(.5)operator(,) float(6)float(.7)operator(]) +ident(nums) operator(=) operator([)float(1.4)operator(,) float(3.5)operator(,) float(6.7)operator(]) comment(# Provided for demonstration purposes only. Use:) comment(# ints = [int(num\) for num in nums] ) keyword(def) ident(int_all)operator(()ident(nums)operator(\))operator(:) @@ -5828,7 +5828,7 @@ keyword(def) ident(int_all)operator(()ident(nums)operator(\))operator(:) keyword(return) ident(retlist) ident(ints) operator(=) ident(int_all)operator(()ident(nums)operator(\)) comment(# nums unchanged) comment(#-----------------------------) -ident(nums) operator(=) operator([)float(1)float(.4)operator(,) float(3)float(.5)operator(,) float(6)float(.7)operator(]) +ident(nums) operator(=) operator([)float(1.4)operator(,) float(3.5)operator(,) float(6.7)operator(]) keyword(def) ident(trunc_em)operator(()ident(nums)operator(\))operator(:) keyword(for) ident(i)operator(,)ident(elem) keyword(in) predefined(enumerate)operator(()ident(nums)operator(\))operator(:) @@ -6669,7 +6669,7 @@ ident(rp) operator(=) ident(employees)operator(.)ident(get)operator(()integer(13 keyword(if) operator(()ident(rp)operator(\))operator(:) keyword(print) string operator(%) ident(rp)operator([)stringoperator(]) -ident(byname)operator([)stringoperator(])operator([)stringoperator(]) operator(*=) float(1)float(.035) +ident(byname)operator([)stringoperator(])operator([)stringoperator(]) operator(*=) float(1.035) ident(peons) operator(=) operator([)ident(r) keyword(for) ident(r) keyword(in) ident(employees)operator(.)ident(values)operator(()operator(\)) keyword(if) ident(r)operator([)stringoperator(]) operator(==) stringoperator(]) ident(tsevens) operator(=) operator([)ident(r) keyword(for) ident(r) keyword(in) ident(employees)operator(.)ident(values)operator(()operator(\)) keyword(if) ident(r)operator([)stringoperator(]) operator(==) integer(27)operator(]) @@ -8270,7 +8270,7 @@ keyword(class) ident(FixNum)operator(:) keyword(def) ident(__div__)operator(()pre_constant(self)operator(,) ident(other)operator(\))operator(:) comment(# Force to use floating point, since 2/3 in Python is 0) comment(# Don't use float(\) since that will convert strings) - keyword(return) ident(FixNum)operator(()operator(()pre_constant(self)operator(.)ident(value)operator(+)float(0)float(.0)operator(\)) operator(/) ident(other)operator(.)ident(value)operator(,) + keyword(return) ident(FixNum)operator(()operator(()pre_constant(self)operator(.)ident(value)operator(+)float(0.0)operator(\)) operator(/) ident(other)operator(.)ident(value)operator(,) predefined(max)operator(()pre_constant(self)operator(.)ident(places)operator(,) ident(other)operator(.)ident(places)operator(\))operator(\)) keyword(def) ident(__str__)operator(()pre_constant(self)operator(\))operator(:) @@ -8916,7 +8916,7 @@ keyword(except) ident(KeyboardError)operator(:) comment(#----------------------------------------) comment(# @@PLEAC@@_15.7) -keyword(print) stringoperator(;) +keyword(print) stringoperator(;) comment(#----------------------------------------) comment(# @@INCOMPLETE@@) @@ -9453,7 +9453,7 @@ keyword(import) ident(sys) keyword(import) ident(signal) keyword(def) ident(ding)operator(()ident(signum)operator(,) ident(frame)operator(\))operator(:) - keyword(print) string + keyword(print) string keyword(return) ident(signal)operator(.)ident(signal)operator(()ident(signal)operator(.)ident(SIGINT)operator(,) ident(ding)operator(\)) diff --git a/test/scanners/python/pygments.expected.raydebug b/test/scanners/python/pygments.expected.raydebug index 9a1a9d2..7084bd7 100644 --- a/test/scanners/python/pygments.expected.raydebug +++ b/test/scanners/python/pygments.expected.raydebug @@ -3742,7 +3742,7 @@ keyword(class) ident(LatexFormatter)operator(()ident(Formatter)operator(\))opera keyword(def) ident(rgbcolor)operator(()ident(col)operator(\))operator(:) keyword(if) ident(col)operator(:) - keyword(return) stringoperator(.)ident(join)operator(()operator([)string operator(%)operator(()predefined(int)operator(()ident(col)operator([)ident(i)operator(]) operator(+) ident(col)operator([)ident(i) operator(+) integer(1)operator(])operator(,) integer(16)operator(\)) operator(/) integer(25)float(5)float(.0)operator(\)) + keyword(return) stringoperator(.)ident(join)operator(()operator([)string operator(%)operator(()predefined(int)operator(()ident(col)operator([)ident(i)operator(]) operator(+) ident(col)operator([)ident(i) operator(+) integer(1)operator(])operator(,) integer(16)operator(\)) operator(/) float(255.0)operator(\)) keyword(for) ident(i) keyword(in) operator(()integer(0)operator(,) integer(2)operator(,) integer(4)operator(\))operator(])operator(\)) keyword(else)operator(:) keyword(return) string @@ -4604,7 +4604,7 @@ ident(__all__) operator(=) operator([)stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(]) -ident(_default_analyse) operator(=) predefined(staticmethod)operator(()keyword(lambda) ident(x)operator(:) float(0)float(.0)operator(\)) +ident(_default_analyse) operator(=) predefined(staticmethod)operator(()keyword(lambda) ident(x)operator(:) float(0.0)operator(\)) keyword(class) ident(LexerMeta)operator(()predefined(type)operator(\))operator(:) @@ -5417,7 +5417,7 @@ keyword(def) ident(guess_lexer_for_filename)operator(()ident(_fn)operator(,) ide ident(result) operator(=) operator([)operator(]) keyword(for) ident(lexer) keyword(in) ident(matching_lexers)operator(:) ident(rv) operator(=) ident(lexer)operator(.)ident(analyse_text)operator(()ident(_text)operator(\)) - keyword(if) ident(rv) operator(==) float(1)float(.0)operator(:) + keyword(if) ident(rv) operator(==) float(1.0)operator(:) keyword(return) ident(lexer)operator(()operator(**)ident(options)operator(\)) ident(result)operator(.)ident(append)operator(()operator(()ident(rv)operator(,) ident(lexer)operator(\))operator(\)) ident(result)operator(.)ident(sort)operator(()operator(\)) @@ -5430,10 +5430,10 @@ keyword(def) ident(guess_lexer)operator(()ident(_text)operator(,) operator(**)id string - ident(best_lexer) operator(=) operator([)float(0)float(.0)operator(,) pre_constant(None)operator(]) + ident(best_lexer) operator(=) operator([)float(0.0)operator(,) pre_constant(None)operator(]) keyword(for) ident(lexer) keyword(in) ident(_iter_lexerclasses)operator(()operator(\))operator(:) ident(rv) operator(=) ident(lexer)operator(.)ident(analyse_text)operator(()ident(_text)operator(\)) - keyword(if) ident(rv) operator(==) float(1)float(.0)operator(:) + keyword(if) ident(rv) operator(==) float(1.0)operator(:) keyword(return) ident(lexer)operator(()operator(**)ident(options)operator(\)) keyword(if) ident(rv) operator(>) ident(best_lexer)operator([)integer(0)operator(])operator(:) ident(best_lexer)operator([)operator(:)operator(]) operator(=) operator(()ident(rv)operator(,) ident(lexer)operator(\)) @@ -10535,8 +10535,8 @@ keyword(class) ident(PerlLexer)operator(()ident(RegexLexer)operator(\))operator( keyword(if) ident(shebang_matches)operator(()ident(text)operator(,) stringoperator(\))operator(:) keyword(return) pre_constant(True) keyword(if) string keyword(in) ident(text)operator(:) - keyword(return) float(0)float(.9) - keyword(return) float(0)float(.1) comment(# who knows, might still be perl!) + keyword(return) float(0.9) + keyword(return) float(0.1) comment(# who knows, might still be perl!) keyword(class) ident(LuaLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -13272,10 +13272,10 @@ keyword(class) ident(CSharpAspxLexer)operator(()ident(DelegatingLexer)operator(\ keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.2) + keyword(return) float(0.2) keyword(elif) ident(re)operator(.)ident(search)operator(()string]+language=[")content(\\')content(]C#)delimiter(')>operator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.15) - keyword(return) float(0)float(.001) comment(# TODO really only for when filename matched...) + keyword(return) float(0.15) + keyword(return) float(0.001) comment(# TODO really only for when filename matched...) keyword(class) ident(VbNetAspxLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.2) + keyword(return) float(0.2) keyword(elif) ident(re)operator(.)ident(search)operator(()string]+language=[")content(\\')content(]vb)delimiter(')>operator(,) ident(text)operator(,) ident(re)operator(.)ident(I)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - keyword(return) float(0)float(.15) + keyword(return) float(0.15) comment(# -*- coding: utf-8 -*-) stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(M)operator(\))operator(:) comment(# comment) - keyword(return) float(0)float(.9) + keyword(return) float(0.9) keyword(elif) ident(re)operator(.)ident(match)operator(()stringoperator(,) ident(text)operator(,) ident(re)operator(.)ident(M)operator(\))operator(:) comment(# system cmd) - keyword(return) float(0)float(.9) - keyword(return) float(0)float(.1) + keyword(return) float(0.9) + keyword(return) float(0.1) ident(line_re) operator(=) ident(re)operator(.)ident(compile)operator(()stringoperator(\)) @@ -16697,7 +16697,7 @@ keyword(class) ident(RagelEmbeddedLexer)operator(()ident(RegexLexer)operator(\)) operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) string keyword(in) ident(text) keyword(or) float(0)float(.1) + keyword(return) string keyword(in) ident(text) keyword(or) float(0.1) keyword(class) ident(RagelRubyLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) string keyword(in) ident(text) keyword(and) string)delimiter(')> keyword(in) ident(text)operator(:) - keyword(return) float(0)float(.4) + keyword(return) float(0.4) keyword(class) ident(SmartyLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -17443,15 +17443,15 @@ keyword(class) ident(SmartyLexer)operator(()ident(RegexLexer)operator(\))operato operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.15) + ident(rv) operator(+=) float(0.15) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.15) + ident(rv) operator(+=) float(0.15) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.15) + ident(rv) operator(+=) float(0.15) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.01) + ident(rv) operator(+=) float(0.01) keyword(return) ident(rv) @@ -17527,13 +17527,13 @@ keyword(class) ident(DjangoLexer)operator(()ident(RegexLexer)operator(\))operato operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(return) ident(rv) @@ -18023,12 +18023,12 @@ keyword(class) ident(HtmlGenshiLexer)operator(()ident(DelegatingLexer)operator(\ operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) + ident(rv) operator(+=) float(0.2) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) - keyword(return) ident(rv) operator(+) ident(HtmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(+=) float(0.2) + keyword(return) ident(rv) operator(+) ident(HtmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(class) ident(GenshiLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18048,12 +18048,12 @@ keyword(class) ident(GenshiLexer)operator(()ident(DelegatingLexer)operator(\))op operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) + ident(rv) operator(+=) float(0.2) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\)) keyword(is) keyword(not) pre_constant(None)operator(:) - ident(rv) operator(+=) float(0)float(.2) - keyword(return) ident(rv) operator(+) ident(XmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(+=) float(0.2) + keyword(return) ident(rv) operator(+) ident(XmlLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(class) ident(JavascriptGenshiLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18075,7 +18075,7 @@ keyword(class) ident(JavascriptGenshiLexer)operator(()ident(DelegatingLexer)oper operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(CssGenshiLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18093,7 +18093,7 @@ keyword(class) ident(CssGenshiLexer)operator(()ident(DelegatingLexer)operator(\) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(GenshiLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(RhtmlLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18114,10 +18114,10 @@ keyword(class) ident(RhtmlLexer)operator(()ident(DelegatingLexer)operator(\))ope predefined(super)operator(()ident(RhtmlLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(ErbLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) comment(# one more than the XmlErbLexer returns) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18136,9 +18136,9 @@ keyword(class) ident(XmlErbLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(XmlErbLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(ErbLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18156,7 +18156,7 @@ keyword(class) ident(CssErbLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(CssErbLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(ErbLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptErbLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18177,7 +18177,7 @@ keyword(class) ident(JavascriptErbLexer)operator(()ident(DelegatingLexer)operato operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(ErbLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(HtmlPhpLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18200,9 +18200,9 @@ keyword(class) ident(HtmlPhpLexer)operator(()ident(DelegatingLexer)operator(\))o predefined(super)operator(()ident(HtmlPhpLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(PhpLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18220,9 +18220,9 @@ keyword(class) ident(XmlPhpLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(XmlPhpLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(PhpLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18240,7 +18240,7 @@ keyword(class) ident(CssPhpLexer)operator(()ident(DelegatingLexer)operator(\))op predefined(super)operator(()ident(CssPhpLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(PhpLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(PhpLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptPhpLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18281,9 +18281,9 @@ keyword(class) ident(HtmlSmartyLexer)operator(()ident(DelegatingLexer)operator(\ predefined(super)operator(()ident(HtmlSmartyLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(SmartyLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18302,9 +18302,9 @@ keyword(class) ident(XmlSmartyLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(XmlSmartyLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(SmartyLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18323,7 +18323,7 @@ keyword(class) ident(CssSmartyLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(CssSmartyLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(SmartyLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptSmartyLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18344,7 +18344,7 @@ keyword(class) ident(JavascriptSmartyLexer)operator(()ident(DelegatingLexer)oper operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(SmartyLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(HtmlDjangoLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18364,9 +18364,9 @@ keyword(class) ident(HtmlDjangoLexer)operator(()ident(DelegatingLexer)operator(\ predefined(super)operator(()ident(HtmlDjangoLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(HtmlLexer)operator(,) ident(DjangoLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.5) + ident(rv) operator(+=) float(0.5) keyword(return) ident(rv) @@ -18385,9 +18385,9 @@ keyword(class) ident(XmlDjangoLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(XmlDjangoLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(DjangoLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(return) ident(rv) @@ -18406,7 +18406,7 @@ keyword(class) ident(CssDjangoLexer)operator(()ident(DelegatingLexer)operator(\) predefined(super)operator(()ident(CssDjangoLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(CssLexer)operator(,) ident(DjangoLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JavascriptDjangoLexer)operator(()ident(DelegatingLexer)operator(\))operator(:) @@ -18431,7 +18431,7 @@ keyword(class) ident(JavascriptDjangoLexer)operator(()ident(DelegatingLexer)oper operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.05) + keyword(return) ident(DjangoLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.05) keyword(class) ident(JspRootLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -18474,11 +18474,11 @@ keyword(class) ident(JspLexer)operator(()ident(DelegatingLexer)operator(\))opera predefined(super)operator(()ident(JspLexer)operator(,) pre_constant(self)operator(\))operator(.)ident(__init__)operator(()ident(XmlLexer)operator(,) ident(JspRootLexer)operator(,) operator(**)ident(options)operator(\)) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) ident(JavaLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0)float(.01) + ident(rv) operator(=) ident(JavaLexer)operator(.)ident(analyse_text)operator(()ident(text)operator(\)) operator(-) float(0.01) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.4) + ident(rv) operator(+=) float(0.4) keyword(if) string keyword(in) ident(text) keyword(and) string)delimiter(')> keyword(in) ident(text)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(return) ident(rv) @@ -18790,7 +18790,7 @@ keyword(class) ident(DiffLexer)operator(()ident(RegexLexer)operator(\))operator( keyword(if) ident(text)operator([)operator(:)integer(5)operator(]) operator(==) stringoperator(:) keyword(return) pre_constant(True) keyword(if) ident(text)operator([)operator(:)integer(4)operator(]) operator(==) stringoperator(:) - keyword(return) float(0)float(.9) + keyword(return) float(0.9) ident(DPATCH_KEYWORDS) operator(=) operator([)stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) stringoperator(,) @@ -19042,7 +19042,7 @@ keyword(class) ident(GroffLexer)operator(()ident(RegexLexer)operator(\))operator keyword(if) ident(text)operator([)operator(:)integer(4)operator(]) operator(==) stringoperator(:) keyword(return) pre_constant(True) keyword(if) ident(text)operator([)integer(1)operator(:)integer(3)operator(])operator(.)ident(isalnum)operator(()operator(\)) keyword(and) ident(text)operator([)integer(3)operator(])operator(.)ident(isspace)operator(()operator(\))operator(:) - keyword(return) float(0)float(.9) + keyword(return) float(0.9) keyword(class) ident(ApacheConfLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -19270,14 +19270,14 @@ keyword(class) ident(RstLexer)operator(()ident(RegexLexer)operator(\))operator(: keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(text)operator([)operator(:)integer(2)operator(]) operator(==) string keyword(and) ident(text)operator([)integer(2)operator(:)integer(3)operator(]) operator(!=) stringoperator(:) - keyword(return) float(0)float(.3) + keyword(return) float(0.3) ident(p1) operator(=) ident(text)operator(.)ident(find)operator(()stringoperator(\)) ident(p2) operator(=) ident(text)operator(.)ident(find)operator(()stringoperator(,) ident(p1) operator(+) integer(1)operator(\)) keyword(if) operator(()ident(p2) operator(>) operator(-)integer(1) keyword(and) comment(# has two lines) ident(p1) operator(*) integer(2) operator(+) integer(1) operator(==) ident(p2) keyword(and) comment(# they are the same length) ident(text)operator([)ident(p1)operator(+)integer(1)operator(]) keyword(in) string keyword(and) comment(# the next line both starts and ends with) ident(text)operator([)ident(p1)operator(+)integer(1)operator(]) operator(==) ident(text)operator([)ident(p2)operator(-)integer(1)operator(])operator(\))operator(:) comment(# ...a sufficiently high header) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) keyword(class) ident(VimLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20215,7 +20215,7 @@ keyword(class) ident(ActionScriptLexer)operator(()ident(RegexLexer)operator(\))o operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.05) + keyword(return) float(0.05) keyword(class) ident(ActionScript3Lexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20293,8 +20293,8 @@ keyword(class) ident(ActionScript3Lexer)operator(()ident(RegexLexer)operator(\)) operator(}) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - keyword(if) ident(re)operator(.)ident(match)operator(()stringoperator(,) ident(text)operator(\))operator(:) keyword(return) float(0)float(.3) - keyword(return) float(0)float(.1) + keyword(if) ident(re)operator(.)ident(match)operator(()stringoperator(,) ident(text)operator(\))operator(:) keyword(return) float(0.3) + keyword(return) float(0.1) keyword(class) ident(CssLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20487,7 +20487,7 @@ keyword(class) ident(HtmlLexer)operator(()ident(RegexLexer)operator(\))operator( keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(html_doctype_matches)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) keyword(class) ident(PhpLexer)operator(()ident(RegexLexer)operator(\))operator(:) @@ -20631,11 +20631,11 @@ keyword(class) ident(PhpLexer)operator(()ident(RegexLexer)operator(\))operator(: keyword(yield) ident(index)operator(,) ident(token)operator(,) ident(value) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) - ident(rv) operator(=) float(0)float(.0) + ident(rv) operator(=) float(0.0) keyword(if) ident(re)operator(.)ident(search)operator(()stringoperator(,) ident(text)operator(\))operator(:) - ident(rv) operator(+=) float(0)float(.3) + ident(rv) operator(+=) float(0.3) keyword(if) string)delimiter(')> keyword(in) ident(text)operator(:) - ident(rv) operator(+=) float(0)float(.1) + ident(rv) operator(+=) float(0.1) keyword(return) ident(rv) @@ -20684,7 +20684,7 @@ keyword(class) ident(XmlLexer)operator(()ident(RegexLexer)operator(\))operator(: keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) keyword(class) ident(XsltLexer)operator(()ident(XmlLexer)operator(\))operator(:) @@ -20720,7 +20720,7 @@ keyword(class) ident(XsltLexer)operator(()ident(XmlLexer)operator(\))operator(:) keyword(def) ident(analyse_text)operator(()ident(text)operator(\))operator(:) keyword(if) ident(looks_like_xml)operator(()ident(text)operator(\)) keyword(and) string keyword(in) ident(text)operator(:) - keyword(return) float(0)float(.8) + keyword(return) float(0.8) @@ -22616,8 +22616,8 @@ keyword(def) ident(make_analysator)operator(()ident(f)operator(\))operator(:) keyword(def) ident(text_analyse)operator(()ident(text)operator(\))operator(:) ident(rv) operator(=) ident(f)operator(()ident(text)operator(\)) keyword(if) keyword(not) ident(rv)operator(:) - keyword(return) float(0)float(.0) - keyword(return) predefined(min)operator(()float(1)float(.0)operator(,) predefined(max)operator(()float(0)float(.0)operator(,) predefined(float)operator(()ident(rv)operator(\))operator(\))operator(\)) + keyword(return) float(0.0) + keyword(return) predefined(min)operator(()float(1.0)operator(,) predefined(max)operator(()float(0.0)operator(,) predefined(float)operator(()ident(rv)operator(\))operator(\))operator(\)) ident(text_analyse)operator(.)ident(__doc__) operator(=) ident(f)operator(.)ident(__doc__) keyword(return) predefined(staticmethod)operator(()ident(text_analyse)operator(\)) @@ -22770,7 +22770,7 @@ ident(misspellings) operator(=) operator([)stringoperator(]) comment(# ALLOW-MISSPELLING) -operator(@)ident(checker)operator(()stringoperator(\)) +decorator(@checker)operator(()stringoperator(\)) keyword(def) ident(check_syntax)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) keyword(try)operator(:) predefined(compile)operator(()stringoperator(.)ident(join)operator(()ident(lines)operator(\))operator(,) ident(fn)operator(,) stringoperator(\)) @@ -22778,7 +22778,7 @@ keyword(def) ident(check_syntax)operator(()ident(fn)operator(,) ident(lines)oper keyword(yield) integer(0)operator(,) string operator(%) ident(err) -operator(@)ident(checker)operator(()stringoperator(\)) +decorator(@checker)operator(()stringoperator(\)) keyword(def) ident(check_style_and_encoding)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) ident(encoding) operator(=) string keyword(for) ident(lno)operator(,) ident(line) keyword(in) predefined(enumerate)operator(()ident(lines)operator(\))operator(:) @@ -22802,7 +22802,7 @@ keyword(def) ident(check_style_and_encoding)operator(()ident(fn)operator(,) iden ident(encoding) operator(=) string -operator(@)ident(checker)operator(()stringoperator(,) ident(only_pkg)operator(=)pre_constant(True)operator(\)) +decorator(@checker)operator(()stringoperator(,) ident(only_pkg)operator(=)pre_constant(True)operator(\)) keyword(def) ident(check_fileheader)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) comment(# line number correction) ident(c) operator(=) integer(1) @@ -22866,7 +22866,7 @@ keyword(def) ident(check_fileheader)operator(()ident(fn)operator(,) ident(lines) keyword(yield) integer(0)operator(,) string -operator(@)ident(checker)operator(()stringoperator(,) stringoperator(,) stringoperator(\)) +decorator(@checker)operator(()stringoperator(,) stringoperator(,) stringoperator(\)) keyword(def) ident(check_whitespace_and_spelling)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) keyword(for) ident(lno)operator(,) ident(line) keyword(in) predefined(enumerate)operator(()ident(lines)operator(\))operator(:) keyword(if) string keyword(in) ident(line)operator(:) @@ -22881,7 +22881,7 @@ keyword(def) ident(check_whitespace_and_spelling)operator(()ident(fn)operator(,) ident(bad_tags) operator(=) operator(()string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')> string)delimiter(')>operator(,) string)delimiter(')>operator(,) string)delimiter(')>operator(,) stringoperator(\)) -operator(@)ident(checker)operator(()stringoperator(\)) +decorator(@checker)operator(()stringoperator(\)) keyword(def) ident(check_xhtml)operator(()ident(fn)operator(,) ident(lines)operator(\))operator(:) keyword(for) ident(lno)operator(,) ident(line) keyword(in) predefined(enumerate)operator(()ident(lines)operator(\))operator(:) keyword(for) ident(bad_tag) keyword(in) ident(bad_tags)operator(:) @@ -24825,7 +24825,7 @@ keyword(class) ident(LexersTest)operator(()ident(unittest)operator(.)ident(TestC ident(a)operator(()predefined(type)operator(()predefined(getattr)operator(()ident(lexer)operator(,) ident(attr)operator(\))operator(\)) keyword(is) predefined(list)operator(,) string operator(%) operator(()ident(lexer)operator(,) ident(attr)operator(\))operator(\)) ident(result) operator(=) ident(lexer)operator(.)ident(analyse_text)operator(()stringoperator(\)) - ident(a)operator(()predefined(isinstance)operator(()ident(result)operator(,) predefined(float)operator(\)) keyword(and) float(0)float(.0) operator(<=) ident(result) operator(<=) float(1)float(.0)operator(\)) + ident(a)operator(()predefined(isinstance)operator(()ident(result)operator(,) predefined(float)operator(\)) keyword(and) float(0.0) operator(<=) ident(result) operator(<=) float(1.0)operator(\)) ident(inst) operator(=) ident(lexer)operator(()ident(opt1)operator(=)stringoperator(,) ident(opt2)operator(=)stringoperator(\)) keyword(if) predefined(issubclass)operator(()ident(lexer)operator(,) ident(RegexLexer)operator(\))operator(:) @@ -25561,9 +25561,9 @@ keyword(class) ident(UtilTest)operator(()ident(unittest)operator(.)ident(TestCas keyword(def) ident(test_analysator)operator(()pre_constant(self)operator(\))operator(:) keyword(class) ident(X)operator(()predefined(object)operator(\))operator(:) keyword(def) ident(analyse)operator(()ident(text)operator(\))operator(:) - keyword(return) float(0)float(.5) + keyword(return) float(0.5) ident(analyse) operator(=) ident(util)operator(.)ident(make_analysator)operator(()ident(analyse)operator(\)) - pre_constant(self)operator(.)ident(assertEquals)operator(()ident(X)operator(.)ident(analyse)operator(()stringoperator(\))operator(,) float(0)float(.5)operator(\)) + pre_constant(self)operator(.)ident(assertEquals)operator(()ident(X)operator(.)ident(analyse)operator(()stringoperator(\))operator(,) float(0.5)operator(\)) keyword(def) ident(test_shebang_matches)operator(()pre_constant(self)operator(\))operator(:) pre_constant(self)operator(.)ident(assert_)operator(()ident(util)operator(.)ident(shebang_matches)operator(()stringoperator(,) stringoperator(\))operator(\)) diff --git a/test/scanners/python/python3.expected.raydebug b/test/scanners/python/python3.expected.raydebug new file mode 100644 index 0000000..6a9c056 --- /dev/null +++ b/test/scanners/python/python3.expected.raydebug @@ -0,0 +1,19 @@ +ident(Old)operator(:) keyword(print) stringoperator(,) integer(2)operator(*)integer(2) +ident(New)operator(:) ident(print)operator(()stringoperator(,) integer(2)operator(*)integer(2)operator(\)) + +ident(Old)operator(:) keyword(print) ident(x)operator(,) comment(# Trailing comma suppresses newline) +ident(New)operator(:) ident(print)operator(()ident(x)operator(,) ident(end)operator(=)stringoperator(\)) comment(# Appends a space instead of a newline) + +ident(Old)operator(:) keyword(print) comment(# Prints a newline) +ident(New)operator(:) ident(print)operator(()operator(\)) comment(# You must call the function!) + +ident(Old)operator(:) keyword(print) operator(>>)ident(sys)operator(.)ident(stderr)operator(,) string +ident(New)operator(:) ident(print)operator(()stringoperator(,) ident(file)operator(=)ident(sys)operator(.)ident(stderr)operator(\)) + +ident(Old)operator(:) keyword(print) operator(()ident(x)operator(,) ident(y)operator(\)) comment(# prints repr((x, y\)\)) +ident(New)operator(:) ident(print)operator(()operator(()ident(x)operator(,) ident(y)operator(\))operator(\)) comment(# Not the same as print(x, y\)!) + +ident(print)operator(()stringoperator(,) integer(2)operator(**)integer(32)operator(,) string possibilities!)delimiter(")>operator(,) ident(sep)operator(=)stringoperator(\)) + + +string \ No newline at end of file diff --git a/test/scanners/python/python3.in.py b/test/scanners/python/python3.in.py new file mode 100644 index 0000000..178a798 --- /dev/null +++ b/test/scanners/python/python3.in.py @@ -0,0 +1,19 @@ +Old: print "The answer is", 2*2 +New: print("The answer is", 2*2) + +Old: print x, # Trailing comma suppresses newline +New: print(x, end=" ") # Appends a space instead of a newline + +Old: print # Prints a newline +New: print() # You must call the function! + +Old: print >>sys.stderr, "fatal error" +New: print("fatal error", file=sys.stderr) + +Old: print (x, y) # prints repr((x, y)) +New: print((x, y)) # Not the same as print(x, y)! + +print("There are <", 2**32, "> possibilities!", sep="") + + +b"byte string\99\"" \ No newline at end of file diff --git a/test/scanners/python/unistring.expected.raydebug b/test/scanners/python/unistring.expected.raydebug index 1f21457..aaf2372 100644 --- a/test/scanners/python/unistring.expected.raydebug +++ b/test/scanners/python/unistring.expected.raydebug @@ -22,7 +22,7 @@ ident(Cn) operator(=) string keyword(try)operator(:) - ident(Cs) operator(=) predefined(eval)operator(()ident(u_prefix) operator(+) stringoperator(\)) + ident(Cs) operator(=) predefined(eval)operator(()ident(u_prefix) operator(+) stringoperator(\)) keyword(except) exception(UnicodeDecodeError)operator(:) ident(Cs) operator(=) string comment(# Jython can't handle isolated surrogates) -- cgit v1.2.1