diff options
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index e94d7b9723..9ea57b6b68 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -49,10 +49,11 @@ Comment = r'#[^\r\n]*' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' -Intnumber = group(Hexnumber, Octnumber, Decnumber) +Hexnumber = r'0[xX][\da-fA-F]*' +Binnumber = r'0[bB][01]*' +Octnumber = r'0[oO][0-7]*' +Decnumber = r'(?:0+|[1-9]\d*)' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) Exponent = r'[eE][-+]?\d+' Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) Expfloat = r'\d+' + Exponent |