python3: deal with astroid's module.file_stream returning bytes

Use tokenize.tokenize() which wants a byte stream.  Everywhere else,
decode as necessary.

1 files changed, 6 insertions, 5 deletions

diff --git a/utils.py b/utils.py
index 4b109dbb0..2dac831d3 100644
--- a/utils.py
+++ b/utils.py
@@ -112,12 +112,13 @@ def category_id(id):
 def tokenize_module(module):
     stream = module.file_stream
     stream.seek(0)
-    if sys.version_info < (3, 0) and module.file_encoding is not None:
-        readline = lambda: stream.readline().decode(module.file_encoding, 
+    readline = stream.readline
+    if sys.version_info < (3, 0):
+        if module.file_encoding is not None:
+            readline = lambda: stream.readline().decode(module.file_encoding,
                                                     'replace')
-    else:
-        readline = stream.readline
-    return list(tokenize.generate_tokens(readline))
+        return list(tokenize.generate_tokens(readline))
+    return list(tokenize.tokenize(readline))
 
     
 class Message(object):