fix for files with coding cookies and BOMs

1 files changed, 5 insertions, 3 deletions

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 89721371b9..519dfa5784 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -333,9 +333,11 @@ def detect_encoding(readline):
             # This behaviour mimics the Python interpreter
             raise SyntaxError("unknown encoding: " + encoding)
 
-        if bom_found and codec.name != 'utf-8':
-            # This behaviour mimics the Python interpreter
-            raise SyntaxError('encoding problem: utf-8')
+        if bom_found:
+            if codec.name != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            encoding += '-sig'
         return encoding
 
     first = read_or_stop()