Merge branch 'tokenize_error' into 'master'

Fix crash when file fails to tokenize but parses

Closes #532

See merge request pycqa/flake8!314

1 files changed, 14 insertions, 0 deletions

diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py
index ba4dd56..43cc168 100644
--- a/tests/integration/test_main.py
+++ b/tests/integration/test_main.py
@@ -85,3 +85,17 @@ Configured `per-file-ignores` setting:
     incorrect/*
     values/*
 '''  # noqa: E501
+
+
+def test_tokenization_error_but_not_syntax_error(tmpdir, capsys):
+    """Test that flake8 does not crash on tokenization errors."""
+    with tmpdir.as_cwd():
+        # this is a crash in the tokenizer, but not in the ast
+        tmpdir.join('t.py').write("b'foo' \\\n")
+
+        app = application.Application()
+        app.run(['t.py'])
+
+    out, err = capsys.readouterr()
+    assert out == 't.py:1:1: E902 TokenError: EOF in multi-line statement\n'
+    assert err == ''