From 2c31e11ba195a336e1b037f42907cd2e1c53e096 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 21 Nov 2009 09:59:43 -0500 Subject: Handle one more bizarro edge case in tokenizing source: I couldn't look at my own phystokens.py file properly! --- test/stress_phystoken.txt | 11 +++++++++++ test/test_phystokens.py | 5 +---- 2 files changed, 12 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/stress_phystoken.txt b/test/stress_phystoken.txt index bd6a453a..654225c3 100644 --- a/test/stress_phystoken.txt +++ b/test/stress_phystoken.txt @@ -19,6 +19,17 @@ fake_back = """\ ouch """ +# Lots of difficulty happens with code like: +# +# fake_back = """\ +# ouch +# """ +# +# Ugh, the edge cases... + +# What about a comment like this\ +"what's this string doing here?" + class C(object): def there(): this = 5 + \ diff --git a/test/test_phystokens.py b/test/test_phystokens.py index fa0fa043..fb0b5535 100644 --- a/test/test_phystokens.py +++ b/test/test_phystokens.py @@ -38,10 +38,7 @@ class PhysTokensTest(CoverageTest): # before comparing. source = re.sub("(?m)[ \t]+$", "", source) tokenized = re.sub("(?m)[ \t]+$", "", tokenized) - #if source != tokenized: - # open("0.py", "w").write(source) - # open("1.py", "w").write(tokenized) - self.assertEqual(source, tokenized) + self.assert_multiline_equal(source, tokenized) def check_file_tokenization(self, fname): """Use the contents of `fname` for `check_tokenization`.""" -- cgit v1.2.1