Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py.

Now pindent.py works with a "with" statement.  pindent.py no longer produces
improper indentation.  pindent.py now works with continued lines broken after
"class" or "def" keywords and with continuations at the start of line.  Added
regression tests for pindent.py.  Modernized pindent.py.

3 files changed, 398 insertions, 107 deletions

diff --git a/Lib/test/test_tools.py b/Lib/test/test_tools.py
index 10ee27ce08..4756e4f6f7 100644
--- a/Lib/test/test_tools.py
+++ b/Lib/test/test_tools.py
@@ -5,10 +5,15 @@ Tools directory of a Python checkout or tarball, such as reindent.py.
 """
 
 import os
+import sys
 import unittest
+import shutil
+import subprocess
 import sysconfig
+import tempfile
+import textwrap
 from test import test_support
-from test.script_helper import assert_python_ok
+from test.script_helper import assert_python_ok, temp_dir
 
 if not sysconfig.is_python_build():
     # XXX some installers do contain the tools, should we detect that
@@ -17,10 +22,11 @@ if not sysconfig.is_python_build():
 
 basepath = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
                         'Tools')
+scriptsdir = os.path.join(basepath, 'scripts')
 
 
 class ReindentTests(unittest.TestCase):
-    script = os.path.join(basepath, 'scripts', 'reindent.py')
+    script = os.path.join(scriptsdir, 'reindent.py')
 
     def test_noargs(self):
         assert_python_ok(self.script)
@@ -31,8 +37,330 @@ class ReindentTests(unittest.TestCase):
         self.assertGreater(err, b'')
 
 
+class PindentTests(unittest.TestCase):
+    script = os.path.join(scriptsdir, 'pindent.py')
+
+    def assertFileEqual(self, fn1, fn2):
+        with open(fn1) as f1, open(fn2) as f2:
+            self.assertEqual(f1.readlines(), f2.readlines())
+
+    def pindent(self, source, *args):
+        proc = subprocess.Popen(
+                (sys.executable, self.script) + args,
+                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                universal_newlines=True)
+        out, err = proc.communicate(source)
+        self.assertIsNone(err)
+        return out
+
+    def lstriplines(self, data):
+        return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n'
+
+    def test_selftest(self):
+        with temp_dir() as directory:
+            data_path = os.path.join(directory, '_test.py')
+            with open(self.script) as f:
+                closed = f.read()
+            with open(data_path, 'w') as f:
+                f.write(closed)
+
+            rc, out, err = assert_python_ok(self.script, '-d', data_path)
+            self.assertEqual(out, b'')
+            self.assertEqual(err, b'')
+            backup = data_path + '~'
+            self.assertTrue(os.path.exists(backup))
+            with open(backup) as f:
+                self.assertEqual(f.read(), closed)
+            with open(data_path) as f:
+                clean = f.read()
+            compile(clean, '_test.py', 'exec')
+            self.assertEqual(self.pindent(clean, '-c'), closed)
+            self.assertEqual(self.pindent(closed, '-d'), clean)
+
+            rc, out, err = assert_python_ok(self.script, '-c', data_path)
+            self.assertEqual(out, b'')
+            self.assertEqual(err, b'')
+            with open(backup) as f:
+                self.assertEqual(f.read(), clean)
+            with open(data_path) as f:
+                self.assertEqual(f.read(), closed)
+
+            broken = self.lstriplines(closed)
+            with open(data_path, 'w') as f:
+                f.write(broken)
+            rc, out, err = assert_python_ok(self.script, '-r', data_path)
+            self.assertEqual(out, b'')
+            self.assertEqual(err, b'')
+            with open(backup) as f:
+                self.assertEqual(f.read(), broken)
+            with open(data_path) as f:
+                indented = f.read()
+            compile(indented, '_test.py', 'exec')
+            self.assertEqual(self.pindent(broken, '-r'), indented)
+
+    def pindent_test(self, clean, closed):
+        self.assertEqual(self.pindent(clean, '-c'), closed)
+        self.assertEqual(self.pindent(closed, '-d'), clean)
+        broken = self.lstriplines(closed)
+        self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed)
+
+    def test_statements(self):
+        clean = textwrap.dedent("""\
+            if a:
+                pass
+
+            if a:
+                pass
+            else:
+                pass
+
+            if a:
+                pass
+            elif:
+                pass
+            else:
+                pass
+
+            while a:
+                break
+
+            while a:
+                break
+            else:
+                pass
+
+            for i in a:
+                break
+
+            for i in a:
+                break
+            else:
+                pass
+
+            try:
+                pass
+            finally:
+                pass
+
+            try:
+                pass
+            except TypeError:
+                pass
+            except ValueError:
+                pass
+            else:
+                pass
+
+            try:
+                pass
+            except TypeError:
+                pass
+            except ValueError:
+                pass
+            finally:
+                pass
+
+            with a:
+                pass
+
+            class A:
+                pass
+
+            def f():
+                pass
+            """)
+
+        closed = textwrap.dedent("""\
+            if a:
+                pass
+            # end if
+
+            if a:
+                pass
+            else:
+                pass
+            # end if
+
+            if a:
+                pass
+            elif:
+                pass
+            else:
+                pass
+            # end if
+
+            while a:
+                break
+            # end while
+
+            while a:
+                break
+            else:
+                pass
+            # end while
+
+            for i in a:
+                break
+            # end for
+
+            for i in a:
+                break
+            else:
+                pass
+            # end for
+
+            try:
+                pass
+            finally:
+                pass
+            # end try
+
+            try:
+                pass
+            except TypeError:
+                pass
+            except ValueError:
+                pass
+            else:
+                pass
+            # end try
+
+            try:
+                pass
+            except TypeError:
+                pass
+            except ValueError:
+                pass
+            finally:
+                pass
+            # end try
+
+            with a:
+                pass
+            # end with
+
+            class A:
+                pass
+            # end class A
+
+            def f():
+                pass
+            # end def f
+            """)
+        self.pindent_test(clean, closed)
+
+    def test_multilevel(self):
+        clean = textwrap.dedent("""\
+            def foobar(a, b):
+                if a == b:
+                    a = a+1
+                elif a < b:
+                    b = b-1
+                    if b > a: a = a-1
+                else:
+                    print 'oops!'
+            """)
+        closed = textwrap.dedent("""\
+            def foobar(a, b):
+                if a == b:
+                    a = a+1
+                elif a < b:
+                    b = b-1
+                    if b > a: a = a-1
+                    # end if
+                else:
+                    print 'oops!'
+                # end if
+            # end def foobar
+            """)
+        self.pindent_test(clean, closed)
+
+    def test_preserve_indents(self):
+        clean = textwrap.dedent("""\
+            if a:
+                     if b:
+                              pass
+            """)
+        closed = textwrap.dedent("""\
+            if a:
+                     if b:
+                              pass
+                     # end if
+            # end if
+            """)
+        self.assertEqual(self.pindent(clean, '-c'), closed)
+        self.assertEqual(self.pindent(closed, '-d'), clean)
+        broken = self.lstriplines(closed)
+        self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed)
+        clean = textwrap.dedent("""\
+            if a:
+            \tif b:
+            \t\tpass
+            """)
+        closed = textwrap.dedent("""\
+            if a:
+            \tif b:
+            \t\tpass
+            \t# end if
+            # end if
+            """)
+        self.assertEqual(self.pindent(clean, '-c'), closed)
+        self.assertEqual(self.pindent(closed, '-d'), clean)
+        broken = self.lstriplines(closed)
+        self.assertEqual(self.pindent(broken, '-r'), closed)
+
+    def test_escaped_newline(self):
+        clean = textwrap.dedent("""\
+            class\\
+            \\
+             A:
+               def\
+            \\
+            f:
+                  pass
+            """)
+        closed = textwrap.dedent("""\
+            class\\
+            \\
+             A:
+               def\
+            \\
+            f:
+                  pass
+               # end def f
+            # end class A
+            """)
+        self.assertEqual(self.pindent(clean, '-c'), closed)
+        self.assertEqual(self.pindent(closed, '-d'), clean)
+
+    def test_empty_line(self):
+        clean = textwrap.dedent("""\
+            if a:
+
+                pass
+            """)
+        closed = textwrap.dedent("""\
+            if a:
+
+                pass
+            # end if
+            """)
+        self.pindent_test(clean, closed)
+
+    def test_oneline(self):
+        clean = textwrap.dedent("""\
+            if a: pass
+            """)
+        closed = textwrap.dedent("""\
+            if a: pass
+            # end if
+            """)
+        self.pindent_test(clean, closed)
+
+
 def test_main():
-    test_support.run_unittest(ReindentTests)
+    test_support.run_unittest(*[obj for obj in globals().values()
+                                    if isinstance(obj, type)])
 
 
 if __name__ == '__main__':
diff --git a/Misc/NEWS b/Misc/NEWS
index 1f4e282d2c..94cc8ac19f 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -634,6 +634,8 @@ Extension Modules
 Tests
 -----
 
+- Issue #15539: Added regression tests for Tools/scripts/pindent.py.
+
 - Issue #15324: Fix regrtest parsing of --fromfile and --randomize options.
 
 - Issue #16618: Add more regression tests for glob.
@@ -709,6 +711,11 @@ Build
 Tools/Demos
 -----------
 
+- Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py.  Now
+  pindent.py works with a "with" statement.  pindent.py no longer produces
+  improper indentation.  pindent.py now works with continued lines broken after
+  "class" or "def" keywords and with continuations at the start of line.
+
 - Issue #13301: use ast.literal_eval() instead of eval() in Tools/i18n/msgfmt.py
   Patch by Serhiy Storchaka.
 
diff --git a/Tools/scripts/pindent.py b/Tools/scripts/pindent.py
index 7bfc4153d2..26d81b1340 100755
--- a/Tools/scripts/pindent.py
+++ b/Tools/scripts/pindent.py
@@ -79,8 +79,9 @@
 # Defaults
 STEPSIZE = 8
 TABSIZE = 8
-EXPANDTABS = 0
+EXPANDTABS = False
 
+import io
 import re
 import sys
 
@@ -89,7 +90,8 @@ next['if'] = next['elif'] = 'elif', 'else', 'end'
 next['while'] = next['for'] = 'else', 'end'
 next['try'] = 'except', 'finally'
 next['except'] = 'except', 'else', 'finally', 'end'
-next['else'] = next['finally'] = next['def'] = next['class'] = 'end'
+next['else'] = next['finally'] = next['with'] = \
+    next['def'] = next['class'] = 'end'
 next['end'] = ()
 start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'
 
@@ -105,11 +107,11 @@ class PythonIndenter:
         self.expandtabs = expandtabs
         self._write = fpo.write
         self.kwprog = re.compile(
-                r'^\s*(?P<kw>[a-z]+)'
-                r'(\s+(?P<id>[a-zA-Z_]\w*))?'
+                r'^(?:\s|\\\n)*(?P<kw>[a-z]+)'
+                r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?'
                 r'[^\w]')
         self.endprog = re.compile(
-                r'^\s*#?\s*end\s+(?P<kw>[a-z]+)'
+                r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)'
                 r'(\s+(?P<id>[a-zA-Z_]\w*))?'
                 r'[^\w]')
         self.wsprog = re.compile(r'^[ \t]*')
@@ -125,7 +127,7 @@ class PythonIndenter:
 
     def readline(self):
         line = self.fpi.readline()
-        if line: self.lineno = self.lineno + 1
+        if line: self.lineno += 1
         # end if
         return line
     # end def readline
@@ -143,27 +145,24 @@ class PythonIndenter:
             line2 = self.readline()
             if not line2: break
             # end if
-            line = line + line2
+            line += line2
         # end while
         return line
     # end def getline
 
-    def putline(self, line, indent = None):
-        if indent is None:
-            self.write(line)
-            return
-        # end if
+    def putline(self, line, indent):
         tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
-        i = 0
-        m = self.wsprog.match(line)
-        if m: i = m.end()
+        i = self.wsprog.match(line).end()
+        line = line[i:]
+        if line[:1] not in ('\n', '\r', ''):
+            line = '\t'*tabs + ' '*spaces + line
         # end if
-        self.write('\t'*tabs + ' '*spaces + line[i:])
+        self.write(line)
     # end def putline
 
     def reformat(self):
         stack = []
-        while 1:
+        while True:
             line = self.getline()
             if not line: break      # EOF
             # end if
@@ -173,10 +172,9 @@ class PythonIndenter:
                 kw2 = m.group('kw')
                 if not stack:
                     self.error('unexpected end')
-                elif stack[-1][0] != kw2:
+                elif stack.pop()[0] != kw2:
                     self.error('unmatched end')
                 # end if
-                del stack[-1:]
                 self.putline(line, len(stack))
                 continue
             # end if
@@ -208,23 +206,23 @@ class PythonIndenter:
     def delete(self):
         begin_counter = 0
         end_counter = 0
-        while 1:
+        while True:
             line = self.getline()
             if not line: break      # EOF
             # end if
             m = self.endprog.match(line)
             if m:
-                end_counter = end_counter + 1
+                end_counter += 1
                 continue
             # end if
             m = self.kwprog.match(line)
             if m:
                 kw = m.group('kw')
                 if kw in start:
-                    begin_counter = begin_counter + 1
+                    begin_counter += 1
                 # end if
             # end if
-            self.putline(line)
+            self.write(line)
         # end while
         if begin_counter - end_counter < 0:
             sys.stderr.write('Warning: input contained more end tags than expected\n')
@@ -234,17 +232,12 @@ class PythonIndenter:
     # end def delete
 
     def complete(self):
-        self.indentsize = 1
         stack = []
         todo = []
-        thisid = ''
-        current, firstkw, lastkw, topid = 0, '', '', ''
-        while 1:
+        currentws = thisid = firstkw = lastkw = topid = ''
+        while True:
             line = self.getline()
-            i = 0
-            m = self.wsprog.match(line)
-            if m: i = m.end()
-            # end if
+            i = self.wsprog.match(line).end()
             m = self.endprog.match(line)
             if m:
                 thiskw = 'end'
@@ -269,7 +262,9 @@ class PythonIndenter:
                     thiskw = ''
                 # end if
             # end if
-            indent = len(line[:i].expandtabs(self.tabsize))
+            indentws = line[:i]
+            indent = len(indentws.expandtabs(self.tabsize))
+            current = len(currentws.expandtabs(self.tabsize))
             while indent < current:
                 if firstkw:
                     if topid:
@@ -278,11 +273,11 @@ class PythonIndenter:
                     else:
                         s = '# end %s\n' % firstkw
                     # end if
-                    self.putline(s, current)
+                    self.write(currentws + s)
                     firstkw = lastkw = ''
                 # end if
-                current, firstkw, lastkw, topid = stack[-1]
-                del stack[-1]
+                currentws, firstkw, lastkw, topid = stack.pop()
+                current = len(currentws.expandtabs(self.tabsize))
             # end while
             if indent == current and firstkw:
                 if thiskw == 'end':
@@ -297,18 +292,18 @@ class PythonIndenter:
                     else:
                         s = '# end %s\n' % firstkw
                     # end if
-                    self.putline(s, current)
+                    self.write(currentws + s)
                     firstkw = lastkw = topid = ''
                 # end if
             # end if
             if indent > current:
-                stack.append((current, firstkw, lastkw, topid))
+                stack.append((currentws, firstkw, lastkw, topid))
                 if thiskw and thiskw not in start:
                     # error
                     thiskw = ''
                 # end if
-                current, firstkw, lastkw, topid = \
-                         indent, thiskw, thiskw, thisid
+                currentws, firstkw, lastkw, topid = \
+                          indentws, thiskw, thiskw, thisid
             # end if
             if thiskw:
                 if thiskw in start:
@@ -326,7 +321,6 @@ class PythonIndenter:
             self.write(line)
         # end while
     # end def complete
-
 # end class PythonIndenter
 
 # Simplified user interface
@@ -352,76 +346,34 @@ def reformat_filter(input = sys.stdin, output = sys.stdout,
     pi.reformat()
 # end def reformat_filter
 
-class StringReader:
-    def __init__(self, buf):
-        self.buf = buf
-        self.pos = 0
-        self.len = len(self.buf)
-    # end def __init__
-    def read(self, n = 0):
-        if n <= 0:
-            n = self.len - self.pos
-        else:
-            n = min(n, self.len - self.pos)
-        # end if
-        r = self.buf[self.pos : self.pos + n]
-        self.pos = self.pos + n
-        return r
-    # end def read
-    def readline(self):
-        i = self.buf.find('\n', self.pos)
-        return self.read(i + 1 - self.pos)
-    # end def readline
-    def readlines(self):
-        lines = []
-        line = self.readline()
-        while line:
-            lines.append(line)
-            line = self.readline()
-        # end while
-        return lines
-    # end def readlines
-    # seek/tell etc. are left as an exercise for the reader
-# end class StringReader
-
-class StringWriter:
-    def __init__(self):
-        self.buf = ''
-    # end def __init__
-    def write(self, s):
-        self.buf = self.buf + s
-    # end def write
-    def getvalue(self):
-        return self.buf
-    # end def getvalue
-# end class StringWriter
-
 def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
-    input = StringReader(source)
-    output = StringWriter()
+    input = io.BytesIO(source)
+    output = io.BytesIO()
     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
     pi.complete()
     return output.getvalue()
 # end def complete_string
 
 def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
-    input = StringReader(source)
-    output = StringWriter()
+    input = io.BytesIO(source)
+    output = io.BytesIO()
     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
     pi.delete()
     return output.getvalue()
 # end def delete_string
 
 def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
-    input = StringReader(source)
-    output = StringWriter()
+    input = io.BytesIO(source)
+    output = io.BytesIO()
     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
     pi.reformat()
     return output.getvalue()
 # end def reformat_string
 
 def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
-    source = open(filename, 'r').read()
+    with open(filename, 'r') as f:
+        source = f.read()
+    # end with
     result = complete_string(source, stepsize, tabsize, expandtabs)
     if source == result: return 0
     # end if
@@ -429,14 +381,16 @@ def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs =
     try: os.rename(filename, filename + '~')
     except os.error: pass
     # end try
-    f = open(filename, 'w')
-    f.write(result)
-    f.close()
+    with open(filename, 'w') as f:
+        f.write(result)
+    # end with
     return 1
 # end def complete_file
 
 def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
-    source = open(filename, 'r').read()
+    with open(filename, 'r') as f:
+        source = f.read()
+    # end with
     result = delete_string(source, stepsize, tabsize, expandtabs)
     if source == result: return 0
     # end if
@@ -444,14 +398,16 @@ def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = E
     try: os.rename(filename, filename + '~')
     except os.error: pass
     # end try
-    f = open(filename, 'w')
-    f.write(result)
-    f.close()
+    with open(filename, 'w') as f:
+        f.write(result)
+    # end with
     return 1
 # end def delete_file
 
 def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
-    source = open(filename, 'r').read()
+    with open(filename, 'r') as f:
+        source = f.read()
+    # end with
     result = reformat_string(source, stepsize, tabsize, expandtabs)
     if source == result: return 0
     # end if
@@ -459,9 +415,9 @@ def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs =
     try: os.rename(filename, filename + '~')
     except os.error: pass
     # end try
-    f = open(filename, 'w')
-    f.write(result)
-    f.close()
+    with open(filename, 'w') as f:
+        f.write(result)
+    # end with
     return 1
 # end def reformat_file
 
@@ -474,7 +430,7 @@ usage: pindent (-c|-d|-r) [-s stepsize] [-t tabsize] [-e] [file] ...
 -r         : reformat a completed program (use #end directives)
 -s stepsize: indentation step (default %(STEPSIZE)d)
 -t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
--e         : expand TABs into spaces (defailt OFF)
+-e         : expand TABs into spaces (default OFF)
 [file] ... : files are changed in place, with backups in file~
 If no files are specified or a single - is given,
 the program acts as a filter (reads stdin, writes stdout).
@@ -517,7 +473,7 @@ def test():
         elif o == '-t':
             tabsize = int(a)
         elif o == '-e':
-            expandtabs = 1
+            expandtabs = True
         # end if
     # end for
     if not action: