Merged revisions 60094-60123 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

*** NOTE ***
I haven't merged the files in Doc/c-api/. I got too many conflicts. Georg,
please split them manually.

........
  r60095 | andrew.kuchling | 2008-01-19 21:12:04 +0100 (Sat, 19 Jan 2008) | 2 lines

  Bug 1277: make Maildir use the user-provided factory instead of hard-wiring MaildirMessage.
  2.5.2 bugfix candidate.
........
  r60097 | georg.brandl | 2008-01-19 21:22:13 +0100 (Sat, 19 Jan 2008) | 4 lines

  #1663329: add os.closerange() to close a range of fds,
  ignoring errors, and use this in subprocess to speed up
  subprocess creation in close_fds mode. Patch by Mike Klaas.
........
  r60099 | georg.brandl | 2008-01-19 21:40:24 +0100 (Sat, 19 Jan 2008) | 2 lines

  #1411695: clarify behavior of xml.sax.utils.[un]escape.
........
  r60101 | andrew.kuchling | 2008-01-19 21:47:59 +0100 (Sat, 19 Jan 2008) | 7 lines

  Patch #1019808 from Federico Schwindt: Return correct socket error when
  a default timeout has been set, by using getsockopt() to get the error
  condition (instead of trying another connect() call, which seems to be
  a Linuxism).

  2.5 bugfix candidate, assuming no one reports any problems with this change.
........
  r60102 | gregory.p.smith | 2008-01-19 21:49:02 +0100 (Sat, 19 Jan 2008) | 3 lines

  fix comment typos, use not arg instead of arg == "", add test coverage
  for inside of the final if needquotes: within subprocess.list2cmdline().
........
  r60103 | georg.brandl | 2008-01-19 21:53:07 +0100 (Sat, 19 Jan 2008) | 2 lines

  #1509: fix sqlite3 docstrings and docs w.r.t. cursor.fetchXXX methods.
........
  r60104 | gregory.p.smith | 2008-01-19 21:57:59 +0100 (Sat, 19 Jan 2008) | 6 lines

  Fixes issue1336 - a race condition could occur when forking if the gc
  kicked in during the critical section.  solution: disable gc during
  that section.  Patch contributed by jpa and updated by me to cover the
  race condition still existing what therve from twistedmatrix pointed
  out (already seen and fixed in twisted's own subprocess code).
........
  r60105 | gregory.p.smith | 2008-01-19 22:00:37 +0100 (Sat, 19 Jan 2008) | 2 lines

  note about r60104
........
  r60106 | andrew.kuchling | 2008-01-19 22:00:38 +0100 (Sat, 19 Jan 2008) | 1 line

  Bug 1296: restore text describing OptionGroup
........
  r60109 | georg.brandl | 2008-01-19 23:08:21 +0100 (Sat, 19 Jan 2008) | 2 lines

  Split the monstrous C API manual files in smaller parts.
........
  r60110 | georg.brandl | 2008-01-19 23:14:27 +0100 (Sat, 19 Jan 2008) | 2 lines

  Missed one big file to split up.
........
  r60111 | gregory.p.smith | 2008-01-19 23:23:56 +0100 (Sat, 19 Jan 2008) | 12 lines

  Undo an unnecessary else: and indentation that r60104 added.

  try:
    ...
  except:
    ...
    raise
  else:
    ...

  the else: is unecessary due to the blind except: with a raise.
........
  r60115 | gregory.p.smith | 2008-01-19 23:49:37 +0100 (Sat, 19 Jan 2008) | 3 lines

  Fix issue 1300: Quote command line arguments that contain a '|' character in
  subprocess.list2cmdline (windows).
........
  r60116 | gregory.p.smith | 2008-01-20 00:10:52 +0100 (Sun, 20 Jan 2008) | 3 lines

  Fixes/Accepts Patch for issue1189216 - Work properly with archives
  that have file headers past the 2**31 byte boundary.
........
  r60119 | andrew.kuchling | 2008-01-20 01:00:38 +0100 (Sun, 20 Jan 2008) | 3 lines

  Patch #1048820 from Stefan Wehr: add insert-mode editing to Textbox.
  Fix an off-by-one error I noticed.
........
  r60120 | andrew.kuchling | 2008-01-20 01:12:19 +0100 (Sun, 20 Jan 2008) | 1 line

  Add an interactive test script for exercising curses
........
  r60121 | gregory.p.smith | 2008-01-20 02:21:03 +0100 (Sun, 20 Jan 2008) | 7 lines

  Fix zipfile decryption.  The check for validity only worked on one
  type of encrypted zip files.  Files using extended local headers
  needed to compare the check byte against different values.  (according
  to reading the infozip unzip crypt.c source code)

  Fixes issue1003.
........
  r60122 | gregory.p.smith | 2008-01-20 02:26:04 +0100 (Sun, 20 Jan 2008) | 2 lines

  note for r60121
........
  r60123 | gregory.p.smith | 2008-01-20 02:32:00 +0100 (Sun, 20 Jan 2008) | 4 lines

  Document that zipfile decryption is insanely slow and fix a typo and
  blatant lie in a docstring (it is not useful for security regardless of
  how you spell it).
........

9 files changed, 153 insertions, 30 deletions

diff --git a/Lib/curses/textpad.py b/Lib/curses/textpad.py
index 24a964a1c7..a05b82bdb6 100644
--- a/Lib/curses/textpad.py
+++ b/Lib/curses/textpad.py
@@ -39,8 +39,9 @@ class Textbox:
     KEY_LEFT = Ctrl-B, KEY_RIGHT = Ctrl-F, KEY_UP = Ctrl-P, KEY_DOWN = Ctrl-N
     KEY_BACKSPACE = Ctrl-h
     """
-    def __init__(self, win):
+    def __init__(self, win, insert_mode=False):
         self.win = win
+        self.insert_mode = insert_mode
         (self.maxy, self.maxx) = win.getmaxyx()
         self.maxy = self.maxy - 1
         self.maxx = self.maxx - 1
@@ -49,9 +50,10 @@ class Textbox:
         win.keypad(1)
 
     def _end_of_line(self, y):
-        "Go to the location of the first blank on the given line."
+        """Go to the location of the first blank on the given line,
+        returning the index of the last non-blank character."""
         last = self.maxx
-        while 1:
+        while True:
             if ascii.ascii(self.win.inch(y, last)) != ascii.SP:
                 last = min(self.maxx, last+1)
                 break
@@ -60,19 +62,31 @@ class Textbox:
             last = last - 1
         return last
 
+    def _insert_printable_char(self, ch):
+        (y, x) = self.win.getyx()
+        if y < self.maxy or x < self.maxx:
+            if self.insert_mode:
+                oldch = self.win.inch()
+            # The try-catch ignores the error we trigger from some curses
+            # versions by trying to write into the lowest-rightmost spot
+            # in the window.
+            try:
+                self.win.addch(ch)
+            except curses.error:
+                pass
+            if self.insert_mode:
+                (backy, backx) = self.win.getyx()
+                if ascii.isprint(oldch):
+                    self._insert_printable_char(oldch)
+                    self.win.move(backy, backx)
+
     def do_command(self, ch):
         "Process a single editing command."
         (y, x) = self.win.getyx()
         self.lastcmd = ch
         if ascii.isprint(ch):
             if y < self.maxy or x < self.maxx:
-                # The try-catch ignores the error we trigger from some curses
-                # versions by trying to write into the lowest-rightmost spot
-                # in the window.
-                try:
-                    self.win.addch(ch)
-                except curses.error:
-                    pass
+                self._insert_printable_char(ch)
         elif ch == ascii.SOH:                           # ^a
             self.win.move(y, 0)
         elif ch in (ascii.STX,curses.KEY_LEFT, ascii.BS,curses.KEY_BACKSPACE):
@@ -139,7 +153,7 @@ class Textbox:
             if stop == 0 and self.stripspaces:
                 continue
             for x in range(self.maxx+1):
-                if self.stripspaces and x == stop:
+                if self.stripspaces and x > stop:
                     break
                 result = result + chr(ascii.ascii(self.win.inch(y, x)))
             if self.maxy > 0:
diff --git a/Lib/mailbox.py b/Lib/mailbox.py
index 13e3eb7efa..d2db53fa69 100755
--- a/Lib/mailbox.py
+++ b/Lib/mailbox.py
@@ -313,7 +313,10 @@ class Maildir(Mailbox):
         subpath = self._lookup(key)
         f = open(os.path.join(self._path, subpath), 'r')
         try:
-            msg = MaildirMessage(f)
+            if self._factory:
+                msg = self._factory(f)
+            else:
+                msg = MaildirMessage(f)
         finally:
             f.close()
         subdir, name = os.path.split(subpath)
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 56e05a3774..5aee34df54 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -289,6 +289,7 @@ mswindows = (sys.platform == "win32")
 import io
 import os
 import traceback
+import gc
 
 # Exception classes used by this module.
 class CalledProcessError(Exception):
@@ -397,8 +398,8 @@ def list2cmdline(seq):
 
     2) A string surrounded by double quotation marks is
        interpreted as a single argument, regardless of white space
-       contained within.  A quoted string can be embedded in an
-       argument.
+       or pipe characters contained within.  A quoted string can be
+       embedded in an argument.
 
     3) A double quotation mark preceded by a backslash is
        interpreted as a literal double quotation mark.
@@ -424,7 +425,7 @@ def list2cmdline(seq):
         if result:
             result.append(' ')
 
-        needquote = (" " in arg) or ("\t" in arg) or arg == ""
+        needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or not arg
         if needquote:
             result.append('"')
 
@@ -433,7 +434,7 @@ def list2cmdline(seq):
                 # Don't know if we need to double yet.
                 bs_buf.append(c)
             elif c == '"':
-                # Double backspaces.
+                # Double backslashes.
                 result.append('\\' * len(bs_buf)*2)
                 bs_buf = []
                 result.append('\\"')
@@ -444,7 +445,7 @@ def list2cmdline(seq):
                     bs_buf = []
                 result.append(c)
 
-        # Add remaining backspaces, if any.
+        # Add remaining backslashes, if any.
         if bs_buf:
             result.extend(bs_buf)
 
@@ -887,13 +888,8 @@ class Popen(object):
 
 
         def _close_fds(self, but):
-            for i in range(3, MAXFD):
-                if i == but:
-                    continue
-                try:
-                    os.close(i)
-                except:
-                    pass
+            os.closerange(3, but)
+            os.closerange(but + 1, MAXFD)
 
 
         def _execute_child(self, args, executable, preexec_fn, close_fds,
@@ -921,7 +917,16 @@ class Popen(object):
             errpipe_read, errpipe_write = os.pipe()
             self._set_cloexec_flag(errpipe_write)
 
-            self.pid = os.fork()
+            gc_was_enabled = gc.isenabled()
+            # Disable gc to avoid bug where gc -> file_dealloc ->
+            # write to stderr -> hang.  http://bugs.python.org/issue1336
+            gc.disable()
+            try:
+                self.pid = os.fork()
+            except:
+                if gc_was_enabled:
+                    gc.enable()
+                raise
             self._child_created = True
             if self.pid == 0:
                 # Child
@@ -982,6 +987,8 @@ class Popen(object):
                 os._exit(255)
 
             # Parent
+            if gc_was_enabled:
+                gc.enable()
             os.close(errpipe_write)
             if p2cread is not None and p2cwrite is not None:
                 os.close(p2cread)
diff --git a/Lib/test/curses_tests.py b/Lib/test/curses_tests.py
new file mode 100644
index 0000000000..7dedbbc937
--- /dev/null
+++ b/Lib/test/curses_tests.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+#
+# $Id: ncurses.py 36559 2004-07-18 05:56:09Z tim_one $
+#
+# Interactive test suite for the curses module.
+# This script displays various things and the user should verify whether
+# they display correctly.
+#
+
+import curses
+from curses import textpad
+
+def test_textpad(stdscr, insert_mode=False):
+    ncols, nlines = 8, 3
+    uly, ulx = 3, 2
+    if insert_mode:
+        mode = 'insert mode'
+    else:
+        mode = 'overwrite mode'
+
+    stdscr.addstr(uly-3, ulx, "Use Ctrl-G to end editing (%s)." % mode)
+    stdscr.addstr(uly-2, ulx, "Be sure to try typing in the lower-right corner.")
+    win = curses.newwin(nlines, ncols, uly, ulx)
+    textpad.rectangle(stdscr, uly-1, ulx-1, uly + nlines, ulx + ncols)
+    stdscr.refresh()
+
+    box = textpad.Textbox(win, insert_mode)
+    contents = box.edit()
+    stdscr.addstr(uly+ncols+2, 0, "Text entered in the box\n")
+    stdscr.addstr(repr(contents))
+    stdscr.addstr('\n')
+    stdscr.addstr('Press any key')
+    stdscr.getch()
+
+    for i in range(3):
+        stdscr.move(uly+ncols+2 + i, 0)
+        stdscr.clrtoeol()
+
+def main(stdscr):
+    stdscr.clear()
+    test_textpad(stdscr, False)
+    test_textpad(stdscr, True)
+
+
+if __name__ == '__main__':
+    curses.wrapper(main)
diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py
index a7142552bb..918116a504 100644
--- a/Lib/test/test_mailbox.py
+++ b/Lib/test/test_mailbox.py
@@ -506,6 +506,20 @@ class TestMaildir(TestMailbox):
         self.assertEqual(msg_returned.get_flags(), 'S')
         self.assertEqual(msg_returned.get_payload(), '3')
 
+    def test_consistent_factory(self):
+        # Add a message.
+        msg = mailbox.MaildirMessage(self._template % 0)
+        msg.set_subdir('cur')
+        msg.set_flags('RF')
+        key = self._box.add(msg)
+
+        # Create new mailbox with
+        class FakeMessage(mailbox.MaildirMessage):
+            pass
+        box = mailbox.Maildir(self._path, factory=FakeMessage)
+        msg2 = box.get_message(key)
+        self.assert_(isinstance(msg2, FakeMessage))
+
     def test_initialize_new(self):
         # Initialize a non-existent mailbox
         self.tearDown()
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 15ed5578f9..6cabb82f55 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -20,6 +20,11 @@ class FileTests(unittest.TestCase):
         os.close(f)
         self.assert_(os.access(test_support.TESTFN, os.W_OK))
 
+    def test_closerange(self):
+        f = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
+        # close a fd that is open, and one that isn't
+        os.closerange(f, f+2)
+        self.assertRaises(OSError, os.write, f, "a")
 
 # Test attributes on return values from os.*stat* family.
 class StatAttributeTests(unittest.TestCase):
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index a46bc382f8..e6acce3c6c 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -413,6 +413,8 @@ class ProcessTestCase(unittest.TestCase):
                          '"a b c" d e')
         self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']),
                          'ab\\"c \\ d')
+        self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']),
+                         'ab\\"c " \\\\" d')
         self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']),
                          'a\\\\\\b "de fg" h')
         self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']),
@@ -423,6 +425,8 @@ class ProcessTestCase(unittest.TestCase):
                          '"a\\\\b\\ c" d e')
         self.assertEqual(subprocess.list2cmdline(['ab', '']),
                          'ab ""')
+        self.assertEqual(subprocess.list2cmdline(['echo', 'foo|bar']),
+                         'echo "foo|bar"')
 
 
     def test_poll(self):
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 7e401da80b..1b14d2aaa5 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -686,31 +686,52 @@ class DecryptionTests(unittest.TestCase):
     b'\x1a\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x01\x00 \x00\xb6\x81'
     b'\x00\x00\x00\x00test.txtPK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x006\x00'
     b'\x00\x00L\x00\x00\x00\x00\x00' )
+    data2 = (
+    b'PK\x03\x04\x14\x00\t\x00\x08\x00\xcf}38xu\xaa\xb2\x14\x00\x00\x00\x00\x02'
+    b'\x00\x00\x04\x00\x15\x00zeroUT\t\x00\x03\xd6\x8b\x92G\xda\x8b\x92GUx\x04'
+    b'\x00\xe8\x03\xe8\x03\xc7<M\xb5a\xceX\xa3Y&\x8b{oE\xd7\x9d\x8c\x98\x02\xc0'
+    b'PK\x07\x08xu\xaa\xb2\x14\x00\x00\x00\x00\x02\x00\x00PK\x01\x02\x17\x03'
+    b'\x14\x00\t\x00\x08\x00\xcf}38xu\xaa\xb2\x14\x00\x00\x00\x00\x02\x00\x00'
+    b'\x04\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00ze'
+    b'roUT\x05\x00\x03\xd6\x8b\x92GUx\x00\x00PK\x05\x06\x00\x00\x00\x00\x01'
+    b'\x00\x01\x00?\x00\x00\x00[\x00\x00\x00\x00\x00' )
 
     plain = b'zipfile.py encryption test'
+    plain2 = b'\x00'*512
 
     def setUp(self):
         fp = open(TESTFN, "wb")
         fp.write(self.data)
         fp.close()
         self.zip = zipfile.ZipFile(TESTFN, "r")
+        fp = open(TESTFN2, "wb")
+        fp.write(self.data2)
+        fp.close()
+        self.zip2 = zipfile.ZipFile(TESTFN2, "r")
 
     def tearDown(self):
         self.zip.close()
         os.unlink(TESTFN)
+        self.zip2.close()
+        os.unlink(TESTFN2)
 
     def testNoPassword(self):
         # Reading the encrypted file without password
         # must generate a RunTime exception
         self.assertRaises(RuntimeError, self.zip.read, "test.txt")
+        self.assertRaises(RuntimeError, self.zip2.read, "zero")
 
     def testBadPassword(self):
         self.zip.setpassword(b"perl")
         self.assertRaises(RuntimeError, self.zip.read, "test.txt")
+        self.zip2.setpassword(b"perl")
+        self.assertRaises(RuntimeError, self.zip2.read, "zero")
 
     def testGoodPassword(self):
         self.zip.setpassword(b"python")
         self.assertEquals(self.zip.read("test.txt"), self.plain)
+        self.zip2.setpassword(b"12345")
+        self.assertEquals(self.zip2.read("zero"), self.plain2)
 
 
 class TestsWithRandomBinaryFiles(unittest.TestCase):
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index 9f98728496..2865c0a2c1 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -34,9 +34,9 @@ ZIP_DEFLATED = 8
 # Other ZIP compression methods not supported
 
 # Here are some struct module formats for reading headers
-structEndArchive = "<4s4H2lH"     # 9 items, end of archive, 22 bytes
+structEndArchive = "<4s4H2LH"     # 9 items, end of archive, 22 bytes
 stringEndArchive = b"PK\005\006"   # magic number for end of archive record
-structCentralDir = "<4s4B4HlLL5HLl"# 19 items, central directory, 46 bytes
+structCentralDir = "<4s4B4HlLL5HLL"# 19 items, central directory, 46 bytes
 stringCentralDir = b"PK\001\002"   # magic number for central directory
 structFileHeader = "<4s2B4HlLL2H"  # 12 items, file header record, 30 bytes
 stringFileHeader = b"PK\003\004"   # magic number for file header
@@ -188,6 +188,7 @@ class ZipInfo (object):
             'CRC',
             'compress_size',
             'file_size',
+            '_raw_time',
         )
 
     def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)):
@@ -303,7 +304,7 @@ class _ZipDecrypter:
 
     ZIP supports a password-based form of encryption. Even though known
     plaintext attacks have been found against it, it is still useful
-    for low-level securicy.
+    to be able to get data out of such a file.
 
     Usage:
         zd = _ZipDecrypter(mypwd)
@@ -690,6 +691,7 @@ class ZipFile:
                 x.CRC, x.compress_size, x.file_size) = centdir[1:12]
             x.volume, x.internal_attr, x.external_attr = centdir[15:18]
             # Convert date/time code to (year, month, day, hour, min, sec)
+            x._raw_time = t
             x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F,
                                      t>>11, (t>>5)&0x3F, (t&0x1F) * 2 )
 
@@ -800,11 +802,18 @@ class ZipFile:
             # The first 12 bytes in the cypher stream is an encryption header
             #  used to strengthen the algorithm. The first 11 bytes are
             #  completely random, while the 12th contains the MSB of the CRC,
+            #  or the MSB of the file time depending on the header type
             #  and is used to check the correctness of the password.
             bytes = zef_file.read(12)
             h = list(map(zd, bytes[0:12]))
-            if h[11] != ((zinfo.CRC>>24) & 255):
-                raise RuntimeError("Bad password for file %s" % name)
+            if zinfo.flag_bits & 0x8:
+                # compare against the file type from extended local headers
+                check_byte = (zinfo._raw_time >> 8) & 0xff
+            else:
+                # compare against the CRC otherwise
+                check_byte = (zinfo.CRC >> 24) & 0xff
+            if h[11] != check_byte:
+                raise RuntimeError("Bad password for file", name)
 
         # build and return a ZipExtFile
         if zd is None: