merge heads

18 files changed, 176 insertions, 37 deletions

diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index f27b2ff1c0..df259109c6 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -598,6 +598,17 @@ The module :mod:`curses` defines the following functions:
       Only one *ch* can be pushed before :meth:`getch` is called.
 
 
+.. function:: unget_wch(ch)
+
+   Push *ch* so the next :meth:`get_wch` will return it.
+
+   .. note::
+
+      Only one *ch* can be pushed before :meth:`get_wch` is called.
+
+   .. versionadded:: 3.3
+
+
 .. function:: ungetmouse(id, x, y, z, bstate)
 
    Push a :const:`KEY_MOUSE` event onto the input queue, associating the given
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index 7c5ce05aa9..f68efede44 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -571,9 +571,9 @@ For an example of the usage of queues for interprocess communication see
       Return ``True`` if the queue is full, ``False`` otherwise.  Because of
       multithreading/multiprocessing semantics, this is not reliable.
 
-   .. method:: put(item[, block[, timeout]])
+   .. method:: put(obj[, block[, timeout]])
 
-      Put item into the queue.  If the optional argument *block* is ``True``
+      Put obj into the queue.  If the optional argument *block* is ``True``
       (the default) and *timeout* is ``None`` (the default), block if necessary until
       a free slot is available.  If *timeout* is a positive number, it blocks at
       most *timeout* seconds and raises the :exc:`queue.Full` exception if no
@@ -582,9 +582,9 @@ For an example of the usage of queues for interprocess communication see
       available, else raise the :exc:`queue.Full` exception (*timeout* is
       ignored in that case).
 
-   .. method:: put_nowait(item)
+   .. method:: put_nowait(obj)
 
-      Equivalent to ``put(item, False)``.
+      Equivalent to ``put(obj, False)``.
 
    .. method:: get([block[, timeout]])
 
diff --git a/Doc/packaging/setupcfg.rst b/Doc/packaging/setupcfg.rst
index ddec41a5f4..1a394b5309 100644
--- a/Doc/packaging/setupcfg.rst
+++ b/Doc/packaging/setupcfg.rst
@@ -72,7 +72,7 @@ Extending files
 ---------------
 
 A configuration file can be extended (i.e. included) by other files.  For this,
-a ``DEFAULT`` section must contain an ``extends`` key which value points to one
+a ``DEFAULT`` section must contain an ``extends`` key whose value points to one
 or more files which will be merged into the current files by adding new sections
 and fields.  If a file loaded by ``extends`` contains sections or keys that
 already exist in the original file, they will not override the previous values.
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index 68298b05e6..50efe72c1c 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -608,7 +608,7 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(
 
 /* Convert the Unicode object to a wide character string. The output string
    always ends with a nul character. If size is not NULL, write the number of
-   wide characters (including the nul character) into *size.
+   wide characters (excluding the null character) into *size.
 
    Returns a buffer allocated by PyMem_Alloc() (use PyMem_Free() to free it)
    on success. On error, returns NULL, *size is undefined and raises a
diff --git a/Lib/distutils/command/install_egg_info.py b/Lib/distutils/command/install_egg_info.py
index c8880310df..c2a7d649c0 100644
--- a/Lib/distutils/command/install_egg_info.py
+++ b/Lib/distutils/command/install_egg_info.py
@@ -40,9 +40,8 @@ class install_egg_info(Command):
                          "Creating "+self.install_dir)
         log.info("Writing %s", target)
         if not self.dry_run:
-            f = open(target, 'w')
-            self.distribution.metadata.write_pkg_file(f)
-            f.close()
+            with open(target, 'w', encoding='UTF-8') as f:
+                self.distribution.metadata.write_pkg_file(f)
 
     def get_outputs(self):
         return self.outputs
diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py
index 21ea61d96a..a9429a4296 100644
--- a/Lib/distutils/command/sdist.py
+++ b/Lib/distutils/command/sdist.py
@@ -306,7 +306,10 @@ class sdist(Command):
 
                 try:
                     self.filelist.process_template_line(line)
-                except DistutilsTemplateError as msg:
+                # the call above can raise a DistutilsTemplateError for
+                # malformed lines, or a ValueError from the lower-level
+                # convert_path function
+                except (DistutilsTemplateError, ValueError) as msg:
                     self.warn("%s, line %d: %s" % (template.filename,
                                                    template.current_line,
                                                    msg))
diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py
index 02cd79ba2f..8ca5b6f4f1 100644
--- a/Lib/distutils/dist.py
+++ b/Lib/distutils/dist.py
@@ -1010,11 +1010,9 @@ class DistributionMetadata:
     def write_pkg_info(self, base_dir):
         """Write the PKG-INFO file into the release tree.
         """
-        pkg_info = open(os.path.join(base_dir, 'PKG-INFO'), 'w')
-        try:
+        with open(os.path.join(base_dir, 'PKG-INFO'), 'w',
+                  encoding='UTF-8') as pkg_info:
             self.write_pkg_file(pkg_info)
-        finally:
-            pkg_info.close()
 
     def write_pkg_file(self, file):
         """Write the PKG-INFO format data to a file object.
diff --git a/Lib/distutils/tests/test_sdist.py b/Lib/distutils/tests/test_sdist.py
index f34f786c92..529b4ef5c6 100644
--- a/Lib/distutils/tests/test_sdist.py
+++ b/Lib/distutils/tests/test_sdist.py
@@ -15,6 +15,7 @@ from distutils.tests.test_config import PyPIRCCommandTestCase
 from distutils.errors import DistutilsOptionError
 from distutils.spawn import find_executable
 from distutils.log import WARN
+from distutils.filelist import FileList
 from distutils.archive_util import ARCHIVE_FORMATS
 
 SETUP_PY = """
@@ -78,9 +79,6 @@ class SDistTestCase(PyPIRCCommandTestCase):
         dist.include_package_data = True
         cmd = sdist(dist)
         cmd.dist_dir = 'dist'
-        def _warn(*args):
-            pass
-        cmd.warn = _warn
         return dist, cmd
 
     @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
@@ -235,7 +233,8 @@ class SDistTestCase(PyPIRCCommandTestCase):
         # with the `check` subcommand
         cmd.ensure_finalized()
         cmd.run()
-        warnings = self.get_logs(WARN)
+        warnings = [msg for msg in self.get_logs(WARN) if
+                    msg.startswith('warning: check:')]
         self.assertEqual(len(warnings), 2)
 
         # trying with a complete set of metadata
@@ -244,7 +243,8 @@ class SDistTestCase(PyPIRCCommandTestCase):
         cmd.ensure_finalized()
         cmd.metadata_check = 0
         cmd.run()
-        warnings = self.get_logs(WARN)
+        warnings = [msg for msg in self.get_logs(WARN) if
+                    msg.startswith('warning: check:')]
         self.assertEqual(len(warnings), 0)
 
     def test_check_metadata_deprecated(self):
@@ -266,7 +266,6 @@ class SDistTestCase(PyPIRCCommandTestCase):
         self.assertEqual(len(output), num_formats)
 
     def test_finalize_options(self):
-
         dist, cmd = self.get_cmd()
         cmd.finalize_options()
 
@@ -286,6 +285,32 @@ class SDistTestCase(PyPIRCCommandTestCase):
         cmd.formats = 'supazipa'
         self.assertRaises(DistutilsOptionError, cmd.finalize_options)
 
+    # the following tests make sure there is a nice error message instead
+    # of a traceback when parsing an invalid manifest template
+
+    def _test_template(self, content):
+        dist, cmd = self.get_cmd()
+        os.chdir(self.tmp_dir)
+        self.write_file('MANIFEST.in', content)
+        cmd.ensure_finalized()
+        cmd.filelist = FileList()
+        cmd.read_template()
+        warnings = self.get_logs(WARN)
+        self.assertEqual(len(warnings), 1)
+
+    def test_invalid_template_unknown_command(self):
+        self._test_template('taunt knights *')
+
+    def test_invalid_template_wrong_arguments(self):
+        # this manifest command takes one argument
+        self._test_template('prune')
+
+    @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only')
+    def test_invalid_template_wrong_path(self):
+        # on Windows, trailing slashes are not allowed
+        # this used to crash instead of raising a warning: #8286
+        self._test_template('include examples/')
+
     @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
     def test_get_file_list(self):
         # make sure MANIFEST is recalculated
diff --git a/Lib/html/parser.py b/Lib/html/parser.py
index 941228072a..a6d5be94fa 100644
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -458,4 +458,4 @@ class HTMLParser(_markupbase.ParserBase):
                     return '&'+s+';'
 
         return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));",
-                      replaceEntities, s, re.ASCII)
+                      replaceEntities, s, flags=re.ASCII)
diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py
index 5e804c27a6..c3d91aa059 100644
--- a/Lib/packaging/tests/test_util.py
+++ b/Lib/packaging/tests/test_util.py
@@ -946,7 +946,7 @@ class PackagingLibChecks(support.TempdirManager,
 
     def _distutils_pkg_info(self):
         tmp = self._distutils_setup_py_pkg()
-        self.write_file([tmp, 'PKG-INFO'], '')
+        self.write_file([tmp, 'PKG-INFO'], '', encoding='UTF-8')
         return tmp
 
     def _setup_cfg_with_no_metadata_pkg(self):
@@ -971,7 +971,7 @@ class PackagingLibChecks(support.TempdirManager,
 
     def _pkg_info_with_no_distutils(self):
         tmp = self._random_setup_py_pkg()
-        self.write_file([tmp, 'PKG-INFO'], '')
+        self.write_file([tmp, 'PKG-INFO'], '', encoding='UTF-8')
         return tmp
 
     def _random_setup_py_pkg(self):
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
index 6bdbf36e83..2560562319 100644
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -2366,17 +2366,11 @@ class TarFile(object):
             try:
                 g = grp.getgrnam(tarinfo.gname)[2]
             except KeyError:
-                try:
-                    g = grp.getgrgid(tarinfo.gid)[2]
-                except KeyError:
-                    g = os.getgid()
+                g = tarinfo.gid
             try:
                 u = pwd.getpwnam(tarinfo.uname)[2]
             except KeyError:
-                try:
-                    u = pwd.getpwuid(tarinfo.uid)[2]
-                except KeyError:
-                    u = os.getuid()
+                u = tarinfo.uid
             try:
                 if tarinfo.issym() and hasattr(os, "lchown"):
                     os.lchown(targetpath, u, g)
diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py
index c767e9388f..b4673e9863 100644
--- a/Lib/test/test_curses.py
+++ b/Lib/test/test_curses.py
@@ -264,6 +264,22 @@ def test_issue6243(stdscr):
     curses.ungetch(1025)
     stdscr.getkey()
 
+def test_unget_wch(stdscr):
+    if not hasattr(curses, 'unget_wch'):
+        return
+    ch = 'a'
+    curses.unget_wch(ch)
+    read = stdscr.get_wch()
+    read = chr(read)
+    if read != ch:
+        raise AssertionError("%r != %r" % (read, ch))
+
+    ch = ord('a')
+    curses.unget_wch(ch)
+    read = stdscr.get_wch()
+    if read != ch:
+        raise AssertionError("%r != %r" % (read, ch))
+
 def main(stdscr):
     curses.savetty()
     try:
@@ -272,6 +288,7 @@ def main(stdscr):
         test_userptr_without_set(stdscr)
         test_resize_term(stdscr)
         test_issue6243(stdscr)
+        test_unget_wch(stdscr)
     finally:
         curses.resetty()
 
diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py
index 637ab01f12..d45e45327f 100644
--- a/Lib/test/test_htmlparser.py
+++ b/Lib/test/test_htmlparser.py
@@ -377,7 +377,8 @@ class HTMLParserTolerantTestCase(TestCaseBase):
         p = html.parser.HTMLParser()
         self.assertEqual(p.unescape('&#bad;'),'&#bad;')
         self.assertEqual(p.unescape('&'),'&')
-
+        # see #12888
+        self.assertEqual(p.unescape('{ ' * 1050), '{ ' * 1050)
 
 def test_main():
     support.run_unittest(HTMLParserTestCase, HTMLParserTolerantTestCase)
diff --git a/Misc/ACKS b/Misc/ACKS
index dd9e9f65fb..fa70c72df5 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -704,6 +704,7 @@ Douglas Orr
 Michele Orrù
 Oleg Oshmyan
 Denis S. Otkidach
+Peter Otten
 Michael Otteneder
 R. M. Oudkerk
 Russel Owen
diff --git a/Misc/NEWS b/Misc/NEWS
index 2389bf54b4..85af2acf3e 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -271,6 +271,23 @@ Core and Builtins
 Library
 -------
 
+- Issue #12567: Add curses.unget_wch() function. Push a character so the next
+  get_wch() will return it.
+
+- Issue #9561: distutils and packaging now writes egg-info files using UTF-8,
+  instead of the locale encoding.
+
+- Issue #8286: The distutils command sdist will print a warning message instead
+  of crashing when an invalid path is given in the manifest template.
+
+- Issue #12841: tarfile unnecessarily checked the existence of numerical user
+  and group ids on extraction. If one of them did not exist the respective id
+  of the current user (i.e. root) was used for the file and ownership
+  information was lost.
+
+- Issue #12888: Fix a bug in HTMLParser.unescape that prevented it to escape
+  more than 128 entities.  Patch by Peter Otten.
+
 - Issue #12878: Expose a __dict__ attribute on io.IOBase and its subclasses.
 
 - Issue #12636: IDLE reads the coding cookie when executing a Python script.
diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c
index 6d720245a7..ef0a66c0ed 100644
--- a/Modules/_cursesmodule.c
+++ b/Modules/_cursesmodule.c
@@ -2696,6 +2696,71 @@ PyCurses_UngetCh(PyObject *self, PyObject *args)
     return PyCursesCheckERR(ungetch(ch), "ungetch");
 }
 
+#ifdef HAVE_NCURSESW
+/* Convert an object to a character (wchar_t):
+
+    - int
+    - str of length 1
+
+   Return 1 on success, 0 on error. */
+static int
+PyCurses_ConvertToWchar_t(PyObject *obj,
+                          wchar_t *wch)
+{
+    if (PyUnicode_Check(obj)) {
+        wchar_t buffer[2];
+        if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) {
+            PyErr_Format(PyExc_TypeError,
+                         "expect bytes or str of length 1, or int, "
+                         "got a str of length %zi",
+                         PyUnicode_GET_SIZE(obj));
+            return 0;
+        }
+        *wch = buffer[0];
+        return 2;
+    }
+    else if (PyLong_CheckExact(obj)) {
+        long value;
+        int overflow;
+        value = PyLong_AsLongAndOverflow(obj, &overflow);
+        if (overflow) {
+            PyErr_SetString(PyExc_OverflowError,
+                            "int doesn't fit in long");
+            return 0;
+        }
+        *wch = (wchar_t)value;
+        if ((long)*wch != value) {
+            PyErr_Format(PyExc_OverflowError,
+                         "character doesn't fit in wchar_t");
+            return 0;
+        }
+        return 1;
+    }
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "expect bytes or str of length 1, or int, got %s",
+                     Py_TYPE(obj)->tp_name);
+        return 0;
+    }
+}
+
+static PyObject *
+PyCurses_Unget_Wch(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    wchar_t wch;
+
+    PyCursesInitialised;
+
+    if (!PyArg_ParseTuple(args,"O", &obj))
+        return NULL;
+
+    if (!PyCurses_ConvertToWchar_t(obj, &wch))
+        return NULL;
+    return PyCursesCheckERR(unget_wch(wch), "unget_wch");
+}
+#endif
+
 static PyObject *
 PyCurses_Use_Env(PyObject *self, PyObject *args)
 {
@@ -2823,6 +2888,9 @@ static PyMethodDef PyCurses_methods[] = {
     {"typeahead",           (PyCFunction)PyCurses_TypeAhead, METH_VARARGS},
     {"unctrl",              (PyCFunction)PyCurses_UnCtrl, METH_VARARGS},
     {"ungetch",             (PyCFunction)PyCurses_UngetCh, METH_VARARGS},
+#ifdef HAVE_NCURSESW
+    {"unget_wch",           (PyCFunction)PyCurses_Unget_Wch, METH_VARARGS},
+#endif
     {"use_env",             (PyCFunction)PyCurses_Use_Env, METH_VARARGS},
 #ifndef STRICT_SYSV_CURSES
     {"use_default_colors",  (PyCFunction)PyCurses_Use_Default_Colors, METH_NOARGS},
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 94b7b61802..174455f07e 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -1204,12 +1204,12 @@ PyUnicode_FromFormat(const char *format, ...)
 /* Helper function for PyUnicode_AsWideChar() and PyUnicode_AsWideCharString():
    convert a Unicode object to a wide character string.
 
-   - If w is NULL: return the number of wide characters (including the nul
+   - If w is NULL: return the number of wide characters (including the null
      character) required to convert the unicode object. Ignore size argument.
 
-   - Otherwise: return the number of wide characters (excluding the nul
+   - Otherwise: return the number of wide characters (excluding the null
      character) written into w. Write at most size wide characters (including
-     the nul character). */
+     the null character). */
 static Py_ssize_t
 unicode_aswidechar(PyUnicodeObject *unicode,
                    wchar_t *w,
@@ -1257,7 +1257,7 @@ unicode_aswidechar(PyUnicodeObject *unicode,
         return w - worig;
     }
     else {
-        nchar = 1; /* nul character at the end */
+        nchar = 1; /* null character at the end */
         while (u != uend) {
             if (0xD800 <= u[0] && u[0] <= 0xDBFF
                 && 0xDC00 <= u[1] && u[1] <= 0xDFFF)
@@ -1295,7 +1295,7 @@ unicode_aswidechar(PyUnicodeObject *unicode,
         return w - worig;
     }
     else {
-        nchar = 1; /* nul character */
+        nchar = 1; /* null character */
         while (u != uend) {
             if (*u > 0xffff)
                 nchar += 2;
diff --git a/Tools/msi/uuids.py b/Tools/msi/uuids.py
index b85947ab56..b401b29074 100644
--- a/Tools/msi/uuids.py
+++ b/Tools/msi/uuids.py
@@ -30,4 +30,9 @@ product_codes = {
     '3.2.1121':'{4f90de4a-83dd-4443-b625-ca130ff361dd}', # 3.2.1rc1
     '3.2.1122':'{dc5eb04d-ff8a-4bed-8f96-23942fd59e5f}', # 3.2.1rc2
     '3.2.1150':'{34b2530c-6349-4292-9dc3-60bda4aed93c}', # 3.2.1
+    '3.2.2121':'{DFB29A53-ACC4-44e6-85A6-D0DA26FE8E4E}', # 3.2.2rc1
+    '3.2.2150':'{4CDE3168-D060-4b7c-BC74-4D8F9BB01AFD}', # 3.2.2
+    '3.2.3121':'{B8E8CFF7-E4C6-4a7c-9F06-BB3A8B75DDA8}', # 3.2.3rc1
+    '3.2.3150':'{789C9644-9F82-44d3-B4CA-AC31F46F5882}', # 3.2.3
+
 }