diff options
145 files changed, 2021 insertions, 987 deletions
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index b4efbf00f7..58c0f3d94d 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -513,7 +513,7 @@ Building values ``None`` is returned. ``y`` (:class:`bytes`) [char \*] - This converts a C string to a Python :func:`bytes` object. If the C + This converts a C string to a Python :class:`bytes` object. If the C string pointer is *NULL*, ``None`` is returned. ``y#`` (:class:`bytes`) [char \*, int] diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index cc2cbda0ae..da2c933207 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -181,8 +181,8 @@ at http://docs.python.org/. PDF, plain text, and downloadable HTML versions are also available at http://docs.python.org/download.html. The documentation is written in reStructuredText and processed by `the Sphinx -documentation tool <http://sphinx.pocoo.org/>`__. The reStructuredText source -for the documentation is part of the Python source distribution. +documentation tool <http://sphinx-doc.org/>`__. The reStructuredText source for +the documentation is part of the Python source distribution. I've never programmed before. Is there a Python tutorial? @@ -269,9 +269,10 @@ Where in the world is www.python.org located? --------------------------------------------- The Python project's infrastructure is located all over the world. -www.python.org is currently in Amsterdam, graciously hosted by `XS4ALL -<http://www.xs4all.nl>`_. `Upfront Systems <http://www.upfrontsystems.co.za>`_ -hosts bugs.python.org. Most other Python services like `PyPI +`www.python.org <http://www.python.org>`_ is currently in Amsterdam, graciously +hosted by `XS4ALL <http://www.xs4all.nl>`_. `Upfront Systems +<http://www.upfrontsystems.co.za>`_ hosts `bugs.python.org +<http://bugs.python.org>`_. Most other Python services like `PyPI <https://pypi.python.org>`_ and hg.python.org are hosted by `Oregon State University Open Source Lab <https://osuosl.org>`_. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 674695b059..563da9ddee 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1694,6 +1694,9 @@ When the above script is run, it prints:: Note that the order of items might be different according to the version of Python used. + +.. _custom-handlers: + .. currentmodule:: logging.config Customizing handlers with :func:`dictConfig` @@ -1948,3 +1951,87 @@ handler. So the only slightly unusual thing which might trip you up is that the parentheses go around the format string and the arguments, not just the format string. That’s because the __ notation is just syntax sugar for a constructor call to one of the ``XXXMessage`` classes shown above. + + +.. _filters-dictconfig: + +.. currentmodule:: logging.config + +Configuring filters with :func:`dictConfig` +------------------------------------------- + +You *can* configure filters using :func:`~logging.config.dictConfig`, though it +might not be obvious at first glance how to do it (hence this recipe). Since +:class:`~logging.Filter` is the only filter class included in the standard +library, and it is unlikely to cater to many requirements (it's only there as a +base class), you will typically need to define your own :class:`~logging.Filter` +subclass with an overridden :meth:`~logging.Filter.filter` method. To do this, +specify the ``()`` key in the configuration dictionary for the filter, +specifying a callable which will be used to create the filter (a class is the +most obvious, but you can provide any callable which returns a +:class:`~logging.Filter` instance). Here is a complete example:: + + import logging + import logging.config + import sys + + class MyFilter(logging.Filter): + def __init__(self, param=None): + self.param = param + + def filter(self, record): + if self.param is None: + allow = True + else: + allow = self.param not in record.msg + if allow: + record.msg = 'changed: ' + record.msg + return allow + + LOGGING = { + 'version': 1, + 'filters': { + 'myfilter': { + '()': MyFilter, + 'param': 'noshow', + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'filters': ['myfilter'] + } + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['console'] + }, + } + + if __name__ == '__main__': + logging.config.dictConfig(LOGGING) + logging.debug('hello') + logging.debug('hello - noshow') + +This example shows how you can pass configuration data to the callable which +constructs the instance, in the form of keyword parameters. When run, the above +script will print:: + + changed: hello + +which shows that the filter is working as configured. + +A couple of extra points to note: + +* If you can't refer to the callable directly in the configuration (e.g. if it + lives in a different module, and you can't import it directly where the + configuration dictionary is), you can use the form ``ext://...`` as described + in :ref:`logging-config-dict-externalobj`. For example, you could have used + the text ``'ext://__main__.MyFilter'`` instead of ``MyFilter`` in the above + example. + +* As well as for filters, this technique can also be used to configure custom + handlers and formatters. See :ref:`logging-config-dict-userdef` for more + information on how logging supports using user-defined objects in its + configuration, and see the other cookbook recipe :ref:`custom-handlers` above. + diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst index 55c61d7e68..adbd3be758 100644 --- a/Doc/library/asynchat.rst +++ b/Doc/library/asynchat.rst @@ -56,8 +56,8 @@ connection requests. have only one method, :meth:`more`, which should return data to be transmitted on the channel. The producer indicates exhaustion (*i.e.* that it contains no more data) by - having its :meth:`more` method return the empty string. At this point the - :class:`async_chat` object removes the producer from the fifo and starts + having its :meth:`more` method return the empty bytes object. At this point + the :class:`async_chat` object removes the producer from the fifo and starts using the next producer, if any. When the producer fifo is empty the :meth:`handle_write` method does nothing. You use the channel object's :meth:`set_terminator` method to describe how to recognize the end of, or @@ -221,7 +221,7 @@ any extraneous data sent by the web client are ignored. :: def found_terminator(self): if self.reading_headers: self.reading_headers = False - self.parse_headers("".join(self.ibuffer)) + self.parse_headers(b"".join(self.ibuffer)) self.ibuffer = [] if self.op.upper() == b"POST": clen = self.headers.getheader("content-length") diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst index 8f494d0e58..1521e72533 100644 --- a/Doc/library/asyncore.rst +++ b/Doc/library/asyncore.rst @@ -208,7 +208,8 @@ any that have been added to the map during asynchronous service) is closed. .. method:: recv(buffer_size) Read at most *buffer_size* bytes from the socket's remote end-point. An - empty string implies that the channel has been closed from the other end. + empty bytes object implies that the channel has been closed from the + other end. .. method:: listen(backlog) diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst index ae0a22156b..50b6979f83 100644 --- a/Doc/library/chunk.rst +++ b/Doc/library/chunk.rst @@ -113,15 +113,15 @@ instance will fail with a :exc:`EOFError` exception. Read at most *size* bytes from the chunk (less if the read hits the end of the chunk before obtaining *size* bytes). If the *size* argument is - negative or omitted, read all data until the end of the chunk. The bytes - are returned as a string object. An empty string is returned when the end - of the chunk is encountered immediately. + negative or omitted, read all data until the end of the chunk. An empty + bytes object is returned when the end of the chunk is encountered + immediately. .. method:: skip() Skip to the end of the chunk. All further calls to :meth:`read` for the - chunk will return ``''``. If you are not interested in the contents of + chunk will return ``b''``. If you are not interested in the contents of the chunk, this method should be called so that the file points to the start of the next chunk. diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index c2f92b305a..575b1ea56c 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -368,7 +368,8 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures that completed + they complete (finished or were cancelled). Any futures given by *fs* that + are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`TimeoutError` if :meth:`~iterator.__next__` is called and the result isn't available after *timeout* seconds from the diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 92339dcd85..5f740a2732 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1,8 +1,8 @@ -:mod:`importlib` -- An implementation of :keyword:`import` -========================================================== +:mod:`importlib` -- The implementation of :keyword:`import` +=========================================================== .. module:: importlib - :synopsis: An implementation of the import machinery. + :synopsis: The implementation of the import machinery. .. moduleauthor:: Brett Cannon <brett@python.org> .. sectionauthor:: Brett Cannon <brett@python.org> @@ -13,17 +13,16 @@ Introduction ------------ -The purpose of the :mod:`importlib` package is two-fold. One is to provide an +The purpose of the :mod:`importlib` package is two-fold. One is to provide the implementation of the :keyword:`import` statement (and thus, by extension, the :func:`__import__` function) in Python source code. This provides an implementation of :keyword:`import` which is portable to any Python -interpreter. This also provides a reference implementation which is easier to +interpreter. This also provides an implementation which is easier to comprehend than one implemented in a programming language other than Python. Two, the components to implement :keyword:`import` are exposed in this package, making it easier for users to create their own custom objects (known generically as an :term:`importer`) to participate in the import process. -Details on custom importers can be found in :pep:`302`. .. seealso:: @@ -53,6 +52,9 @@ Details on custom importers can be found in :pep:`302`. :pep:`366` Main module explicit relative imports + :pep:`451` + A ModuleSpec Type for the Import System + :pep:`3120` Using UTF-8 as the Default Source Encoding diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 7612544ad0..d4cf9054cb 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -435,12 +435,21 @@ function. no metadata about their arguments. -.. class:: Signature +.. class:: Signature(parameters=None, \*, return_annotation=Signature.empty) A Signature object represents the call signature of a function and its return annotation. For each parameter accepted by the function it stores a :class:`Parameter` object in its :attr:`parameters` collection. + The optional *parameters* argument is a sequence of :class:`Parameter` + objects, which is validated to check that there are no parameters with + duplicate names, and that the parameters are in the right order, i.e. + positional-only first, then positional-or-keyword, and that parameters with + defaults follow parameters without defaults. + + The optional *return_annotation* argument, can be an arbitrary Python object, + is the "return" annotation of the callable. + Signature objects are *immutable*. Use :meth:`Signature.replace` to make a modified copy. @@ -489,7 +498,7 @@ function. "(a, b) -> 'new return anno'" -.. class:: Parameter +.. class:: Parameter(name, kind, \*, default=Parameter.empty, annotation=Parameter.empty) Parameter objects are *immutable*. Instead of modifying a Parameter object, you can use :meth:`Parameter.replace` to create a modified copy. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index bd0ce67470..3d723c09db 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -844,13 +844,14 @@ Text I/O Whether line buffering is enabled. -.. class:: StringIO(initial_value='', newline=None) +.. class:: StringIO(initial_value='', newline='\\n') An in-memory stream for text I/O. The initial value of the buffer (an empty string by default) can be set by providing *initial_value*. The *newline* argument works like that of - :class:`TextIOWrapper`. The default is to do no newline translation. + :class:`TextIOWrapper`. The default is to consider only ``\n`` characters + as end of lines and to do no newline translation. :class:`StringIO` provides this method in addition to those from :class:`TextIOBase` and its parents: diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index c00b53275d..453fab5694 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -81,8 +81,9 @@ in :mod:`logging` itself) and defining handlers which are declared either in .. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True) Reads the logging configuration from a :mod:`configparser`\-format file - named *fname*. This function can be called several times from an - application, allowing an end user to select from various pre-canned + named *fname*. The format of the file should be as described in + :ref:`logging-config-fileformat`. This function can be called several times + from an application, allowing an end user to select from various pre-canned configurations (if the developer provides a mechanism to present the choices and load the chosen configuration). diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 13b6041d04..d78159d64b 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -97,12 +97,14 @@ The module defines the following user-callable items: This function creates a temporary directory using :func:`mkdtemp` (the supplied arguments are passed directly to the underlying function). The resulting object can be used as a context manager (see - :ref:`context-managers`). On completion of the context (or destruction - of the temporary directory object), the newly created temporary directory + :ref:`context-managers`). On completion of the context or destruction + of the temporary directory object the newly created temporary directory and all its contents are removed from the filesystem. - The directory name can be retrieved from the :attr:`name` attribute - of the returned object. + The directory name can be retrieved from the :attr:`name` attribute of the + returned object. When the returned object is used as a context manager, the + :attr:`name` will be assigned to the target of the :keyword:`as` clause in + the :keyword:`with` statement, if there is one. The directory can be explicitly cleaned up by calling the :func:`cleanup` method. diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index fcfd514d01..fa4b46897a 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1754,13 +1754,14 @@ Loading and running tests applications which run test suites should provide alternate implementations. By default this runner shows :exc:`DeprecationWarning`, - :exc:`PendingDeprecationWarning`, and :exc:`ImportWarning` even if they are - :ref:`ignored by default <warning-ignored>`. Deprecation warnings caused by - :ref:`deprecated unittest methods <deprecated-aliases>` are also - special-cased and, when the warning filters are ``'default'`` or ``'always'``, - they will appear only once per-module, in order to avoid too many warning - messages. This behavior can be overridden using the :option:`-Wd` or - :option:`-Wa` options and leaving *warnings* to ``None``. + :exc:`PendingDeprecationWarning`, :exc:`ResourceWarning` and + :exc:`ImportWarning` even if they are :ref:`ignored by default <warning- + ignored>`. Deprecation warnings caused by :ref:`deprecated unittest methods + <deprecated-aliases>` are also special-cased and, when the warning filters + are ``'default'`` or ``'always'``, they will appear only once per-module, in + order to avoid too many warning messages. This behavior can be overridden + using the :option:`-Wd` or :option:`-Wa` options and leaving *warnings* to + ``None``. .. versionchanged:: 3.2 Added the ``warnings`` argument. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index cf652199e3..3f2dcbb71b 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -457,7 +457,7 @@ Functions is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). Returns a list of (optionally) encoded strings containing the XML data. It does not guarantee any specific sequence, except that - ``"".join(tostringlist(element)) == tostring(element)``. + ``b"".join(tostringlist(element)) == tostring(element)``. .. versionadded:: 3.2 diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index c3a26f3c1d..f793baec9b 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -29,6 +29,7 @@ definition of the Python bindings for the DOM and SAX interfaces. The XML handling submodules are: * :mod:`xml.etree.ElementTree`: the ElementTree API, a simple and lightweight + XML processor .. diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index d9a29a8814..b178fe106a 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -197,7 +197,7 @@ Decompression objects support the following methods and attributes: .. attribute:: Decompress.unused_data A bytes object which contains any bytes past the end of the compressed data. That is, - this remains ``""`` until the last byte that contains compression data is + this remains ``b""`` until the last byte that contains compression data is available. If the whole bytestring turned out to contain compressed data, this is ``b""``, an empty bytes object. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 66358c877d..06baba0d60 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -319,27 +319,25 @@ Yield expressions yield_atom: "(" `yield_expression` ")" yield_expression: "yield" [`expression_list` | "from" `expression`] -The :keyword:`yield` expression is only used when defining a :term:`generator` -function, -and can only be used in the body of a function definition. Using a -:keyword:`yield` expression in a function definition is sufficient to cause that -definition to create a generator function instead of a normal function. +The yield expression is only used when defining a :term:`generator` function and +thus can only be used in the body of a function definition. Using a yield +expression in a function's body causes that function to be a generator. When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of a generator function. The execution starts when one of the generator's methods is called. At that -time, the execution proceeds to the first :keyword:`yield` expression, where it -is suspended again, returning the value of :token:`expression_list` to -generator's caller. By suspended we mean that all local state is retained, -including the current bindings of local variables, the instruction pointer, and -the internal evaluation stack. When the execution is resumed by calling one of -the generator's methods, the function can proceed exactly as if the -:keyword:`yield` expression was just another external call. The value of the -:keyword:`yield` expression after resuming depends on the method which resumed -the execution. If :meth:`~generator.__next__` is used (typically via either a -:keyword:`for` or the :func:`next` builtin) then the result is :const:`None`, -otherwise, if :meth:`~generator.send` is used, then the result will be the -value passed in to that method. +time, the execution proceeds to the first yield expression, where it is +suspended again, returning the value of :token:`expression_list` to generator's +caller. By suspended, we mean that all local state is retained, including the +current bindings of local variables, the instruction pointer, and the internal +evaluation stack. When the execution is resumed by calling one of the +generator's methods, the function can proceed exactly as if the yield expression +was just another external call. The value of the yield expression after +resuming depends on the method which resumed the execution. If +:meth:`~generator.__next__` is used (typically via either a :keyword:`for` or +the :func:`next` builtin) then the result is :const:`None`. Otherwise, if +:meth:`~generator.send` is used, then the result will be the value passed in to +that method. .. index:: single: coroutine @@ -349,11 +347,11 @@ suspended. The only difference is that a generator function cannot control where should the execution continue after it yields; the control is always transferred to the generator's caller. -:keyword:`yield` expressions are allowed in the :keyword:`try` clause of a -:keyword:`try` ... :keyword:`finally` construct. If the generator is not -resumed before it is finalized (by reaching a zero reference count or by being -garbage collected), the generator-iterator's :meth:`~generator.close` method -will be called, allowing any pending :keyword:`finally` clauses to execute. +yield expressions are allowed in the :keyword:`try` clause of a :keyword:`try` +... :keyword:`finally` construct. If the generator is not resumed before it is +finalized (by reaching a zero reference count or by being garbage collected), +the generator-iterator's :meth:`~generator.close` method will be called, +allowing any pending :keyword:`finally` clauses to execute. When ``yield from <expr>`` is used, it treats the supplied expression as a subiterator. All values produced by that subiterator are passed directly @@ -373,11 +371,23 @@ the yield expression. It can be either set explicitly when raising .. versionchanged:: 3.3 Added ``yield from <expr>`` to delegate control flow to a subiterator -The parentheses can be omitted when the :keyword:`yield` expression is the -sole expression on the right hand side of an assignment statement. +The parentheses may be omitted when the yield expression is the sole expression +on the right hand side of an assignment statement. -.. index:: object: generator +.. seealso:: + + :pep:`0255` - Simple Generators + The proposal for adding generators and the :keyword:`yield` statement to Python. + + :pep:`0342` - Coroutines via Enhanced Generators + The proposal to enhance the API and syntax of generators, making them + usable as simple coroutines. + :pep:`0380` - Syntax for Delegating to a Subgenerator + The proposal to introduce the :token:`yield_from` syntax, making delegation + to sub-generators easy. + +.. index:: object: generator Generator-iterator methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -395,13 +405,12 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.__next__() Starts the execution of a generator function or resumes it at the last - executed :keyword:`yield` expression. When a generator function is resumed - with a :meth:`~generator.__next__` method, the current :keyword:`yield` - expression always evaluates to :const:`None`. The execution then continues - to the next :keyword:`yield` expression, where the generator is suspended - again, and the value of the :token:`expression_list` is returned to - :meth:`next`'s caller. - If the generator exits without yielding another value, a :exc:`StopIteration` + executed yield expression. When a generator function is resumed with a + :meth:`~generator.__next__` method, the current yield expression always + evaluates to :const:`None`. The execution then continues to the next yield + expression, where the generator is suspended again, and the value of the + :token:`expression_list` is returned to :meth:`next`'s caller. If the + generator exits without yielding another value, a :exc:`StopIteration` exception is raised. This method is normally called implicitly, e.g. by a :keyword:`for` loop, or @@ -411,12 +420,12 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.send(value) Resumes the execution and "sends" a value into the generator function. The - ``value`` argument becomes the result of the current :keyword:`yield` - expression. The :meth:`send` method returns the next value yielded by the - generator, or raises :exc:`StopIteration` if the generator exits without - yielding another value. When :meth:`send` is called to start the generator, - it must be called with :const:`None` as the argument, because there is no - :keyword:`yield` expression that could receive the value. + *value* argument becomes the result of the current yield expression. The + :meth:`send` method returns the next value yielded by the generator, or + raises :exc:`StopIteration` if the generator exits without yielding another + value. When :meth:`send` is called to start the generator, it must be called + with :const:`None` as the argument, because there is no yield expression that + could receive the value. .. method:: generator.throw(type[, value[, traceback]]) @@ -478,20 +487,6 @@ For examples using ``yield from``, see :ref:`pep-380` in "What's New in Python." -.. seealso:: - - :pep:`0255` - Simple Generators - The proposal for adding generators and the :keyword:`yield` statement to Python. - - :pep:`0342` - Coroutines via Enhanced Generators - The proposal to enhance the API and syntax of generators, making them - usable as simple coroutines. - - :pep:`0380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. - - .. _primaries: Primaries diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index b9ebaaa554..40bbc391d2 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -445,53 +445,26 @@ The :keyword:`yield` statement .. productionlist:: yield_stmt: `yield_expression` -The :keyword:`yield` statement is only used when defining a generator function, -and is only used in the body of the generator function. Using a :keyword:`yield` -statement in a function definition is sufficient to cause that definition to -create a generator function instead of a normal function. - -When a generator function is called, it returns an iterator known as a generator -iterator, or more commonly, a generator. The body of the generator function is -executed by calling the :func:`next` function on the generator repeatedly until -it raises an exception. - -When a :keyword:`yield` statement is executed, the state of the generator is -frozen and the value of :token:`expression_list` is returned to :meth:`next`'s -caller. By "frozen" we mean that all local state is retained, including the -current bindings of local variables, the instruction pointer, and the internal -evaluation stack: enough information is saved so that the next time :func:`next` -is invoked, the function can proceed exactly as if the :keyword:`yield` -statement were just another external call. - -The :keyword:`yield` statement is allowed in the :keyword:`try` clause of a -:keyword:`try` ... :keyword:`finally` construct. If the generator is not -resumed before it is finalized (by reaching a zero reference count or by being -garbage collected), the generator-iterator's :meth:`close` method will be -called, allowing any pending :keyword:`finally` clauses to execute. - -When ``yield from <expr>`` is used, it treats the supplied expression as -a subiterator, producing values from it until the underlying iterator is -exhausted. - - .. versionchanged:: 3.3 - Added ``yield from <expr>`` to delegate control flow to a subiterator - -For full details of :keyword:`yield` semantics, refer to the :ref:`yieldexpr` -section. +A :keyword:`yield` statement is semantically equivalent to a :ref:`yield +expression <yieldexpr>`. The yield statement can be used to omit the parentheses +that would otherwise be required in the equivalent yield expression +statement. For example, the yield statements :: -.. seealso:: + yield <expr> + yield from <expr> - :pep:`0255` - Simple Generators - The proposal for adding generators and the :keyword:`yield` statement to Python. +are equivalent to the yield expression statements :: - :pep:`0342` - Coroutines via Enhanced Generators - The proposal to enhance the API and syntax of generators, making them - usable as simple coroutines. + (yield <expr>) + (yield from <expr>) - :pep:`0380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. +Yield expressions and statements are only used when defining a :term:`generator` +function, and are only used in the body of the generator function. Using yield +in a function definition is sufficient to cause that definition to create a +generator function instead of a normal function. +For full details of :keyword:`yield` semantics, refer to the +:ref:`yieldexpr` section. .. _raise: diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index b1456ae57f..97aea4fbcc 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -370,7 +370,7 @@ defined to allow. For example:: return False retries = retries - 1 if retries < 0: - raise IOError('refusenik user') + raise IOError('uncooperative user') print(complaint) This function can be called in several ways: @@ -756,4 +756,3 @@ extracted for you: .. [#] Actually, *call by object reference* would be a better description, since if a mutable object is passed, the caller will see any changes the callee makes to it (items inserted into a list). - diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index b44a2fea41..cda63e4928 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1767,7 +1767,7 @@ sched select ------ -Solaris and derivatives platforms have a new class :class:`select.devpoll` +Solaris and derivative platforms have a new class :class:`select.devpoll` for high performance asynchronous sockets via :file:`/dev/poll`. (Contributed by Jesús Cea Avión in :issue:`6397`.) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index aab60db4be..9a2a1aa69c 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -2044,7 +2044,7 @@ class StringIO(TextIOWrapper): def __init__(self, initial_value="", newline="\n"): super(StringIO, self).__init__(BytesIO(), encoding="utf-8", - errors="strict", + errors="surrogatepass", newline=newline) # Issue #5645: make universal newlines semantics the same as in the # C version, even under Windows. @@ -2060,7 +2060,13 @@ class StringIO(TextIOWrapper): def getvalue(self): self.flush() - return self.buffer.getvalue().decode(self._encoding, self._errors) + decoder = self._decoder or self._get_decoder() + old_state = decoder.getstate() + decoder.reset() + try: + return decoder.decode(self.buffer.getvalue(), final=True) + finally: + decoder.setstate(old_state) def __repr__(self): # TextIOWrapper tells the encoding in its repr. In StringIO, diff --git a/Lib/codecs.py b/Lib/codecs.py index 6a6eb900c7..01ae0f3ea6 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -463,15 +463,12 @@ class StreamReader(Codec): # read until we get the required number of characters (if available) while True: # can the request be satisfied from the character buffer? - if chars < 0: - if size < 0: - if self.charbuffer: - break - elif len(self.charbuffer) >= size: - break - else: + if chars >= 0: if len(self.charbuffer) >= chars: break + elif size >= 0: + if len(self.charbuffer) >= size: + break # we need more data if size < 0: newdata = self.stream.read() @@ -479,6 +476,8 @@ class StreamReader(Codec): newdata = self.stream.read(size) # decode bytes (those remaining from the last call included) data = self.bytebuffer + newdata + if not data: + break try: newchars, decodedbytes = self.decode(data, self.errors) except UnicodeDecodeError as exc: diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index ca3aebd9c7..d45a404d37 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -181,7 +181,8 @@ def as_completed(fs, timeout=None): Returns: An iterator that yields the given Futures as they complete (finished or - cancelled). + cancelled). If any given Futures are duplicated, they will be returned + once. Raises: TimeoutError: If the entire result iterator could not be generated @@ -190,11 +191,12 @@ def as_completed(fs, timeout=None): if timeout is not None: end_time = timeout + time.time() + fs = set(fs) with _AcquireFutures(fs): finished = set( f for f in fs if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]) - pending = set(fs) - finished + pending = fs - finished waiter = _create_and_install_waiters(fs, _AS_COMPLETED) try: diff --git a/Lib/configparser.py b/Lib/configparser.py index aa401fc0a3..aebf8a0fb9 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -286,7 +286,7 @@ class ParsingError(Error): raise ValueError("Required argument `source' not given.") elif filename: source = filename - Error.__init__(self, 'Source contains parsing errors: %s' % source) + Error.__init__(self, 'Source contains parsing errors: %r' % source) self.source = source self.errors = [] self.args = (source, ) @@ -322,7 +322,7 @@ class MissingSectionHeaderError(ParsingError): def __init__(self, filename, lineno, line): Error.__init__( self, - 'File contains no section headers.\nfile: %s, line: %d\n%r' % + 'File contains no section headers.\nfile: %r, line: %d\n%r' % (filename, lineno, line)) self.source = filename self.lineno = lineno diff --git a/Lib/distutils/command/register.py b/Lib/distutils/command/register.py index 55656c2353..b49f86fe58 100644 --- a/Lib/distutils/command/register.py +++ b/Lib/distutils/command/register.py @@ -300,5 +300,5 @@ Your selection [default 1]: ''', log.INFO) result = 200, 'OK' if self.show_response: dashes = '-' * 75 - self.announce('%s%s%s' % (dashes, data, dashes)) + self.announce('%s%r%s' % (dashes, data, dashes)) return result diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 291437c586..0369e01547 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1559,6 +1559,13 @@ def get_bare_quoted_string(value): while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) + elif value[:2] == '=?': + try: + token, value = get_encoded_word(value) + bare_quoted_string.defects.append(errors.InvalidHeaderDefect( + "encoded word inside quoted string")) + except errors.HeaderParseError: + token, value = get_qcontent(value) else: token, value = get_qcontent(value) bare_quoted_string.append(token) diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 4ea0b559b9..e4a86d49d8 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -12,6 +12,7 @@ import time import random import warnings +from copy import deepcopy from io import StringIO, BytesIO from email._policybase import compat32 from email.header import Header @@ -173,10 +174,18 @@ class Generator: # necessary. oldfp = self._fp try: + self._munge_cte = None self._fp = sfp = self._new_buffer() self._dispatch(msg) finally: self._fp = oldfp + munge_cte = self._munge_cte + del self._munge_cte + # If we munged the cte, copy the message again and re-fix the CTE. + if munge_cte: + msg = deepcopy(msg) + msg.replace_header('content-transfer-encoding', munge_cte[0]) + msg.replace_header('content-type', munge_cte[1]) # Write the headers. First we see if the message object wants to # handle that itself. If not, we'll do it generically. meth = getattr(msg, '_write_headers', None) @@ -225,9 +234,14 @@ class Generator: if _has_surrogates(msg._payload): charset = msg.get_param('charset') if charset is not None: + # XXX: This copy stuff is an ugly hack to avoid modifying the + # existing message. + msg = deepcopy(msg) del msg['content-transfer-encoding'] msg.set_payload(payload, charset) payload = msg.get_payload() + self._munge_cte = (msg['content-transfer-encoding'], + msg['content-type']) if self._mangle_from_: payload = fcre.sub('>From ', payload) self._write_lines(payload) @@ -285,9 +299,8 @@ class Generator: # body-part self._fp.write(body_part) # close-delimiter transport-padding - self.write(self._NL + '--' + boundary + '--') + self.write(self._NL + '--' + boundary + '--' + self._NL) if msg.epilogue is not None: - self.write(self._NL) if self._mangle_from_: epilogue = fcre.sub('>From ', msg.epilogue) else: diff --git a/Lib/email/message.py b/Lib/email/message.py index 63b51f6007..afe350c902 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -277,8 +277,6 @@ class Message: """ if hasattr(payload, 'encode'): if charset is None: - # We should check for ASCII-only here, but we can't do that - # for backward compatibility reasons. Fixed in 3.4. self._payload = payload return if not isinstance(charset, Charset): @@ -326,8 +324,9 @@ class Message: try: cte(self) except TypeError: - # This if is for backward compatibility and will be removed - # in 3.4 when the ascii check is added to set_payload. + # This 'if' is for backward compatibility, it allows unicode + # through even though that won't work correctly if the + # message is serialized. payload = self._payload if payload: try: diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 93a625c8b4..f76c21eb1b 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -337,6 +337,10 @@ def collapse_rfc2231_value(value, errors='replace', # object. We do not want bytes() normal utf-8 decoder, we want a straight # interpretation of the string as character bytes. charset, language, text = value + if charset is None: + # Issue 17369: if charset/lang is None, decode_rfc2231 couldn't parse + # the value, so use the fallback_charset. + charset = fallback_charset rawbytes = bytes(text, 'raw-unicode-escape') try: return str(rawbytes, charset, errors) diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 2d3bef351b..63fe77425b 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -228,9 +228,9 @@ class HTMLParser(_markupbase.ParserBase): i = self.updatepos(i, k) continue else: - if ";" in rawdata[i:]: #bail by consuming &# - self.handle_data(rawdata[0:2]) - i = self.updatepos(i, 2) + if ";" in rawdata[i:]: # bail by consuming &# + self.handle_data(rawdata[i:i+2]) + i = self.updatepos(i, i+2) break elif startswith('&', i): match = entityref.match(rawdata, i) diff --git a/Lib/http/server.py b/Lib/http/server.py index 7050b95352..dab1eb65c4 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -670,8 +670,10 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """Serve a GET request.""" f = self.send_head() if f: - self.copyfile(f, self.wfile) - f.close() + try: + self.copyfile(f, self.wfile) + finally: + f.close() def do_HEAD(self): """Serve a HEAD request.""" @@ -712,13 +714,17 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): except IOError: self.send_error(404, "File not found") return None - self.send_response(200) - self.send_header("Content-type", ctype) - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) - self.end_headers() - return f + try: + self.send_response(200) + self.send_header("Content-type", ctype) + fs = os.fstat(f.fileno()) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + return f + except: + f.close() + raise def list_directory(self, path): """Helper to produce a directory listing (absent index.html). diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py index 4f88da6f5d..81bd5f1890 100644 --- a/Lib/idlelib/CallTips.py +++ b/Lib/idlelib/CallTips.py @@ -5,16 +5,16 @@ parameter and docstring information when you type an opening parenthesis, and which disappear when you type a closing parenthesis. """ +import __main__ +import inspect import re import sys +import textwrap import types -import inspect from idlelib import CallTipWindow from idlelib.HyperParser import HyperParser -import __main__ - class CallTips: menudefs = [ @@ -117,8 +117,9 @@ def get_entity(expression): return None # The following are used in get_argspec and some in tests -_MAX_COLS = 79 +_MAX_COLS = 85 _MAX_LINES = 5 # enough for bytes +_INDENT = ' '*4 # for wrapped signatures _first_param = re.compile('(?<=\()\w*\,?\s*') _default_callable_argspec = "See source or doc" @@ -149,13 +150,15 @@ def get_argspec(ob): isinstance(ob_call, types.MethodType)): argspec = _first_param.sub("", argspec) + lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) + if len(argspec) > _MAX_COLS else [argspec] if argspec else []) + if isinstance(ob_call, types.MethodType): doc = ob_call.__doc__ else: doc = getattr(ob, "__doc__", "") if doc: - lines = [argspec] if argspec else [] - for line in doc.split('\n', 5)[:_MAX_LINES]: + for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: line = line.strip() if not line: break diff --git a/Lib/idlelib/Debugger.py b/Lib/idlelib/Debugger.py index ed66084e91..d4872ed42a 100644 --- a/Lib/idlelib/Debugger.py +++ b/Lib/idlelib/Debugger.py @@ -254,8 +254,7 @@ class Debugger: self.sync_source_line() def show_frame(self, stackitem): - frame, lineno = stackitem - self.frame = frame + self.frame = stackitem[0] # lineno is stackitem[1] self.show_variables() localsviewer = None diff --git a/Lib/idlelib/Icons/idle.ico b/Lib/idlelib/Icons/idle.ico Binary files differnew file mode 100644 index 0000000000..3357aef148 --- /dev/null +++ b/Lib/idlelib/Icons/idle.ico diff --git a/Lib/idlelib/Icons/idle_16.gif b/Lib/idlelib/Icons/idle_16.gif Binary files differnew file mode 100644 index 0000000000..9f001b1d79 --- /dev/null +++ b/Lib/idlelib/Icons/idle_16.gif diff --git a/Lib/idlelib/Icons/idle_16.png b/Lib/idlelib/Icons/idle_16.png Binary files differnew file mode 100644 index 0000000000..6abde0af90 --- /dev/null +++ b/Lib/idlelib/Icons/idle_16.png diff --git a/Lib/idlelib/Icons/idle_32.gif b/Lib/idlelib/Icons/idle_32.gif Binary files differnew file mode 100644 index 0000000000..af5b2d52cc --- /dev/null +++ b/Lib/idlelib/Icons/idle_32.gif diff --git a/Lib/idlelib/Icons/idle_32.png b/Lib/idlelib/Icons/idle_32.png Binary files differnew file mode 100644 index 0000000000..41b70dbc37 --- /dev/null +++ b/Lib/idlelib/Icons/idle_32.png diff --git a/Lib/idlelib/Icons/idle_48.gif b/Lib/idlelib/Icons/idle_48.gif Binary files differnew file mode 100644 index 0000000000..fc5304f31e --- /dev/null +++ b/Lib/idlelib/Icons/idle_48.gif diff --git a/Lib/idlelib/Icons/idle_48.png b/Lib/idlelib/Icons/idle_48.png Binary files differnew file mode 100644 index 0000000000..e5fa9280e2 --- /dev/null +++ b/Lib/idlelib/Icons/idle_48.png diff --git a/Lib/idlelib/Icons/python.gif b/Lib/idlelib/Icons/python.gif Binary files differindex 58271edec4..b189c2c2d2 100644 --- a/Lib/idlelib/Icons/python.gif +++ b/Lib/idlelib/Icons/python.gif diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index 55fd3574d4..2e5ebb233b 100755 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -16,7 +16,7 @@ import io import linecache from code import InteractiveInterpreter -from platform import python_version +from platform import python_version, system try: from tkinter import * @@ -1524,6 +1524,18 @@ def main(): # start editor and/or shell windows: root = Tk(className="Idle") + # set application icon + icondir = os.path.join(os.path.dirname(__file__), 'Icons') + if system() == 'Windows': + iconfile = os.path.join(icondir, 'idle.ico') + root.wm_iconbitmap(default=iconfile) + elif TkVersion >= 8.5: + ext = '.png' if TkVersion >= 8.6 else '.gif' + iconfiles = [os.path.join(icondir, 'idle_%d%s' % (size, ext)) + for size in (16, 32, 48)] + icons = [PhotoImage(file=iconfile) for iconfile in iconfiles] + root.wm_iconphoto(True, *icons) + fixwordbreaks(root) root.withdraw() flist = PyShellFileList(root) diff --git a/Lib/idlelib/SearchEngine.py b/Lib/idlelib/SearchEngine.py index bbd221dec2..9d3c4cb78a 100644 --- a/Lib/idlelib/SearchEngine.py +++ b/Lib/idlelib/SearchEngine.py @@ -83,11 +83,9 @@ class SearchEngine: try: prog = re.compile(pat, flags) except re.error as what: - try: - msg, col = what - except: - msg = str(what) - col = -1 + args = what.args + msg = args[0] + col = arg[1] if len(args) >= 2 else -1 self.report_error(pat, msg, col) return None return prog diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py index 1f4a3a5768..efe5c4326c 100644 --- a/Lib/idlelib/configDialog.py +++ b/Lib/idlelib/configDialog.py @@ -82,9 +82,10 @@ class ConfigDialog(Toplevel): else: extraKwds=dict(padx=6, pady=3) - self.buttonHelp = Button(frameActionButtons,text='Help', - command=self.Help,takefocus=FALSE, - **extraKwds) +# Comment out button creation and packing until implement self.Help +## self.buttonHelp = Button(frameActionButtons,text='Help', +## command=self.Help,takefocus=FALSE, +## **extraKwds) self.buttonOk = Button(frameActionButtons,text='Ok', command=self.Ok,takefocus=FALSE, **extraKwds) @@ -98,7 +99,7 @@ class ConfigDialog(Toplevel): self.CreatePageHighlight() self.CreatePageKeys() self.CreatePageGeneral() - self.buttonHelp.pack(side=RIGHT,padx=5) +## self.buttonHelp.pack(side=RIGHT,padx=5) self.buttonOk.pack(side=LEFT,padx=5) self.buttonApply.pack(side=LEFT,padx=5) self.buttonCancel.pack(side=LEFT,padx=5) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index eca24ec129..f3637646c4 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -1,5 +1,6 @@ import unittest import idlelib.CallTips as ct +import textwrap import types default_tip = ct._default_callable_argspec @@ -64,20 +65,35 @@ class Get_signatureTest(unittest.TestCase): gtest(types.MethodType, "method(function, instance)") gtest(SB(), default_tip) + def test_signature_wrap(self): + self.assertEqual(signature(textwrap.TextWrapper), '''\ +(width=70, initial_indent='', subsequent_indent='', expand_tabs=True, + replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, + drop_whitespace=True, break_on_hyphens=True, tabsize=8)''') + + def test_docline_truncation(self): + def f(): pass + f.__doc__ = 'a'*300 + self.assertEqual(signature(f), '()\n' + 'a' * (ct._MAX_COLS-3) + '...') + def test_multiline_docstring(self): # Test fewer lines than max. self.assertEqual(signature(list), "list() -> new empty list\n" "list(iterable) -> new list initialized from iterable's items") - # Test max lines and line (currently) too long. - self.assertEqual(signature(bytes), -"bytes(iterable_of_ints) -> bytes\n" -"bytes(string, encoding[, errors]) -> bytes\n" -"bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n" -#bytes(int) -> bytes object of size given by the parameter initialized with null bytes -"bytes(int) -> bytes object of size given by the parameter initialized with n...\n" -"bytes() -> empty bytes object") + # Test max lines + self.assertEqual(signature(bytes), '''\ +bytes(iterable_of_ints) -> bytes +bytes(string, encoding[, errors]) -> bytes +bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer +bytes(int) -> bytes object of size given by the parameter initialized with null bytes +bytes() -> empty bytes object''') + + # Test more than max lines + def f(): pass + f.__doc__ = 'a\n' * 15 + self.assertEqual(signature(f), '()' + '\na' * ct._MAX_LINES) def test_functions(self): def t1(): 'doc' @@ -106,6 +122,16 @@ class Get_signatureTest(unittest.TestCase): (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),): self.assertEqual(signature(meth), mtip + "\ndoc") + def test_starred_parameter(self): + # test that starred first parameter is *not* removed from argspec + class C: + def m1(*args): pass + def m2(**kwds): pass + c = C() + for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"), + (C.m2, "(**kwds)"), (c.m2, "(**kwds)"),): + self.assertEqual(signature(meth), mtip) + def test_non_ascii_name(self): # test that re works to delete a first parameter name that # includes non-ascii chars, such as various forms of A. diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 3f6656a762..ad5f4e9de8 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1063,6 +1063,11 @@ class IMAP4: del self.tagged_commands[tag] return result + # If we've seen a BYE at this point, the socket will be + # closed, so report the BYE now. + + self._check_bye() + # Some have reported "unexpected response" exceptions. # Note that ignoring them here causes loops. # Instead, send me details of the unexpected response and diff --git a/Lib/imghdr.py b/Lib/imghdr.py index 6ee45daab8..bdd47eea0d 100644 --- a/Lib/imghdr.py +++ b/Lib/imghdr.py @@ -7,18 +7,16 @@ __all__ = ["what"] #-------------------------# def what(file, h=None): - if h is None: - if isinstance(file, str): - f = open(file, 'rb') - h = f.read(32) - else: - location = file.tell() - h = file.read(32) - file.seek(location) - f = None - else: - f = None + f = None try: + if h is None: + if isinstance(file, str): + f = open(file, 'rb') + h = f.read(32) + else: + location = file.tell() + h = file.read(32) + file.seek(location) for tf in tests: res = tf(h, f) if res: diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index eabfd6f0a1..ecf3f4491c 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -456,8 +456,8 @@ class _IPAddressBase(_TotalOrderingMixin): raise AddressValueError(msg % (address, address_len, expected_len, self._version)) - def _ip_int_from_prefix(self, prefixlen=None): - """Turn the prefix length netmask into a int for comparison. + def _ip_int_from_prefix(self, prefixlen): + """Turn the prefix length into a bitwise netmask Args: prefixlen: An integer, the prefix length. @@ -466,36 +466,92 @@ class _IPAddressBase(_TotalOrderingMixin): An integer. """ - if prefixlen is None: - prefixlen = self._prefixlen return self._ALL_ONES ^ (self._ALL_ONES >> prefixlen) - def _prefix_from_ip_int(self, ip_int, mask=32): - """Return prefix length from the decimal netmask. + def _prefix_from_ip_int(self, ip_int): + """Return prefix length from the bitwise netmask. Args: - ip_int: An integer, the IP address. - mask: The netmask. Defaults to 32. + ip_int: An integer, the netmask in axpanded bitwise format + + Returns: + An integer, the prefix length. + + Raises: + ValueError: If the input intermingles zeroes & ones + """ + trailing_zeroes = _count_righthand_zero_bits(ip_int, + self._max_prefixlen) + prefixlen = self._max_prefixlen - trailing_zeroes + leading_ones = ip_int >> trailing_zeroes + all_ones = (1 << prefixlen) - 1 + if leading_ones != all_ones: + byteslen = self._max_prefixlen // 8 + details = ip_int.to_bytes(byteslen, 'big') + msg = 'Netmask pattern %r mixes zeroes & ones' + raise ValueError(msg % details) + return prefixlen + + def _report_invalid_netmask(self, netmask_str): + msg = '%r is not a valid netmask' % netmask_str + raise NetmaskValueError(msg) from None + + def _prefix_from_prefix_string(self, prefixlen_str): + """Return prefix length from a numeric string + + Args: + prefixlen_str: The string to be converted Returns: An integer, the prefix length. + Raises: + NetmaskValueError: If the input is not a valid netmask """ - return mask - _count_righthand_zero_bits(ip_int, mask) + # int allows a leading +/- as well as surrounding whitespace, + # so we ensure that isn't the case + if not _BaseV4._DECIMAL_DIGITS.issuperset(prefixlen_str): + self._report_invalid_netmask(prefixlen_str) + try: + prefixlen = int(prefixlen_str) + except ValueError: + self._report_invalid_netmask(prefixlen_str) + if not (0 <= prefixlen <= self._max_prefixlen): + self._report_invalid_netmask(prefixlen_str) + return prefixlen - def _ip_string_from_prefix(self, prefixlen=None): - """Turn a prefix length into a dotted decimal string. + def _prefix_from_ip_string(self, ip_str): + """Turn a netmask/hostmask string into a prefix length Args: - prefixlen: An integer, the netmask prefix length. + ip_str: The netmask/hostmask to be converted Returns: - A string, the dotted decimal netmask string. + An integer, the prefix length. + Raises: + NetmaskValueError: If the input is not a valid netmask/hostmask """ - if not prefixlen: - prefixlen = self._prefixlen - return self._string_from_ip_int(self._ip_int_from_prefix(prefixlen)) + # Parse the netmask/hostmask like an IP address. + try: + ip_int = self._ip_int_from_string(ip_str) + except AddressValueError: + self._report_invalid_netmask(ip_str) + + # Try matching a netmask (this would be /1*0*/ as a bitwise regexp). + # Note that the two ambiguous cases (all-ones and all-zeroes) are + # treated as netmasks. + try: + return self._prefix_from_ip_int(ip_int) + except ValueError: + pass + + # Invert the bits, and try matching a /0+1+/ hostmask instead. + ip_int ^= self._ALL_ONES + try: + return self._prefix_from_ip_int(ip_int) + except ValueError: + self._report_invalid_netmask(ip_str) class _BaseAddress(_IPAddressBase): @@ -504,7 +560,6 @@ class _BaseAddress(_IPAddressBase): This IP class contains the version independent methods which are used by single IP addresses. - """ def __init__(self, address): @@ -873,7 +928,7 @@ class _BaseNetwork(_IPAddressBase): raise ValueError('prefix length diff must be > 0') new_prefixlen = self._prefixlen + prefixlen_diff - if not self._is_valid_netmask(str(new_prefixlen)): + if new_prefixlen > self._max_prefixlen: raise ValueError( 'prefix length diff %d is invalid for netblock %s' % ( new_prefixlen, self)) @@ -1428,33 +1483,16 @@ class IPv4Network(_BaseV4, _BaseNetwork): self.network_address = IPv4Address(self._ip_int_from_string(addr[0])) if len(addr) == 2: - mask = addr[1].split('.') - - if len(mask) == 4: - # We have dotted decimal netmask. - if self._is_valid_netmask(addr[1]): - self.netmask = IPv4Address(self._ip_int_from_string( - addr[1])) - elif self._is_hostmask(addr[1]): - self.netmask = IPv4Address( - self._ip_int_from_string(addr[1]) ^ self._ALL_ONES) - else: - raise NetmaskValueError('%r is not a valid netmask' - % addr[1]) - - self._prefixlen = self._prefix_from_ip_int(int(self.netmask)) - else: - # We have a netmask in prefix length form. - if not self._is_valid_netmask(addr[1]): - raise NetmaskValueError('%r is not a valid netmask' - % addr[1]) - self._prefixlen = int(addr[1]) - self.netmask = IPv4Address(self._ip_int_from_prefix( - self._prefixlen)) + try: + # Check for a netmask in prefix length form + self._prefixlen = self._prefix_from_prefix_string(addr[1]) + except NetmaskValueError: + # Check for a netmask or hostmask in dotted-quad form. + # This may raise NetmaskValueError. + self._prefixlen = self._prefix_from_ip_string(addr[1]) else: self._prefixlen = self._max_prefixlen - self.netmask = IPv4Address(self._ip_int_from_prefix( - self._prefixlen)) + self.netmask = IPv4Address(self._ip_int_from_prefix(self._prefixlen)) if strict: if (IPv4Address(int(self.network_address) & int(self.netmask)) != @@ -2042,11 +2080,8 @@ class IPv6Network(_BaseV6, _BaseNetwork): self.network_address = IPv6Address(self._ip_int_from_string(addr[0])) if len(addr) == 2: - if self._is_valid_netmask(addr[1]): - self._prefixlen = int(addr[1]) - else: - raise NetmaskValueError('%r is not a valid netmask' - % addr[1]) + # This may raise NetmaskValueError + self._prefixlen = self._prefix_from_prefix_string(addr[1]) else: self._prefixlen = self._max_prefixlen @@ -2061,23 +2096,6 @@ class IPv6Network(_BaseV6, _BaseNetwork): if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ - def _is_valid_netmask(self, prefixlen): - """Verify that the netmask/prefixlen is valid. - - Args: - prefixlen: A string, the netmask in prefix length format. - - Returns: - A boolean, True if the prefix represents a valid IPv6 - netmask. - - """ - try: - prefixlen = int(prefixlen) - except ValueError: - return False - return 0 <= prefixlen <= self._max_prefixlen - @property def is_site_local(self): """Test if the address is reserved for site-local. diff --git a/Lib/mailcap.py b/Lib/mailcap.py index 99f4958bf7..0c0b19c47c 100644 --- a/Lib/mailcap.py +++ b/Lib/mailcap.py @@ -22,8 +22,8 @@ def getcaps(): fp = open(mailcap, 'r') except IOError: continue - morecaps = readmailcapfile(fp) - fp.close() + with fp: + morecaps = readmailcapfile(fp) for key, value in morecaps.items(): if not key in caps: caps[key] = value diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 3e742a755b..cdebf7a66d 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -363,9 +363,10 @@ def read_mime_types(file): f = open(file) except IOError: return None - db = MimeTypes() - db.readfp(f, True) - return db.types_map[True] + with f: + db = MimeTypes() + db.readfp(f, True) + return db.types_map[True] def _default_mime_types(): diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 2a0bc2fa72..22589d0422 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -395,13 +395,23 @@ class Connection(_ConnectionBase): return buf def _send_bytes(self, buf): - # For wire compatibility with 3.2 and lower n = len(buf) - self._send(struct.pack("!i", n)) - # The condition is necessary to avoid "broken pipe" errors - # when sending a 0-length buffer if the other end closed the pipe. - if n > 0: - self._send(buf) + # For wire compatibility with 3.2 and lower + header = struct.pack("!i", n) + if n > 16384: + # The payload is large so Nagle's algorithm won't be triggered + # and we'd better avoid the cost of concatenation. + chunks = [header, buf] + elif n > 0: + # Issue # 20540: concatenate before sending, to avoid delays due + # to Nagle's algorithm on a TCP socket. + chunks = [header + buf] + else: + # This code path is necessary to avoid "broken pipe" errors + # when sending a 0-length buffer if the other end closed the pipe. + chunks = [header] + for chunk in chunks: + self._send(chunk) def _recv_bytes(self, maxsize=None): buf = self._recv(4) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index b05436ed80..5a012bdfb2 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -104,82 +104,36 @@ def isabs(s): # Join two (or more) paths. - -def join(a, *p): - """Join two or more pathname components, inserting "\\" as needed. - If any component is an absolute path, all previous path components - will be discarded.""" - sep = _get_sep(a) - seps = _get_bothseps(a) - colon = _get_colon(a) - path = a - for b in p: - b_wins = 0 # set to 1 iff b makes path irrelevant - if not path: - b_wins = 1 - - elif isabs(b): - # This probably wipes out path so far. However, it's more - # complicated if path begins with a drive letter. You get a+b - # (minus redundant slashes) in these four cases: - # 1. join('c:', '/a') == 'c:/a' - # 2. join('//computer/share', '/a') == '//computer/share/a' - # 3. join('c:/', '/a') == 'c:/a' - # 4. join('//computer/share/', '/a') == '//computer/share/a' - # But b wins in all of these cases: - # 5. join('c:/a', '/b') == '/b' - # 6. join('//computer/share/a', '/b') == '/b' - # 7. join('c:', 'd:/') == 'd:/' - # 8. join('c:', '//computer/share/') == '//computer/share/' - # 9. join('//computer/share', 'd:/') == 'd:/' - # 10. join('//computer/share', '//computer/share/') == '//computer/share/' - # 11. join('c:/', 'd:/') == 'd:/' - # 12. join('c:/', '//computer/share/') == '//computer/share/' - # 13. join('//computer/share/', 'd:/') == 'd:/' - # 14. join('//computer/share/', '//computer/share/') == '//computer/share/' - b_prefix, b_rest = splitdrive(b) - - # if b has a prefix, it always wins. - if b_prefix: - b_wins = 1 - else: - # b doesn't have a prefix. - # but isabs(b) returned true. - # and therefore b_rest[0] must be a slash. - # (but let's check that.) - assert(b_rest and b_rest[0] in seps) - - # so, b still wins if path has a rest that's more than a sep. - # you get a+b if path_rest is empty or only has a sep. - # (see cases 1-4 for times when b loses.) - path_rest = splitdrive(path)[1] - b_wins = path_rest and path_rest not in seps - - if b_wins: - path = b - else: - # Join, and ensure there's a separator. - assert len(path) > 0 - if path[-1:] in seps: - if b and b[:1] in seps: - path += b[1:] - else: - path += b - elif path[-1:] == colon: - path += b - elif b: - if b[:1] in seps: - path += b - else: - path += sep + b - else: - # path is not empty and does not end with a backslash, - # but b is empty; since, e.g., split('a/') produces - # ('a', ''), it's best if join() adds a backslash in - # this case. - path += sep - - return path +def join(path, *paths): + sep = _get_sep(path) + seps = _get_bothseps(path) + colon = _get_colon(path) + result_drive, result_path = splitdrive(path) + for p in paths: + p_drive, p_path = splitdrive(p) + if p_path and p_path[0] in seps: + # Second path is absolute + if p_drive or not result_drive: + result_drive = p_drive + result_path = p_path + continue + elif p_drive and p_drive != result_drive: + if p_drive.lower() != result_drive.lower(): + # Different drives => ignore the first path entirely + result_drive = p_drive + result_path = p_path + continue + # Same drive in different case + result_drive = p_drive + # Second path is relative to the first + if result_path and result_path[-1] not in seps: + result_path = result_path + sep + result_path = result_path + p_path + ## add separator between UNC and non-absolute path + if (result_path and result_path[0] not in seps and + result_drive and result_drive[-1:] != colon): + return result_drive + sep + result_path + return result_drive + result_path # Split a path in a drive specification (a drive letter followed by a @@ -267,10 +267,12 @@ _pattern_type = type(sre_compile.compile("", 0)) _MAXCACHE = 512 def _compile(pattern, flags): # internal: compile pattern - try: - return _cache[type(pattern), pattern, flags] - except KeyError: - pass + bypass_cache = flags & DEBUG + if not bypass_cache: + try: + return _cache[type(pattern), pattern, flags] + except KeyError: + pass if isinstance(pattern, _pattern_type): if flags: raise ValueError( @@ -279,9 +281,10 @@ def _compile(pattern, flags): if not sre_compile.isstring(pattern): raise TypeError("first argument must be string or compiled pattern") p = sre_compile.compile(pattern, flags) - if len(_cache) >= _MAXCACHE: - _cache.clear() - _cache[type(pattern), pattern, flags] = p + if not bypass_cache: + if len(_cache) >= _MAXCACHE: + _cache.clear() + _cache[type(pattern), pattern, flags] = p return p def _compile_repl(repl, pattern): diff --git a/Lib/shutil.py b/Lib/shutil.py index a5b2d240fe..5320f23646 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -680,17 +680,15 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): zip_filename, base_dir) if not dry_run: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_DEFLATED) - - for dirpath, dirnames, filenames in os.walk(base_dir): - for name in filenames: - path = os.path.normpath(os.path.join(dirpath, name)) - if os.path.isfile(path): - zip.write(path, path) - if logger is not None: - logger.info("adding '%s'", path) - zip.close() + with zipfile.ZipFile(zip_filename, "w", + compression=zipfile.ZIP_DEFLATED) as zf: + for dirpath, dirnames, filenames in os.walk(base_dir): + for name in filenames: + path = os.path.normpath(os.path.join(dirpath, name)) + if os.path.isfile(path): + zf.write(path, path) + if logger is not None: + logger.info("adding '%s'", path) return zip_filename diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 78e4fcfc59..d75a4e0404 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -707,6 +707,9 @@ _PLATFORM_DEFAULT_CLOSE_FDS = object() class Popen(object): + + _child_created = False # Set here since __del__ checks it + def __init__(self, args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, @@ -717,7 +720,6 @@ class Popen(object): """Create new Popen instance.""" _cleanup() - self._child_created = False self._input = None self._communication_started = False if bufsize is None: @@ -859,11 +861,8 @@ class Popen(object): # Wait for the process to terminate, to avoid zombies. self.wait() - def __del__(self, _maxsize=sys.maxsize, _active=_active): - # If __init__ hasn't had a chance to execute (e.g. if it - # was passed an undeclared keyword argument), we don't - # have a _child_created attribute at all. - if not getattr(self, '_child_created', False): + def __del__(self, _maxsize=sys.maxsize): + if not self._child_created: # We didn't get to successfully create a child process. return # In case the child hasn't been waited on, check if it's done. @@ -1446,7 +1445,7 @@ class Popen(object): _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, _WEXITSTATUS=os.WEXITSTATUS): # This method is called (indirectly) by __del__, so it cannot - # refer to anything outside of its local scope.""" + # refer to anything outside of its local scope. if _WIFSIGNALED(sts): self.returncode = -_WTERMSIG(sts) elif _WIFEXITED(sts): diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d31bc70337..f6d7f79390 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -281,6 +281,12 @@ def filemode(mode): DeprecationWarning, 2) return stat.filemode(mode) +def _safe_print(s): + encoding = getattr(sys.stdout, 'encoding', None) + if encoding is not None: + s = s.encode(encoding, 'backslashreplace').decode(encoding) + print(s, end=' ') + class TarError(Exception): """Base exception.""" @@ -1870,24 +1876,24 @@ class TarFile(object): for tarinfo in self: if verbose: - print(stat.filemode(tarinfo.mode), end=' ') - print("%s/%s" % (tarinfo.uname or tarinfo.uid, - tarinfo.gname or tarinfo.gid), end=' ') + _safe_print(stat.filemode(tarinfo.mode)) + _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, + tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): - print("%10s" % ("%d,%d" \ - % (tarinfo.devmajor, tarinfo.devminor)), end=' ') + _safe_print("%10s" % + ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) else: - print("%10d" % tarinfo.size, end=' ') - print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6], end=' ') + _safe_print("%10d" % tarinfo.size) + _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ + % time.localtime(tarinfo.mtime)[:6]) - print(tarinfo.name + ("/" if tarinfo.isdir() else ""), end=' ') + _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) if verbose: if tarinfo.issym(): - print("->", tarinfo.linkname, end=' ') + _safe_print("-> " + tarinfo.linkname) if tarinfo.islnk(): - print("link to", tarinfo.linkname, end=' ') + _safe_print("link to " + tarinfo.linkname) print() def add(self, name, arcname=None, recursive=True, exclude=None, *, filter=None): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b4b5c88338..6bc842f6d8 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -27,11 +27,12 @@ __all__ = [ # Imports. +import atexit as _atexit import functools as _functools import warnings as _warnings -import sys as _sys import io as _io import os as _os +import shutil as _shutil import errno as _errno from random import Random as _Random @@ -355,10 +356,13 @@ class _TemporaryFileCloser: underlying file object, without adding a __del__ method to the temporary file.""" + # Set here since __del__ checks it + file = None + close_called = False + def __init__(self, file, name, delete=True): self.file = file self.name = name - self.close_called = False self.delete = delete # NT provides delete-on-close as a primitive, so we don't need @@ -370,14 +374,13 @@ class _TemporaryFileCloser: # that this must be referenced as self.unlink, because the # name TemporaryFileWrapper may also get None'd out before # __del__ is called. - unlink = _os.unlink - def close(self): - if not self.close_called: + def close(self, unlink=_os.unlink): + if not self.close_called and self.file is not None: self.close_called = True self.file.close() if self.delete: - self.unlink(self.name) + unlink(self.name) # Need to ensure the file is deleted on __del__ def __del__(self): @@ -677,9 +680,11 @@ class TemporaryDirectory(object): in it are removed. """ + # Handle mkdtemp raising an exception + name = None + _closed = False + def __init__(self, suffix="", prefix=template, dir=None): - self._closed = False - self.name = None # Handle mkdtemp raising an exception self.name = mkdtemp(suffix, prefix, dir) def __repr__(self): @@ -688,23 +693,24 @@ class TemporaryDirectory(object): def __enter__(self): return self.name - def cleanup(self, _warn=False): + def cleanup(self, _warn=False, _warnings=_warnings): if self.name and not self._closed: try: - self._rmtree(self.name) + _shutil.rmtree(self.name) except (TypeError, AttributeError) as ex: - # Issue #10188: Emit a warning on stderr - # if the directory could not be cleaned - # up due to missing globals - if "None" not in str(ex): + if "None" not in '%s' % (ex,): raise - print("ERROR: {!r} while cleaning up {!r}".format(ex, self,), - file=_sys.stderr) - return + self._rmtree(self.name) self._closed = True - if _warn: - self._warn("Implicitly cleaning up {!r}".format(self), - ResourceWarning) + if _warn and _warnings.warn: + try: + _warnings.warn("Implicitly cleaning up {!r}".format(self), + ResourceWarning) + except: + if _is_running: + raise + # Don't raise an exception if modules needed for emitting + # a warning are already cleaned in shutdown process. def __exit__(self, exc, value, tb): self.cleanup() @@ -713,36 +719,27 @@ class TemporaryDirectory(object): # Issue a ResourceWarning if implicit cleanup needed self.cleanup(_warn=True) - # XXX (ncoghlan): The following code attempts to make - # this class tolerant of the module nulling out process - # that happens during CPython interpreter shutdown - # Alas, it doesn't actually manage it. See issue #10188 - _listdir = staticmethod(_os.listdir) - _path_join = staticmethod(_os.path.join) - _isdir = staticmethod(_os.path.isdir) - _islink = staticmethod(_os.path.islink) - _remove = staticmethod(_os.remove) - _rmdir = staticmethod(_os.rmdir) - _os_error = OSError - _warn = _warnings.warn - - def _rmtree(self, path): + def _rmtree(self, path, _OSError=OSError, _sep=_os.path.sep, + _listdir=_os.listdir, _remove=_os.remove, _rmdir=_os.rmdir): # Essentially a stripped down version of shutil.rmtree. We can't # use globals because they may be None'ed out at shutdown. - for name in self._listdir(path): - fullname = self._path_join(path, name) - try: - isdir = self._isdir(fullname) and not self._islink(fullname) - except self._os_error: - isdir = False - if isdir: - self._rmtree(fullname) - else: - try: - self._remove(fullname) - except self._os_error: - pass + if not isinstance(path, str): + _sep = _sep.encode() try: - self._rmdir(path) - except self._os_error: + for name in _listdir(path): + fullname = path + _sep + name + try: + _remove(fullname) + except _OSError: + self._rmtree(fullname) + _rmdir(path) + except _OSError: pass + +_is_running = True + +def _on_shutdown(): + global _is_running + _is_running = False + +_atexit.register(_on_shutdown) diff --git a/Lib/test/imghdrdata/python.bmp b/Lib/test/imghdrdata/python.bmp Binary files differnew file mode 100644 index 0000000000..675f95191a --- /dev/null +++ b/Lib/test/imghdrdata/python.bmp diff --git a/Lib/test/imghdrdata/python.gif b/Lib/test/imghdrdata/python.gif Binary files differnew file mode 100644 index 0000000000..96fd9fef76 --- /dev/null +++ b/Lib/test/imghdrdata/python.gif diff --git a/Lib/test/imghdrdata/python.jpg b/Lib/test/imghdrdata/python.jpg Binary files differnew file mode 100644 index 0000000000..21222c09f5 --- /dev/null +++ b/Lib/test/imghdrdata/python.jpg diff --git a/Lib/test/imghdrdata/python.pbm b/Lib/test/imghdrdata/python.pbm new file mode 100644 index 0000000000..1848ba7ff0 --- /dev/null +++ b/Lib/test/imghdrdata/python.pbm @@ -0,0 +1,3 @@ +P4 +16 16 +[a_X ?
\ No newline at end of file diff --git a/Lib/test/imghdrdata/python.pgm b/Lib/test/imghdrdata/python.pgm Binary files differnew file mode 100644 index 0000000000..8349f2a53a --- /dev/null +++ b/Lib/test/imghdrdata/python.pgm diff --git a/Lib/test/imghdrdata/python.png b/Lib/test/imghdrdata/python.png Binary files differnew file mode 100644 index 0000000000..1a987f79fc --- /dev/null +++ b/Lib/test/imghdrdata/python.png diff --git a/Lib/test/imghdrdata/python.ppm b/Lib/test/imghdrdata/python.ppm Binary files differnew file mode 100644 index 0000000000..7d9cdb3215 --- /dev/null +++ b/Lib/test/imghdrdata/python.ppm diff --git a/Lib/test/imghdrdata/python.ras b/Lib/test/imghdrdata/python.ras Binary files differnew file mode 100644 index 0000000000..130e96f817 --- /dev/null +++ b/Lib/test/imghdrdata/python.ras diff --git a/Lib/test/imghdrdata/python.sgi b/Lib/test/imghdrdata/python.sgi Binary files differnew file mode 100644 index 0000000000..ffe9081c7a --- /dev/null +++ b/Lib/test/imghdrdata/python.sgi diff --git a/Lib/test/imghdrdata/python.tiff b/Lib/test/imghdrdata/python.tiff Binary files differnew file mode 100644 index 0000000000..39d0bfcec0 --- /dev/null +++ b/Lib/test/imghdrdata/python.tiff diff --git a/Lib/test/imghdrdata/python.xbm b/Lib/test/imghdrdata/python.xbm new file mode 100644 index 0000000000..cfbee2e980 --- /dev/null +++ b/Lib/test/imghdrdata/python.xbm @@ -0,0 +1,6 @@ +#define python_width 16 +#define python_height 16 +static char python_bits[] = { + 0xDF, 0xFE, 0x8F, 0xFD, 0x5F, 0xFB, 0xAB, 0xFE, 0xB5, 0x8D, 0xDA, 0x8F, + 0xA5, 0x86, 0xFA, 0x83, 0x1A, 0x80, 0x0D, 0x80, 0x0D, 0x80, 0x0F, 0xE0, + 0x0F, 0xF8, 0x0F, 0xF8, 0x0F, 0xFC, 0xFF, 0xFF, }; diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 1b5db9406b..4800d6d7f2 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -5,7 +5,6 @@ Common tests shared by test_str, test_unicode, test_userstring and test_string. import unittest, string, sys, struct from test import support from collections import UserList -import _testcapi class Sequence: def __init__(self, seq='wxyz'): self.seq = seq @@ -1185,19 +1184,27 @@ class MixinStrUnicodeUserStringTest: # Outrageously large width or precision should raise ValueError. self.checkraises(ValueError, '%%%df' % (2**64), '__mod__', (3.2)) self.checkraises(ValueError, '%%.%df' % (2**64), '__mod__', (3.2)) + self.checkraises(OverflowError, '%*s', '__mod__', + (sys.maxsize + 1, '')) + self.checkraises(OverflowError, '%.*f', '__mod__', + (sys.maxsize + 1, 1. / 7)) + + class X(object): pass + self.checkraises(TypeError, 'abc', '__mod__', X()) + @support.cpython_only + def test_formatting_c_limits(self): + from _testcapi import PY_SSIZE_T_MAX, INT_MAX, UINT_MAX + SIZE_MAX = (1 << (PY_SSIZE_T_MAX.bit_length() + 1)) - 1 self.checkraises(OverflowError, '%*s', '__mod__', - (_testcapi.PY_SSIZE_T_MAX + 1, '')) + (PY_SSIZE_T_MAX + 1, '')) self.checkraises(OverflowError, '%.*f', '__mod__', - (_testcapi.INT_MAX + 1, 1. / 7)) + (INT_MAX + 1, 1. / 7)) # Issue 15989 self.checkraises(OverflowError, '%*s', '__mod__', - (1 << (_testcapi.PY_SSIZE_T_MAX.bit_length() + 1), '')) + (SIZE_MAX + 1, '')) self.checkraises(OverflowError, '%.*f', '__mod__', - (_testcapi.UINT_MAX + 1, 1. / 7)) - - class X(object): pass - self.checkraises(TypeError, 'abc', '__mod__', X()) + (UINT_MAX + 1, 1. / 7)) def test_floatformatting(self): # float formatting diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 233b13adcd..5c03f54d5c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -25,7 +25,6 @@ import fnmatch import logging.handlers import struct import tempfile -import _testcapi try: import _thread, threading @@ -1349,6 +1348,7 @@ _TPFLAGS_HAVE_GC = 1<<14 _TPFLAGS_HEAPTYPE = 1<<9 def check_sizeof(test, o, size): + import _testcapi result = sys.getsizeof(o) # add GC header size if ((type(o) == type) and (o.__flags__ & _TPFLAGS_HEAPTYPE) or\ diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 9013a7b188..50c4bba24f 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -17,7 +17,8 @@ try: import threading except ImportError: threading = None -import _testcapi +# Skip this test if the _testcapi module isn't available. +_testcapi = support.import_module('_testcapi') def testfunction(self): diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index e604c926b9..86e1f3a5f0 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -227,7 +227,7 @@ class CgiTests(unittest.TestCase): # if we're not chunking properly, readline is only called twice # (by read_binary); if we are chunking properly, it will be called 5 times # as long as the chunksize is 1 << 16. - self.assertTrue(f.numcalls > 2) + self.assertGreater(f.numcalls, 2) f.close() def test_fieldstorage_multipart(self): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 3377a7b07a..21b12a56e4 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -104,7 +104,7 @@ consts: ('None',) import unittest import weakref -import _testcapi +from test.support import run_doctest, run_unittest, cpython_only def consts(t): @@ -126,7 +126,9 @@ def dump(co): class CodeTest(unittest.TestCase): + @cpython_only def test_newempty(self): + import _testcapi co = _testcapi.code_newempty("filename", "funcname", 15) self.assertEqual(co.co_filename, "filename") self.assertEqual(co.co_name, "funcname") @@ -159,7 +161,6 @@ class CodeWeakRefTest(unittest.TestCase): def test_main(verbose=None): - from test.support import run_doctest, run_unittest from test import test_code run_doctest(test_code, verbose) run_unittest(CodeTest, CodeWeakRefTest) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 35170579a7..cb618ece8c 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,4 +1,3 @@ -import _testcapi import codecs import io import locale @@ -146,19 +145,20 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(readalllines(s, True, 10), sexpected) self.assertEqual(readalllines(s, False, 10), sexpectednoends) + lineends = ("\n", "\r\n", "\r", "\u2028") # Test long lines (multiple calls to read() in readline()) vw = [] vwo = [] - for (i, lineend) in enumerate("\n \r\n \r \u2028".split()): - vw.append((i*200)*"\3042" + lineend) - vwo.append((i*200)*"\3042") - self.assertEqual(readalllines("".join(vw), True), "".join(vw)) - self.assertEqual(readalllines("".join(vw), False),"".join(vwo)) + for (i, lineend) in enumerate(lineends): + vw.append((i*200+200)*"\u3042" + lineend) + vwo.append((i*200+200)*"\u3042") + self.assertEqual(readalllines("".join(vw), True), "|".join(vw)) + self.assertEqual(readalllines("".join(vw), False), "|".join(vwo)) # Test lines where the first read might end with \r, so the # reader has to look ahead whether this is a lone \r or a \r\n for size in range(80): - for lineend in "\n \r\n \r \u2028".split(): + for lineend in lineends: s = 10*(size*"a" + lineend + "xxx\n") reader = getreader(s) for i in range(10): @@ -166,12 +166,54 @@ class ReadTest(MixInCheckStateHandling): reader.readline(keepends=True), size*"a" + lineend, ) + self.assertEqual( + reader.readline(keepends=True), + "xxx\n", + ) reader = getreader(s) for i in range(10): self.assertEqual( reader.readline(keepends=False), size*"a", ) + self.assertEqual( + reader.readline(keepends=False), + "xxx", + ) + + def test_mixed_readline_and_read(self): + lines = ["Humpty Dumpty sat on a wall,\n", + "Humpty Dumpty had a great fall.\r\n", + "All the king's horses and all the king's men\r", + "Couldn't put Humpty together again."] + data = ''.join(lines) + def getreader(): + stream = io.BytesIO(data.encode(self.encoding)) + return codecs.getreader(self.encoding)(stream) + + # Issue #8260: Test readline() followed by read() + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.read(), ''.join(lines[1:])) + self.assertEqual(f.read(), '') + + # Issue #16636: Test readline() followed by readlines() + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.readlines(), lines[1:]) + self.assertEqual(f.read(), '') + + # Test read() followed by read() + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.read(), data[5:]) + self.assertEqual(f.read(), '') + + # Issue #12446: Test read() followed by readlines() + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:]) + self.assertEqual(f.read(), '') def test_bug1175396(self): s = [ @@ -810,13 +852,40 @@ class UTF7Test(ReadTest, unittest.TestCase): def test_partial(self): self.check_partial( - "a+-b", + 'a+-b\x00c\x80d\u0100e\U00010000f', [ - "a", - "a", - "a+", - "a+-", - "a+-b", + 'a', + 'a', + 'a+', + 'a+-', + 'a+-b', + 'a+-b', + 'a+-b', + 'a+-b', + 'a+-b', + 'a+-b\x00', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c\x80', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d\u0100', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e\U00010000', + 'a+-b\x00c\x80d\u0100e\U00010000f', ] ) @@ -1667,7 +1736,7 @@ broken_incremental_coders = broken_unicode_with_streams + [ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): def test_basics(self): - s = "abc123" # all codecs should be able to encode these + s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: name = codecs.lookup(encoding).name if encoding.endswith("_codec"): @@ -1679,9 +1748,9 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): with support.check_warnings(): # unicode-internal has been deprecated (b, size) = codecs.getencoder(encoding)(s) - self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding)) + self.assertEqual(size, len(s), "encoding=%r" % encoding) (chars, size) = codecs.getdecoder(encoding)(b) - self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding)) + self.assertEqual(chars, s, "encoding=%r" % encoding) if encoding not in broken_unicode_with_streams: # check stream reader/writer @@ -1699,15 +1768,13 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): for c in encodedresult: q.write(bytes([c])) decodedresult += reader.read() - self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding)) + self.assertEqual(decodedresult, s, "encoding=%r" % encoding) if encoding not in broken_incremental_coders: - # check incremental decoder/encoder (fetched via the Python - # and C API) and iterencode()/iterdecode() + # check incremental decoder/encoder and iterencode()/iterdecode() try: encoder = codecs.getincrementalencoder(encoding)() - cencoder = _testcapi.codec_incrementalencoder(encoding) - except LookupError: # no IncrementalEncoder + except LookupError: # no IncrementalEncoder pass else: # check incremental decoder/encoder @@ -1720,45 +1787,71 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): for c in encodedresult: decodedresult += decoder.decode(bytes([c])) decodedresult += decoder.decode(b"", True) - self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding)) + self.assertEqual(decodedresult, s, + "encoding=%r" % encoding) + # check iterencode()/iterdecode() + result = "".join(codecs.iterdecode( + codecs.iterencode(s, encoding), encoding)) + self.assertEqual(result, s, "encoding=%r" % encoding) + + # check iterencode()/iterdecode() with empty string + result = "".join(codecs.iterdecode( + codecs.iterencode("", encoding), encoding)) + self.assertEqual(result, "") + + if encoding not in ("idna", "mbcs"): + # check incremental decoder/encoder with errors argument + try: + encoder = codecs.getincrementalencoder(encoding)("ignore") + except LookupError: # no IncrementalEncoder + pass + else: + encodedresult = b"".join(encoder.encode(c) for c in s) + decoder = codecs.getincrementaldecoder(encoding)("ignore") + decodedresult = "".join(decoder.decode(bytes([c])) + for c in encodedresult) + self.assertEqual(decodedresult, s, + "encoding=%r" % encoding) + + @support.cpython_only + def test_basics_capi(self): + from _testcapi import codec_incrementalencoder, codec_incrementaldecoder + s = "abc123" # all codecs should be able to encode these + for encoding in all_unicode_encodings: + if encoding not in broken_incremental_coders: + # check incremental decoder/encoder (fetched via the C API) + try: + cencoder = codec_incrementalencoder(encoding) + except LookupError: # no IncrementalEncoder + pass + else: # check C API encodedresult = b"" for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) decodedresult += cdecoder.decode(b"", True) - self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding)) - - # check iterencode()/iterdecode() - result = "".join(codecs.iterdecode(codecs.iterencode(s, encoding), encoding)) - self.assertEqual(result, s, "%r != %r (encoding=%r)" % (result, s, encoding)) - - # check iterencode()/iterdecode() with empty string - result = "".join(codecs.iterdecode(codecs.iterencode("", encoding), encoding)) - self.assertEqual(result, "") + self.assertEqual(decodedresult, s, + "encoding=%r" % encoding) if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - encoder = codecs.getincrementalencoder(encoding)("ignore") - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") - except LookupError: # no IncrementalEncoder + cencoder = codec_incrementalencoder(encoding, "ignore") + except LookupError: # no IncrementalEncoder pass else: - encodedresult = b"".join(encoder.encode(c) for c in s) - decoder = codecs.getincrementaldecoder(encoding)("ignore") - decodedresult = "".join(decoder.decode(bytes([c])) for c in encodedresult) - self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding)) - encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") - decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) - self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding)) + cdecoder = codec_incrementaldecoder(encoding, "ignore") + decodedresult = "".join(cdecoder.decode(bytes([c])) + for c in encodedresult) + self.assertEqual(decodedresult, s, + "encoding=%r" % encoding) def test_seek(self): # all codecs should be able to encode these @@ -2307,8 +2400,6 @@ class TransformCodecTest(unittest.TestCase): def test_readline(self): for encoding in bytes_transform_encodings: - if encoding in ['uu_codec', 'zlib_codec']: - continue sin = codecs.encode(b"\x80", encoding) reader = codecs.getreader(encoding)(io.BytesIO(sin)) sout = reader.readline() diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 39230e1cf0..04c4c3717e 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -344,6 +344,13 @@ class AsCompletedTests: SUCCESSFUL_FUTURE]), completed_futures) + def test_duplicate_futures(self): + # Issue 20367. Duplicate futures should not raise exceptions or give + # duplicate responses. + future1 = self.executor.submit(time.sleep, 2) + completed = [f for f in futures.as_completed([future1,future1])] + self.assertEqual(len(completed), 1) + class ThreadPoolAsCompletedTests(ThreadPoolMixin, AsCompletedTests, unittest.TestCase): pass diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 8940ae9d54..1a891a8063 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -2046,6 +2046,7 @@ order (MRO) for bases """ prop2 = property(fset=setter) self.assertEqual(prop2.__doc__, None) + @support.cpython_only def test_testcapi_no_segfault(self): # this segfaulted in 2.5b2 try: diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py index 819d9dc883..9129ac0cb1 100644 --- a/Lib/test/test_devpoll.py +++ b/Lib/test/test_devpoll.py @@ -3,8 +3,7 @@ # Initial tests are copied as is from "test_poll.py" import os, select, random, unittest, sys -from test.support import TESTFN, run_unittest -from _testcapi import USHRT_MAX +from test.support import TESTFN, run_unittest, cpython_only try: select.devpoll @@ -94,8 +93,18 @@ class DevPollTests(unittest.TestCase): pollster.register(w) # Issue #17919 self.assertRaises(OverflowError, pollster.register, 0, -1) - self.assertRaises(OverflowError, pollster.register, 0, USHRT_MAX + 1) + self.assertRaises(OverflowError, pollster.register, 0, 1 << 64) self.assertRaises(OverflowError, pollster.modify, 1, -1) + self.assertRaises(OverflowError, pollster.modify, 1, 1 << 64) + + @cpython_only + def test_events_mask_overflow_c_limits(self): + from _testcapi import USHRT_MAX + pollster = select.devpoll() + w, r = os.pipe() + pollster.register(w) + # Issue #17919 + self.assertRaises(OverflowError, pollster.register, 0, USHRT_MAX + 1) self.assertRaises(OverflowError, pollster.modify, 1, USHRT_MAX + 1) diff --git a/Lib/test/test_email/data/msg_02.txt b/Lib/test/test_email/data/msg_02.txt index 43f248038a..5d0a7e16c8 100644 --- a/Lib/test/test_email/data/msg_02.txt +++ b/Lib/test/test_email/data/msg_02.txt @@ -119,6 +119,7 @@ hello --__--__---- + --192.168.1.2.889.32614.987812255.500.21814 Content-type: text/plain; charset=us-ascii Content-description: Digest Footer diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 646082b4a4..32996ca4c8 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -540,6 +540,15 @@ class TestParser(TestParserMixin, TestEmailBase): self._test_get_x(parser.get_bare_quoted_string, '""', '""', '', [], '') + # Issue 16983: apply postel's law to some bad encoding. + def test_encoded_word_inside_quotes(self): + self._test_get_x(parser.get_bare_quoted_string, + '"=?utf-8?Q?not_really_valid?="', + '"not really valid"', + 'not really valid', + [errors.InvalidHeaderDefect], + '') + # get_comment def test_get_comment_only(self): diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index c787695e12..51fe756df7 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -1711,7 +1711,8 @@ From: bperson@dom.ain --BOUNDARY ---BOUNDARY--''') +--BOUNDARY-- +''') def test_no_parts_in_a_multipart_with_empty_epilogue(self): outer = MIMEBase('multipart', 'mixed') @@ -1756,7 +1757,8 @@ MIME-Version: 1.0 Content-Transfer-Encoding: 7bit hello world ---BOUNDARY--''') +--BOUNDARY-- +''') def test_seq_parts_in_a_multipart_with_empty_preamble(self): eq = self.ndiffAssertEqual @@ -1782,7 +1784,8 @@ MIME-Version: 1.0 Content-Transfer-Encoding: 7bit hello world ---BOUNDARY--''') +--BOUNDARY-- +''') def test_seq_parts_in_a_multipart_with_none_preamble(self): @@ -1808,7 +1811,8 @@ MIME-Version: 1.0 Content-Transfer-Encoding: 7bit hello world ---BOUNDARY--''') +--BOUNDARY-- +''') def test_seq_parts_in_a_multipart_with_none_epilogue(self): @@ -1834,7 +1838,8 @@ MIME-Version: 1.0 Content-Transfer-Encoding: 7bit hello world ---BOUNDARY--''') +--BOUNDARY-- +''') def test_seq_parts_in_a_multipart_with_empty_epilogue(self): @@ -3495,7 +3500,7 @@ Here's the message body self.assertTrue(msg.get_payload(0).get_payload().endswith('\r\n')) -class Test8BitBytesHandling(unittest.TestCase): +class Test8BitBytesHandling(TestEmailBase): # In Python3 all input is string, but that doesn't work if the actual input # uses an 8bit transfer encoding. To hack around that, in email 5.1 we # decode byte streams using the surrogateescape error handler, and @@ -3748,6 +3753,16 @@ class Test8BitBytesHandling(unittest.TestCase): email.generator.Generator(out).flatten(msg) self.assertEqual(out.getvalue(), self.non_latin_bin_msg_as7bit_wrapped) + def test_str_generator_should_not_mutate_msg_when_handling_8bit(self): + msg = email.message_from_bytes(self.non_latin_bin_msg) + out = BytesIO() + BytesGenerator(out).flatten(msg) + orig_value = out.getvalue() + Generator(StringIO()).flatten(msg) # Should not mutate msg! + out = BytesIO() + BytesGenerator(out).flatten(msg) + self.assertEqual(out.getvalue(), orig_value) + def test_bytes_generator_with_unix_from(self): # The unixfrom contains a current date, so we can't check it # literally. Just make sure the first word is 'From' and the @@ -5018,6 +5033,26 @@ Content-Type: application/x-foo; name*0=\"Frank's\"; name*1=\" Document\" self.assertNotIsInstance(param, tuple) self.assertEqual(param, "Frank's Document") + def test_rfc2231_missing_tick(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="'This%20is%20broken"; +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + "'This is broken") + + def test_rfc2231_missing_tick_with_encoded_non_ascii(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="'This%20is%E2broken"; +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + "'This is\ufffdbroken") + # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_in_value_with_charset_and_lang def test_rfc2231_tick_attack_extended(self): eq = self.assertEqual diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index f829f83e32..adaf3e8fe4 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -1143,6 +1143,16 @@ class TestAddressHeader(TestHeaderBase): 'example.com', None), + 'rfc2047_atom_in_quoted_string_is_decoded': + ('"=?utf-8?q?=C3=89ric?=" <foo@example.com>', + [errors.InvalidHeaderDefect], + 'Éric <foo@example.com>', + 'Éric', + 'foo@example.com', + 'foo', + 'example.com', + None), + } # XXX: Need many more examples, and in particular some with names in diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index fe660bf9b4..8d11d90abb 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -8,8 +8,8 @@ import weakref import errno from test.support import (TESTFN, captured_output, check_impl_detail, - cpython_only, gc_collect, run_unittest, no_tracing, - unlink) + check_warnings, cpython_only, gc_collect, + no_tracing, run_unittest, unlink) class NaiveException(Exception): def __init__(self, x): @@ -832,6 +832,7 @@ class ExceptionTests(unittest.TestCase): self.assertIn("maximum recursion depth exceeded", str(v)) + @cpython_only def test_MemoryError(self): # PyErr_NoMemory always raises the same exception instance. # Check that the traceback is not doubled. @@ -890,6 +891,7 @@ class ExceptionTests(unittest.TestCase): self.assertEqual(error5.a, 1) self.assertEqual(error5.__doc__, "") + @cpython_only def test_memory_error_cleanup(self): # Issue #5437: preallocated MemoryError instances should not keep # traceback objects alive. @@ -960,9 +962,10 @@ class ImportErrorTests(unittest.TestCase): def test_non_str_argument(self): # Issue #15778 - arg = b'abc' - exc = ImportError(arg) - self.assertEqual(str(arg), str(exc)) + with check_warnings(('', BytesWarning), quiet=True): + arg = b'abc' + exc = ImportError(arg) + self.assertEqual(str(arg), str(exc)) def test_main(): diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index b8cda2f108..1810c4e95e 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -7,9 +7,9 @@ import platform import os import struct import sys -import _testcapi import unittest -from test.support import verbose, TESTFN, unlink, run_unittest, import_module +from test.support import (verbose, TESTFN, unlink, run_unittest, import_module, + cpython_only) # Skip test if no fcntl module. fcntl = import_module('fcntl') @@ -50,6 +50,12 @@ def get_lockdata(): lockdata = get_lockdata() +class BadFile: + def __init__(self, fn): + self.fn = fn + def fileno(self): + return self.fn + class TestFcntl(unittest.TestCase): def setUp(self): @@ -81,24 +87,27 @@ class TestFcntl(unittest.TestCase): self.f.close() def test_fcntl_bad_file(self): - class F: - def __init__(self, fn): - self.fn = fn - def fileno(self): - return self.fn - self.assertRaises(ValueError, fcntl.fcntl, -1, fcntl.F_SETFL, os.O_NONBLOCK) - self.assertRaises(ValueError, fcntl.fcntl, F(-1), fcntl.F_SETFL, os.O_NONBLOCK) - self.assertRaises(TypeError, fcntl.fcntl, 'spam', fcntl.F_SETFL, os.O_NONBLOCK) - self.assertRaises(TypeError, fcntl.fcntl, F('spam'), fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(ValueError): + fcntl.fcntl(-1, fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(ValueError): + fcntl.fcntl(BadFile(-1), fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(TypeError): + fcntl.fcntl('spam', fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(TypeError): + fcntl.fcntl(BadFile('spam'), fcntl.F_SETFL, os.O_NONBLOCK) + + @cpython_only + def test_fcntl_bad_file_overflow(self): + from _testcapi import INT_MAX, INT_MIN # Issue 15989 - self.assertRaises(OverflowError, fcntl.fcntl, _testcapi.INT_MAX + 1, - fcntl.F_SETFL, os.O_NONBLOCK) - self.assertRaises(OverflowError, fcntl.fcntl, F(_testcapi.INT_MAX + 1), - fcntl.F_SETFL, os.O_NONBLOCK) - self.assertRaises(OverflowError, fcntl.fcntl, _testcapi.INT_MIN - 1, - fcntl.F_SETFL, os.O_NONBLOCK) - self.assertRaises(OverflowError, fcntl.fcntl, F(_testcapi.INT_MIN - 1), - fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(OverflowError): + fcntl.fcntl(INT_MAX + 1, fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(OverflowError): + fcntl.fcntl(BadFile(INT_MAX + 1), fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(OverflowError): + fcntl.fcntl(INT_MIN - 1, fcntl.F_SETFL, os.O_NONBLOCK) + with self.assertRaises(OverflowError): + fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( platform.machine().startswith('arm') and platform.system() == 'Linux', diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index a6853a147e..444be91920 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -8,9 +8,8 @@ import unittest from array import array from weakref import proxy from functools import wraps -import _testcapi -from test.support import TESTFN, check_warnings, run_unittest, make_bad_fd +from test.support import TESTFN, check_warnings, run_unittest, make_bad_fd, cpython_only from collections import UserList from _io import FileIO as _FileIO @@ -362,7 +361,11 @@ class OtherFileTests(unittest.TestCase): if sys.platform == 'win32': import msvcrt self.assertRaises(IOError, msvcrt.get_osfhandle, make_bad_fd()) + + @cpython_only + def testInvalidFd_overflow(self): # Issue 15989 + import _testcapi self.assertRaises(TypeError, _FileIO, _testcapi.INT_MAX + 1) self.assertRaises(TypeError, _FileIO, _testcapi.INT_MIN - 1) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index bd159f5689..ea5a731d98 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -313,22 +313,32 @@ def test_main(): support.run_unittest(FormatTest) def test_precision(self): - INT_MAX = 2147483647 - f = 1.2 self.assertEqual(format(f, ".0f"), "1") self.assertEqual(format(f, ".3f"), "1.200") with self.assertRaises(ValueError) as cm: - format(f, ".%sf" % (INT_MAX + 1)) + format(f, ".%sf" % (sys.maxsize + 1)) self.assertEqual(str(cm.exception), "precision too big") c = complex(f) - self.assertEqual(format(f, ".0f"), "1") - self.assertEqual(format(f, ".3f"), "1.200") + self.assertEqual(format(c, ".0f"), "1+0j") + self.assertEqual(format(c, ".3f"), "1.200+0.000j") with self.assertRaises(ValueError) as cm: - format(f, ".%sf" % (INT_MAX + 1)) + format(c, ".%sf" % (sys.maxsize + 1)) self.assertEqual(str(cm.exception), "precision too big") + @support.cpython_only + def test_precision_c_limits(self): + from _testcapi import INT_MAX + + f = 1.2 + with self.assertRaises(ValueError) as cm: + format(f, ".%sf" % (INT_MAX + 1)) + + c = complex(f) + with self.assertRaises(ValueError) as cm: + format(c, ".%sf" % (INT_MAX + 1)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 64cd80bc14..6c95c491ff 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -965,7 +965,7 @@ class TestTimeouts(TestCase): def testTimeoutDefault(self): # default -- use global socket timeout - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: ftp = ftplib.FTP(HOST) @@ -977,13 +977,13 @@ class TestTimeouts(TestCase): def testTimeoutNone(self): # no timeout -- do not use global socket timeout - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: ftp = ftplib.FTP(HOST, timeout=None) finally: socket.setdefaulttimeout(None) - self.assertTrue(ftp.sock.gettimeout() is None) + self.assertIsNone(ftp.sock.gettimeout()) self.evt.wait() ftp.close() diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 715b5c6a32..fe64ee1669 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -1,5 +1,7 @@ import unittest from test import support +# Skip this test if the _testcapi module isn't available. +support.import_module('_testcapi') from _testcapi import getargs_keywords, getargs_keyword_only """ diff --git a/Lib/test/test_hash.py b/Lib/test/test_hash.py index e3ab6e4385..22ebbccb22 100644 --- a/Lib/test/test_hash.py +++ b/Lib/test/test_hash.py @@ -138,7 +138,7 @@ class HashRandomizationTests: # an object to be tested def get_hash_command(self, repr_): - return 'print(hash(%s))' % repr_ + return 'print(hash(eval(%a)))' % repr_ def get_hash(self, repr_, seed=None): env = os.environ.copy() diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index c977a9dd4d..11d9c9ce8b 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -151,6 +151,12 @@ text ("data", "&#bad;"), ("endtag", "p"), ]) + # add the [] as a workaround to avoid buffering (see #20288) + self._run_check(["<div>&#bad;</div>"], [ + ("starttag", "div", []), + ("data", "&#bad;"), + ("endtag", "div"), + ]) def test_unclosed_entityref(self): self._run_check("&entityref foo", [ diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 4410a93f4f..c8ded92478 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -132,7 +132,7 @@ class HeaderTests(TestCase): conn.sock = FakeSocket(None) conn.putrequest('GET','/') conn.putheader('Content-length', 42) - self.assertTrue(b'Content-length: 42' in conn._buffer) + self.assertIn(b'Content-length: 42', conn._buffer) def test_ipv6host_header(self): # Default host header on IPv6 transaction should wrapped by [] if @@ -699,7 +699,7 @@ class TimeoutTest(TestCase): # and into the socket. # default -- use global socket timeout - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT) @@ -710,7 +710,7 @@ class TimeoutTest(TestCase): httpConn.close() # no timeout -- do not use global socket default - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index dfccb6bd63..4e62963725 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -531,7 +531,7 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) - self.assertTrue(match is not None) + self.assertIsNotNone(match) def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') diff --git a/Lib/test/test_imghdr.py b/Lib/test/test_imghdr.py new file mode 100644 index 0000000000..0ad4343f52 --- /dev/null +++ b/Lib/test/test_imghdr.py @@ -0,0 +1,131 @@ +import imghdr +import io +import os +import unittest +import warnings +from test.support import findfile, TESTFN, unlink + +TEST_FILES = ( + ('python.png', 'png'), + ('python.gif', 'gif'), + ('python.bmp', 'bmp'), + ('python.ppm', 'ppm'), + ('python.pgm', 'pgm'), + ('python.pbm', 'pbm'), + ('python.jpg', 'jpeg'), + ('python.ras', 'rast'), + ('python.sgi', 'rgb'), + ('python.tiff', 'tiff'), + ('python.xbm', 'xbm') +) + +class UnseekableIO(io.FileIO): + def tell(self): + raise io.UnsupportedOperation + + def seek(self, *args, **kwargs): + raise io.UnsupportedOperation + +class TestImghdr(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.testfile = findfile('python.png', subdir='imghdrdata') + with open(cls.testfile, 'rb') as stream: + cls.testdata = stream.read() + + def tearDown(self): + unlink(TESTFN) + + def test_data(self): + for filename, expected in TEST_FILES: + filename = findfile(filename, subdir='imghdrdata') + self.assertEqual(imghdr.what(filename), expected) + with open(filename, 'rb') as stream: + self.assertEqual(imghdr.what(stream), expected) + with open(filename, 'rb') as stream: + data = stream.read() + self.assertEqual(imghdr.what(None, data), expected) + self.assertEqual(imghdr.what(None, bytearray(data)), expected) + + def test_register_test(self): + def test_jumbo(h, file): + if h.startswith(b'eggs'): + return 'ham' + imghdr.tests.append(test_jumbo) + self.addCleanup(imghdr.tests.pop) + self.assertEqual(imghdr.what(None, b'eggs'), 'ham') + + def test_file_pos(self): + with open(TESTFN, 'wb') as stream: + stream.write(b'ababagalamaga') + pos = stream.tell() + stream.write(self.testdata) + with open(TESTFN, 'rb') as stream: + stream.seek(pos) + self.assertEqual(imghdr.what(stream), 'png') + self.assertEqual(stream.tell(), pos) + + def test_bad_args(self): + with self.assertRaises(TypeError): + imghdr.what() + with self.assertRaises(AttributeError): + imghdr.what(None) + with self.assertRaises(TypeError): + imghdr.what(self.testfile, 1) + with self.assertRaises(AttributeError): + imghdr.what(os.fsencode(self.testfile)) + with open(self.testfile, 'rb') as f: + with self.assertRaises(AttributeError): + imghdr.what(f.fileno()) + + def test_invalid_headers(self): + for header in (b'\211PN\r\n', + b'\001\331', + b'\x59\xA6', + b'cutecat', + b'000000JFI', + b'GIF80'): + self.assertIsNone(imghdr.what(None, header)) + + def test_string_data(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", BytesWarning) + for filename, _ in TEST_FILES: + filename = findfile(filename, subdir='imghdrdata') + with open(filename, 'rb') as stream: + data = stream.read().decode('latin1') + with self.assertRaises(TypeError): + imghdr.what(io.StringIO(data)) + with self.assertRaises(TypeError): + imghdr.what(None, data) + + def test_missing_file(self): + with self.assertRaises(FileNotFoundError): + imghdr.what('missing') + + def test_closed_file(self): + stream = open(self.testfile, 'rb') + stream.close() + with self.assertRaises(ValueError) as cm: + imghdr.what(stream) + stream = io.BytesIO(self.testdata) + stream.close() + with self.assertRaises(ValueError) as cm: + imghdr.what(stream) + + def test_unseekable(self): + with open(TESTFN, 'wb') as stream: + stream.write(self.testdata) + with UnseekableIO(TESTFN, 'rb') as stream: + with self.assertRaises(io.UnsupportedOperation): + imghdr.what(stream) + + def test_output_stream(self): + with open(TESTFN, 'wb') as stream: + stream.write(self.testdata) + stream.seek(0) + with self.assertRaises(OSError) as cm: + imghdr.what(stream) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index e843cacae1..5c2c79bbe5 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -32,7 +32,6 @@ import time import unittest import warnings import weakref -import _testcapi from collections import deque, UserList from itertools import cycle, count from test import support @@ -1977,8 +1976,10 @@ class TextIOWrapperTest(unittest.TestCase): os.environ.clear() os.environ.update(old_environ) - # Issue 15989 + @support.cpython_only def test_device_encoding(self): + # Issue 15989 + import _testcapi b = self.BytesIO() b.fileno = lambda: _testcapi.INT_MAX + 1 self.assertRaises(OverflowError, self.TextIOWrapper, b) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 99c54f1614..5bfcf41429 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -398,18 +398,47 @@ class NetmaskTestMixin_v4(CommonTestMixin_v4): assertBadAddress("::1.2.3.4", "Only decimal digits") assertBadAddress("1.2.3.256", re.escape("256 (> 255)")) + def test_valid_netmask(self): + self.assertEqual(str(self.factory('192.0.2.0/255.255.255.0')), + '192.0.2.0/24') + for i in range(0, 33): + # Generate and re-parse the CIDR format (trivial). + net_str = '0.0.0.0/%d' % i + net = self.factory(net_str) + self.assertEqual(str(net), net_str) + # Generate and re-parse the expanded netmask. + self.assertEqual( + str(self.factory('0.0.0.0/%s' % net.netmask)), net_str) + # Zero prefix is treated as decimal. + self.assertEqual(str(self.factory('0.0.0.0/0%d' % i)), net_str) + # Generate and re-parse the expanded hostmask. The ambiguous + # cases (/0 and /32) are treated as netmasks. + if i in (32, 0): + net_str = '0.0.0.0/%d' % (32 - i) + self.assertEqual( + str(self.factory('0.0.0.0/%s' % net.hostmask)), net_str) + def test_netmask_errors(self): def assertBadNetmask(addr, netmask): - msg = "%r is not a valid netmask" - with self.assertNetmaskError(msg % netmask): + msg = "%r is not a valid netmask" % netmask + with self.assertNetmaskError(re.escape(msg)): self.factory("%s/%s" % (addr, netmask)) assertBadNetmask("1.2.3.4", "") + assertBadNetmask("1.2.3.4", "-1") + assertBadNetmask("1.2.3.4", "+1") + assertBadNetmask("1.2.3.4", " 1 ") + assertBadNetmask("1.2.3.4", "0x1") assertBadNetmask("1.2.3.4", "33") assertBadNetmask("1.2.3.4", "254.254.255.256") + assertBadNetmask("1.2.3.4", "1.a.2.3") assertBadNetmask("1.1.1.1", "254.xyz.2.3") assertBadNetmask("1.1.1.1", "240.255.0.0") + assertBadNetmask("1.1.1.1", "255.254.128.0") + assertBadNetmask("1.1.1.1", "0.1.127.255") assertBadNetmask("1.1.1.1", "pudding") + assertBadNetmask("1.1.1.1", "::") + class InterfaceTestCase_v4(BaseTestCase, NetmaskTestMixin_v4): factory = ipaddress.IPv4Interface @@ -438,17 +467,34 @@ class NetmaskTestMixin_v6(CommonTestMixin_v6): assertBadAddress("10/8", "At least 3 parts") assertBadAddress("1234:axy::b", "Only hex digits") + def test_valid_netmask(self): + # We only support CIDR for IPv6, because expanded netmasks are not + # standard notation. + self.assertEqual(str(self.factory('2001:db8::/32')), '2001:db8::/32') + for i in range(0, 129): + # Generate and re-parse the CIDR format (trivial). + net_str = '::/%d' % i + self.assertEqual(str(self.factory(net_str)), net_str) + # Zero prefix is treated as decimal. + self.assertEqual(str(self.factory('::/0%d' % i)), net_str) + def test_netmask_errors(self): def assertBadNetmask(addr, netmask): - msg = "%r is not a valid netmask" - with self.assertNetmaskError(msg % netmask): + msg = "%r is not a valid netmask" % netmask + with self.assertNetmaskError(re.escape(msg)): self.factory("%s/%s" % (addr, netmask)) assertBadNetmask("::1", "") assertBadNetmask("::1", "::1") assertBadNetmask("::1", "1::") + assertBadNetmask("::1", "-1") + assertBadNetmask("::1", "+1") + assertBadNetmask("::1", " 1 ") + assertBadNetmask("::1", "0x1") assertBadNetmask("::1", "129") + assertBadNetmask("::1", "1.2.3.4") assertBadNetmask("::1", "pudding") + assertBadNetmask("::", "::") class InterfaceTestCase_v6(BaseTestCase, NetmaskTestMixin_v6): factory = ipaddress.IPv6Interface @@ -694,16 +740,14 @@ class IpaddrUnitTest(unittest.TestCase): def testZeroNetmask(self): ipv4_zero_netmask = ipaddress.IPv4Interface('1.2.3.4/0') self.assertEqual(int(ipv4_zero_netmask.network.netmask), 0) - self.assertTrue(ipv4_zero_netmask.network._is_valid_netmask( - str(0))) + self.assertEqual(ipv4_zero_netmask._prefix_from_prefix_string('0'), 0) self.assertTrue(ipv4_zero_netmask._is_valid_netmask('0')) self.assertTrue(ipv4_zero_netmask._is_valid_netmask('0.0.0.0')) self.assertFalse(ipv4_zero_netmask._is_valid_netmask('invalid')) ipv6_zero_netmask = ipaddress.IPv6Interface('::1/0') self.assertEqual(int(ipv6_zero_netmask.network.netmask), 0) - self.assertTrue(ipv6_zero_netmask.network._is_valid_netmask( - str(0))) + self.assertEqual(ipv6_zero_netmask._prefix_from_prefix_string('0'), 0) def testIPv4NetAndHostmasks(self): net = self.ipv4_network @@ -719,7 +763,7 @@ class IpaddrUnitTest(unittest.TestCase): self.assertFalse(net._is_hostmask('1.2.3.4')) net = ipaddress.IPv4Network('127.0.0.0/0.0.0.255') - self.assertEqual(24, net.prefixlen) + self.assertEqual(net.prefixlen, 24) def testGetBroadcast(self): self.assertEqual(int(self.ipv4_network.broadcast_address), 16909311) @@ -877,13 +921,13 @@ class IpaddrUnitTest(unittest.TestCase): 36893488147419103232) def testContains(self): - self.assertTrue(ipaddress.IPv4Interface('1.2.3.128/25') in - self.ipv4_network) - self.assertFalse(ipaddress.IPv4Interface('1.2.4.1/24') in + self.assertIn(ipaddress.IPv4Interface('1.2.3.128/25'), + self.ipv4_network) + self.assertNotIn(ipaddress.IPv4Interface('1.2.4.1/24'), self.ipv4_network) # We can test addresses and string as well. addr1 = ipaddress.IPv4Address('1.2.3.37') - self.assertTrue(addr1 in self.ipv4_network) + self.assertIn(addr1, self.ipv4_network) # issue 61, bad network comparison on like-ip'd network objects # with identical broadcast addresses. self.assertFalse(ipaddress.IPv4Network('1.1.0.0/16').__contains__( @@ -1271,11 +1315,6 @@ class IpaddrUnitTest(unittest.TestCase): self.assertEqual(ipaddress.IPv6Interface('::1:0:0:0:0').packed, b'\x00' * 6 + b'\x00\x01' + b'\x00' * 8) - def testIpStrFromPrefixlen(self): - ipv4 = ipaddress.IPv4Interface('1.2.3.4/24') - self.assertEqual(ipv4._ip_string_from_prefix(), '255.255.255.0') - self.assertEqual(ipv4._ip_string_from_prefix(28), '255.255.255.240') - def testIpType(self): ipv4net = ipaddress.ip_network('1.2.3.4') ipv4addr = ipaddress.ip_address('1.2.3.4') @@ -1461,20 +1500,14 @@ class IpaddrUnitTest(unittest.TestCase): dummy[self.ipv6_address] = None dummy[ip1] = None dummy[ip2] = None - self.assertTrue(self.ipv4_address in dummy) - self.assertTrue(ip2 in dummy) + self.assertIn(self.ipv4_address, dummy) + self.assertIn(ip2, dummy) def testIPBases(self): net = self.ipv4_network self.assertEqual('1.2.3.0/24', net.compressed) - self.assertEqual( - net._ip_int_from_prefix(24), - net._ip_int_from_prefix(None)) net = self.ipv6_network self.assertRaises(ValueError, net._string_from_ip_int, 2**128 + 1) - self.assertEqual( - self.ipv6_address._string_from_ip_int(self.ipv6_address._ip), - self.ipv6_address._string_from_ip_int(None)) def testIPv6NetworkHelpers(self): net = self.ipv6_network @@ -1575,9 +1608,9 @@ class IpaddrUnitTest(unittest.TestCase): def testNetworkElementCaching(self): # V4 - make sure we're empty - self.assertFalse('network_address' in self.ipv4_network._cache) - self.assertFalse('broadcast_address' in self.ipv4_network._cache) - self.assertFalse('hostmask' in self.ipv4_network._cache) + self.assertNotIn('network_address', self.ipv4_network._cache) + self.assertNotIn('broadcast_address', self.ipv4_network._cache) + self.assertNotIn('hostmask', self.ipv4_network._cache) # V4 - populate and test self.assertEqual(self.ipv4_network.network_address, @@ -1588,12 +1621,12 @@ class IpaddrUnitTest(unittest.TestCase): ipaddress.IPv4Address('0.0.0.255')) # V4 - check we're cached - self.assertTrue('broadcast_address' in self.ipv4_network._cache) - self.assertTrue('hostmask' in self.ipv4_network._cache) + self.assertIn('broadcast_address', self.ipv4_network._cache) + self.assertIn('hostmask', self.ipv4_network._cache) # V6 - make sure we're empty - self.assertFalse('broadcast_address' in self.ipv6_network._cache) - self.assertFalse('hostmask' in self.ipv6_network._cache) + self.assertNotIn('broadcast_address', self.ipv6_network._cache) + self.assertNotIn('hostmask', self.ipv6_network._cache) # V6 - populate and test self.assertEqual(self.ipv6_network.network_address, @@ -1613,11 +1646,10 @@ class IpaddrUnitTest(unittest.TestCase): ipaddress.IPv6Address('::ffff:ffff:ffff:ffff')) # V6 - check we're cached - self.assertTrue('broadcast_address' in self.ipv6_network._cache) - self.assertTrue('hostmask' in self.ipv6_network._cache) - self.assertTrue( - 'broadcast_address' in self.ipv6_interface.network._cache) - self.assertTrue('hostmask' in self.ipv6_interface.network._cache) + self.assertIn('broadcast_address', self.ipv6_network._cache) + self.assertIn('hostmask', self.ipv6_network._cache) + self.assertIn('broadcast_address', self.ipv6_interface.network._cache) + self.assertIn('hostmask', self.ipv6_interface.network._cache) def testTeredo(self): # stolen from wikipedia diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index baf1d6a3b2..3f9c1e295b 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -130,7 +130,7 @@ class LongTest(unittest.TestCase): # The sign of the number is also random. def getran(self, ndigits): - self.assertTrue(ndigits > 0) + self.assertGreater(ndigits, 0) nbits_hi = ndigits * SHIFT nbits_lo = nbits_hi - SHIFT + 1 answer = 0 @@ -865,21 +865,21 @@ class LongTest(unittest.TestCase): def test_small_ints(self): for i in range(-5, 257): - self.assertTrue(i is i + 0) - self.assertTrue(i is i * 1) - self.assertTrue(i is i - 0) - self.assertTrue(i is i // 1) - self.assertTrue(i is i & -1) - self.assertTrue(i is i | 0) - self.assertTrue(i is i ^ 0) - self.assertTrue(i is ~~i) - self.assertTrue(i is i**1) - self.assertTrue(i is int(str(i))) - self.assertTrue(i is i<<2>>2, str(i)) + self.assertIs(i, i + 0) + self.assertIs(i, i * 1) + self.assertIs(i, i - 0) + self.assertIs(i, i // 1) + self.assertIs(i, i & -1) + self.assertIs(i, i | 0) + self.assertIs(i, i ^ 0) + self.assertIs(i, ~~i) + self.assertIs(i, i**1) + self.assertIs(i, int(str(i))) + self.assertIs(i, i<<2>>2, str(i)) # corner cases i = 1 << 70 - self.assertTrue(i - i is 0) - self.assertTrue(0 * i is 0) + self.assertIs(i - i, 0) + self.assertIs(0 * i, 0) def test_bit_length(self): tiny = 1e-10 @@ -926,7 +926,7 @@ class LongTest(unittest.TestCase): got = round(k+offset, -1) expected = v+offset self.assertEqual(got, expected) - self.assertTrue(type(got) is int) + self.assertIs(type(got), int) # larger second argument self.assertEqual(round(-150, -2), -200) @@ -965,7 +965,7 @@ class LongTest(unittest.TestCase): got = round(10**k + 324678, -3) expect = 10**k + 325000 self.assertEqual(got, expect) - self.assertTrue(type(got) is int) + self.assertIs(type(got), int) # nonnegative second argument: round(x, n) should just return x for n in range(5): @@ -973,7 +973,7 @@ class LongTest(unittest.TestCase): x = random.randrange(-10000, 10000) got = round(x, n) self.assertEqual(got, x) - self.assertTrue(type(got) is int) + self.assertIs(type(got), int) for huge_n in 2**31-1, 2**31, 2**63-1, 2**63, 2**100, 10**100: self.assertEqual(round(8979323, huge_n), 8979323) @@ -982,7 +982,7 @@ class LongTest(unittest.TestCase): x = random.randrange(-10000, 10000) got = round(x) self.assertEqual(got, x) - self.assertTrue(type(got) is int) + self.assertIs(type(got), int) # bad second argument bad_exponents = ('brian', 2.0, 0j, None) @@ -1187,15 +1187,15 @@ class LongTest(unittest.TestCase): class myint(int): pass - self.assertTrue(type(myint.from_bytes(b'\x00', 'big')) is myint) + self.assertIs(type(myint.from_bytes(b'\x00', 'big')), myint) self.assertEqual(myint.from_bytes(b'\x01', 'big'), 1) - self.assertTrue( - type(myint.from_bytes(b'\x00', 'big', signed=False)) is myint) + self.assertIs( + type(myint.from_bytes(b'\x00', 'big', signed=False)), myint) self.assertEqual(myint.from_bytes(b'\x01', 'big', signed=False), 1) - self.assertTrue(type(myint.from_bytes(b'\x00', 'little')) is myint) + self.assertIs(type(myint.from_bytes(b'\x00', 'little')), myint) self.assertEqual(myint.from_bytes(b'\x01', 'little'), 1) - self.assertTrue(type(myint.from_bytes( - b'\x00', 'little', signed=False)) is myint) + self.assertIs(type(myint.from_bytes( + b'\x00', 'little', signed=False)), myint) self.assertEqual(myint.from_bytes(b'\x01', 'little', signed=False), 1) self.assertEqual( int.from_bytes([255, 0, 0], 'big', signed=True), -65536) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index f2e4c63240..5e9e30acf1 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -233,7 +233,7 @@ class TestMailbox(TestBase): msg = self._box.get(key0) self.assertEqual(msg['from'], 'foo') self.assertEqual(msg.get_payload(), '0\n') - self.assertIs(self._box.get('foo'), None) + self.assertIsNone(self._box.get('foo')) self.assertIs(self._box.get('foo', False), False) self._box.close() self._box = self._factory(self._path) @@ -760,7 +760,7 @@ class TestMaildir(TestMailbox, unittest.TestCase): "tmp")), "File in wrong location: '%s'" % head) match = pattern.match(tail) - self.assertIsNot(match, None, "Invalid file name: '%s'" % tail) + self.assertIsNotNone(match, "Invalid file name: '%s'" % tail) groups = match.groups() if previous_groups is not None: self.assertGreaterEqual(int(groups[0]), int(previous_groups[0]), @@ -1394,7 +1394,7 @@ class TestMessage(TestBase, unittest.TestCase): self.assertIsInstance(msg, self._factory) self.assertEqual(msg.keys(), []) self.assertFalse(msg.is_multipart()) - self.assertEqual(msg.get_payload(), None) + self.assertIsNone(msg.get_payload()) def test_initialize_incorrectly(self): # Initialize with invalid argument @@ -1405,7 +1405,7 @@ class TestMessage(TestBase, unittest.TestCase): eMM = email.message_from_string(_sample_message) msg = self._factory(_sample_message) for attr in eMM.__dict__: - self.assertTrue(attr in msg.__dict__, + self.assertIn(attr, msg.__dict__, '{} attribute does not exist'.format(attr)) def test_become_message(self): @@ -1547,8 +1547,9 @@ class _TestMboxMMDFMessage: # Check contents of "From " line if sender is None: sender = "MAILER-DAEMON" - self.assertTrue(re.match(sender + r" \w{3} \w{3} [\d ]\d [\d ]\d:\d{2}:" - r"\d{2} \d{4}", msg.get_from()) is not None) + self.assertIsNotNone(re.match( + sender + r" \w{3} \w{3} [\d ]\d [\d ]\d:\d{2}:\d{2} \d{4}", + msg.get_from())) class TestMboxMessage(_TestMboxMMDFMessage, TestMessage): @@ -1622,19 +1623,19 @@ class TestBabylMessage(TestMessage, unittest.TestCase): msg = mailbox.BabylMessage(_sample_message) visible = msg.get_visible() self.assertEqual(visible.keys(), []) - self.assertIs(visible.get_payload(), None) + self.assertIsNone(visible.get_payload()) visible['User-Agent'] = 'FooBar 1.0' visible['X-Whatever'] = 'Blah' self.assertEqual(msg.get_visible().keys(), []) msg.set_visible(visible) visible = msg.get_visible() - self.assertTrue(visible.keys() == ['User-Agent', 'X-Whatever']) - self.assertTrue(visible['User-Agent'] == 'FooBar 1.0') + self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever']) + self.assertEqual(visible['User-Agent'], 'FooBar 1.0') self.assertEqual(visible['X-Whatever'], 'Blah') - self.assertIs(visible.get_payload(), None) + self.assertIsNone(visible.get_payload()) msg.update_visible() self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever']) - self.assertIs(visible.get_payload(), None) + self.assertIsNone(visible.get_payload()) visible = msg.get_visible() self.assertEqual(visible.keys(), ['User-Agent', 'Date', 'From', 'To', 'Subject']) @@ -2156,34 +2157,34 @@ class MaildirTestCase(unittest.TestCase): self.mbox = mailbox.Maildir(support.TESTFN) #self.assertTrue(hasattr(self.mbox, "boxes")) #self.assertEqual(len(self.mbox.boxes), 0) - self.assertIs(self.mbox.next(), None) - self.assertIs(self.mbox.next(), None) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) def test_nonempty_maildir_cur(self): self.createMessage("cur") self.mbox = mailbox.Maildir(support.TESTFN) #self.assertEqual(len(self.mbox.boxes), 1) - self.assertIsNot(self.mbox.next(), None) - self.assertIs(self.mbox.next(), None) - self.assertIs(self.mbox.next(), None) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) def test_nonempty_maildir_new(self): self.createMessage("new") self.mbox = mailbox.Maildir(support.TESTFN) #self.assertEqual(len(self.mbox.boxes), 1) - self.assertIsNot(self.mbox.next(), None) - self.assertIs(self.mbox.next(), None) - self.assertIs(self.mbox.next(), None) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) def test_nonempty_maildir_both(self): self.createMessage("cur") self.createMessage("new") self.mbox = mailbox.Maildir(support.TESTFN) #self.assertEqual(len(self.mbox.boxes), 2) - self.assertIsNot(self.mbox.next(), None) - self.assertIsNot(self.mbox.next(), None) - self.assertIs(self.mbox.next(), None) - self.assertIs(self.mbox.next(), None) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) ## End: tests from the original module (for backward compatibility). diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index d611a3138b..4fa9a195f9 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -536,6 +536,17 @@ class TextIOTestMixin: self.assertIsNone(memio.errors) self.assertFalse(memio.line_buffering) + def test_newline_default(self): + memio = self.ioclass("a\nb\r\nc\rd") + self.assertEqual(list(memio), ["a\n", "b\r\n", "c\rd"]) + self.assertEqual(memio.getvalue(), "a\nb\r\nc\rd") + + memio = self.ioclass() + self.assertEqual(memio.write("a\nb\r\nc\rd"), 8) + memio.seek(0) + self.assertEqual(list(memio), ["a\n", "b\r\n", "c\rd"]) + self.assertEqual(memio.getvalue(), "a\nb\r\nc\rd") + def test_newline_none(self): # newline=None memio = self.ioclass("a\nb\r\nc\rd", newline=None) @@ -545,12 +556,16 @@ class TextIOTestMixin: self.assertEqual(memio.read(2), "\nb") self.assertEqual(memio.read(2), "\nc") self.assertEqual(memio.read(1), "\n") + self.assertEqual(memio.getvalue(), "a\nb\nc\nd") + memio = self.ioclass(newline=None) self.assertEqual(2, memio.write("a\n")) self.assertEqual(3, memio.write("b\r\n")) self.assertEqual(3, memio.write("c\rd")) memio.seek(0) self.assertEqual(memio.read(), "a\nb\nc\nd") + self.assertEqual(memio.getvalue(), "a\nb\nc\nd") + memio = self.ioclass("a\r\nb", newline=None) self.assertEqual(memio.read(3), "a\nb") @@ -562,6 +577,8 @@ class TextIOTestMixin: self.assertEqual(memio.read(4), "a\nb\r") self.assertEqual(memio.read(2), "\nc") self.assertEqual(memio.read(1), "\r") + self.assertEqual(memio.getvalue(), "a\nb\r\nc\rd") + memio = self.ioclass(newline="") self.assertEqual(2, memio.write("a\n")) self.assertEqual(2, memio.write("b\r")) @@ -569,11 +586,19 @@ class TextIOTestMixin: self.assertEqual(2, memio.write("\rd")) memio.seek(0) self.assertEqual(list(memio), ["a\n", "b\r\n", "c\r", "d"]) + self.assertEqual(memio.getvalue(), "a\nb\r\nc\rd") def test_newline_lf(self): # newline="\n" - memio = self.ioclass("a\nb\r\nc\rd") + memio = self.ioclass("a\nb\r\nc\rd", newline="\n") self.assertEqual(list(memio), ["a\n", "b\r\n", "c\rd"]) + self.assertEqual(memio.getvalue(), "a\nb\r\nc\rd") + + memio = self.ioclass(newline="\n") + self.assertEqual(memio.write("a\nb\r\nc\rd"), 8) + memio.seek(0) + self.assertEqual(list(memio), ["a\n", "b\r\n", "c\rd"]) + self.assertEqual(memio.getvalue(), "a\nb\r\nc\rd") def test_newline_cr(self): # newline="\r" @@ -581,6 +606,15 @@ class TextIOTestMixin: self.assertEqual(memio.read(), "a\rb\r\rc\rd") memio.seek(0) self.assertEqual(list(memio), ["a\r", "b\r", "\r", "c\r", "d"]) + self.assertEqual(memio.getvalue(), "a\rb\r\rc\rd") + + memio = self.ioclass(newline="\r") + self.assertEqual(memio.write("a\nb\r\nc\rd"), 8) + memio.seek(0) + self.assertEqual(list(memio), ["a\r", "b\r", "\r", "c\r", "d"]) + memio.seek(0) + self.assertEqual(memio.readlines(), ["a\r", "b\r", "\r", "c\r", "d"]) + self.assertEqual(memio.getvalue(), "a\rb\r\rc\rd") def test_newline_crlf(self): # newline="\r\n" @@ -588,11 +622,21 @@ class TextIOTestMixin: self.assertEqual(memio.read(), "a\r\nb\r\r\nc\rd") memio.seek(0) self.assertEqual(list(memio), ["a\r\n", "b\r\r\n", "c\rd"]) + memio.seek(0) + self.assertEqual(memio.readlines(), ["a\r\n", "b\r\r\n", "c\rd"]) + self.assertEqual(memio.getvalue(), "a\r\nb\r\r\nc\rd") + + memio = self.ioclass(newline="\r\n") + self.assertEqual(memio.write("a\nb\r\nc\rd"), 8) + memio.seek(0) + self.assertEqual(list(memio), ["a\r\n", "b\r\r\n", "c\rd"]) + self.assertEqual(memio.getvalue(), "a\r\nb\r\r\nc\rd") def test_issue5265(self): # StringIO can duplicate newlines in universal newlines mode memio = self.ioclass("a\r\nb\r\n", newline=None) self.assertEqual(memio.read(5), "a\nb\n") + self.assertEqual(memio.getvalue(), "a\nb\n") def test_newline_argument(self): self.assertRaises(TypeError, self.ioclass, newline=b"\n") @@ -609,6 +653,15 @@ class PyStringIOTest(MemoryTestMixin, MemorySeekTestMixin, UnsupportedOperation = pyio.UnsupportedOperation EOF = "" + def test_lone_surrogates(self): + # Issue #20424 + memio = self.ioclass('\ud800') + self.assertEqual(memio.read(), '\ud800') + + memio = self.ioclass() + memio.write('\ud800') + self.assertEqual(memio.getvalue(), '\ud800') + class PyStringIOPickleTest(TextIOTestMixin, unittest.TestCase): """Test if pickle restores properly the internal state of StringIO. diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index b0f3011e31..2c4d27e6f5 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -126,10 +126,7 @@ class TestNtpath(unittest.TestCase): tester('ntpath.join("/a")', '/a') tester('ntpath.join("\\a")', '\\a') tester('ntpath.join("a:")', 'a:') - tester('ntpath.join("a:", "b")', 'a:b') - tester('ntpath.join("a:", "/b")', 'a:/b') tester('ntpath.join("a:", "\\b")', 'a:\\b') - tester('ntpath.join("a", "/b")', '/b') tester('ntpath.join("a", "\\b")', '\\b') tester('ntpath.join("a", "b", "c")', 'a\\b\\c') tester('ntpath.join("a\\", "b", "c")', 'a\\b\\c') @@ -137,42 +134,48 @@ class TestNtpath(unittest.TestCase): tester('ntpath.join("a", "b", "\\c")', '\\c') tester('ntpath.join("d:\\", "\\pleep")', 'd:\\pleep') tester('ntpath.join("d:\\", "a", "b")', 'd:\\a\\b') - tester("ntpath.join('c:', '/a')", 'c:/a') - tester("ntpath.join('c:/', '/a')", 'c:/a') - tester("ntpath.join('c:/a', '/b')", '/b') - tester("ntpath.join('c:', 'd:/')", 'd:/') - tester("ntpath.join('c:/', 'd:/')", 'd:/') - tester("ntpath.join('c:/', 'd:/a/b')", 'd:/a/b') - - tester("ntpath.join('')", '') - tester("ntpath.join('', '', '', '', '')", '') - tester("ntpath.join('a')", 'a') + tester("ntpath.join('', 'a')", 'a') tester("ntpath.join('', '', '', '', 'a')", 'a') tester("ntpath.join('a', '')", 'a\\') tester("ntpath.join('a', '', '', '', '')", 'a\\') tester("ntpath.join('a\\', '')", 'a\\') tester("ntpath.join('a\\', '', '', '', '')", 'a\\') + tester("ntpath.join('a/', '')", 'a/') + + tester("ntpath.join('a/b', 'x/y')", 'a/b\\x/y') + tester("ntpath.join('/a/b', 'x/y')", '/a/b\\x/y') + tester("ntpath.join('/a/b/', 'x/y')", '/a/b/x/y') + tester("ntpath.join('c:', 'x/y')", 'c:x/y') + tester("ntpath.join('c:a/b', 'x/y')", 'c:a/b\\x/y') + tester("ntpath.join('c:a/b/', 'x/y')", 'c:a/b/x/y') + tester("ntpath.join('c:/', 'x/y')", 'c:/x/y') + tester("ntpath.join('c:/a/b', 'x/y')", 'c:/a/b\\x/y') + tester("ntpath.join('c:/a/b/', 'x/y')", 'c:/a/b/x/y') + tester("ntpath.join('//computer/share', 'x/y')", '//computer/share\\x/y') + tester("ntpath.join('//computer/share/', 'x/y')", '//computer/share/x/y') + tester("ntpath.join('//computer/share/a/b', 'x/y')", '//computer/share/a/b\\x/y') + + tester("ntpath.join('a/b', '/x/y')", '/x/y') + tester("ntpath.join('/a/b', '/x/y')", '/x/y') + tester("ntpath.join('c:', '/x/y')", 'c:/x/y') + tester("ntpath.join('c:a/b', '/x/y')", 'c:/x/y') + tester("ntpath.join('c:/', '/x/y')", 'c:/x/y') + tester("ntpath.join('c:/a/b', '/x/y')", 'c:/x/y') + tester("ntpath.join('//computer/share', '/x/y')", '//computer/share/x/y') + tester("ntpath.join('//computer/share/', '/x/y')", '//computer/share/x/y') + tester("ntpath.join('//computer/share/a', '/x/y')", '//computer/share/x/y') + + tester("ntpath.join('c:', 'C:x/y')", 'C:x/y') + tester("ntpath.join('c:a/b', 'C:x/y')", 'C:a/b\\x/y') + tester("ntpath.join('c:/', 'C:x/y')", 'C:/x/y') + tester("ntpath.join('c:/a/b', 'C:x/y')", 'C:/a/b\\x/y') - # from comment in ntpath.join - tester("ntpath.join('c:', '/a')", 'c:/a') - tester("ntpath.join('//computer/share', '/a')", '//computer/share/a') - tester("ntpath.join('c:/', '/a')", 'c:/a') - tester("ntpath.join('//computer/share/', '/a')", '//computer/share/a') - tester("ntpath.join('c:/a', '/b')", '/b') - tester("ntpath.join('//computer/share/a', '/b')", '/b') - tester("ntpath.join('c:', 'd:/')", 'd:/') - tester("ntpath.join('c:', '//computer/share/')", '//computer/share/') - tester("ntpath.join('//computer/share', 'd:/')", 'd:/') - tester("ntpath.join('//computer/share', '//computer/share/')", '//computer/share/') - tester("ntpath.join('c:/', 'd:/')", 'd:/') - tester("ntpath.join('c:/', '//computer/share/')", '//computer/share/') - tester("ntpath.join('//computer/share/', 'd:/')", 'd:/') - tester("ntpath.join('//computer/share/', '//computer/share/')", '//computer/share/') - - tester("ntpath.join('c:', '//computer/share/')", '//computer/share/') - tester("ntpath.join('c:/', '//computer/share/')", '//computer/share/') - tester("ntpath.join('c:/', '//computer/share/a/b')", '//computer/share/a/b') + for x in ('', 'a/b', '/a/b', 'c:', 'c:a/b', 'c:/', 'c:/a/b', + '//computer/share', '//computer/share/', '//computer/share/a/b'): + for y in ('d:', 'd:x/y', 'd:/', 'd:/x/y', + '//machine/common', '//machine/common/', '//machine/common/x/y'): + tester("ntpath.join(%r, %r)" % (x, y), y) tester("ntpath.join('\\\\computer\\share\\', 'a', 'b')", '\\\\computer\\share\\a\\b') tester("ntpath.join('\\\\computer\\share', 'a', 'b')", '\\\\computer\\share\\a\\b') diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 3c86ef639f..a07a7199c1 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -3,14 +3,13 @@ import os import random import select -from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX try: import threading except ImportError: threading = None import time import unittest -from test.support import TESTFN, run_unittest, reap_threads +from test.support import TESTFN, run_unittest, reap_threads, cpython_only try: select.poll @@ -161,8 +160,18 @@ class PollTests(unittest.TestCase): # Issues #15989, #17919 self.assertRaises(OverflowError, pollster.register, 0, -1) - self.assertRaises(OverflowError, pollster.register, 0, USHRT_MAX + 1) + self.assertRaises(OverflowError, pollster.register, 0, 1 << 64) self.assertRaises(OverflowError, pollster.modify, 1, -1) + self.assertRaises(OverflowError, pollster.modify, 1, 1 << 64) + + @cpython_only + def test_poll_c_limits(self): + from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + pollster = select.poll() + pollster.register(1) + + # Issues #15989, #17919 + self.assertRaises(OverflowError, pollster.register, 0, USHRT_MAX + 1) self.assertRaises(OverflowError, pollster.modify, 1, USHRT_MAX + 1) self.assertRaises(OverflowError, pollster.poll, INT_MAX + 1) self.assertRaises(OverflowError, pollster.poll, UINT_MAX + 1) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index cbce8524eb..5809be66bb 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -350,7 +350,7 @@ class TestTimeouts(TestCase): serv.close() def testTimeoutDefault(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: pop = poplib.POP3(HOST, self.port) @@ -360,13 +360,13 @@ class TestTimeouts(TestCase): pop.sock.close() def testTimeoutNone(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: pop = poplib.POP3(HOST, self.port, timeout=None) finally: socket.setdefaulttimeout(None) - self.assertTrue(pop.sock.gettimeout() is None) + self.assertIsNone(pop.sock.gettimeout()) pop.sock.close() def testTimeoutValue(self): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 60806aacf9..9408e5c1b5 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -17,7 +17,6 @@ import stat import tempfile import unittest import warnings -import _testcapi _DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(), support.TESTFN + '-dummy-symlink') @@ -615,7 +614,12 @@ class PosixTester(unittest.TestCase): except OSError: pass + @support.cpython_only + @unittest.skipUnless(hasattr(os, 'pipe2'), "test needs os.pipe2()") + @support.requires_linux_version(2, 6, 27) + def test_pipe2_c_limits(self): # Issue 15989 + import _testcapi self.assertRaises(OverflowError, os.pipe2, _testcapi.INT_MAX + 1) self.assertRaises(OverflowError, os.pipe2, _testcapi.UINT_MAX + 1) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index f093812442..1c6f45dfa6 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,5 +1,5 @@ from test.support import verbose, run_unittest, gc_collect, bigmemtest, _2G, \ - cpython_only + cpython_only, captured_stdout import io import re from re import Scanner @@ -1064,6 +1064,19 @@ class ReTests(unittest.TestCase): self.assertEqual(m.group(1), "") self.assertEqual(m.group(2), "y") + def test_debug_flag(self): + with captured_stdout() as out: + re.compile('foo', re.DEBUG) + self.assertEqual(out.getvalue().splitlines(), + ['literal 102 ', 'literal 111 ', 'literal 111 ']) + # Debug output is output again even a second time (bypassing + # the cache -- issue #20426). + with captured_stdout() as out: + re.compile('foo', re.DEBUG) + self.assertEqual(out.getvalue().splitlines(), + ['literal 102 ', 'literal 111 ', 'literal 111 ']) + + def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR if verbose: diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index d798068056..3bbc6b7e53 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -96,7 +96,7 @@ class GeneralTests(unittest.TestCase): def testTimeoutDefault(self): mock_socket.reply_with(b"220 Hola mundo") - self.assertTrue(mock_socket.getdefaulttimeout() is None) + self.assertIsNone(mock_socket.getdefaulttimeout()) mock_socket.setdefaulttimeout(30) self.assertEqual(mock_socket.getdefaulttimeout(), 30) try: @@ -108,13 +108,13 @@ class GeneralTests(unittest.TestCase): def testTimeoutNone(self): mock_socket.reply_with(b"220 Hola mundo") - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: smtp = smtplib.SMTP(HOST, self.port, timeout=None) finally: socket.setdefaulttimeout(None) - self.assertTrue(smtp.sock.gettimeout() is None) + self.assertIsNone(smtp.sock.gettimeout()) smtp.close() def testTimeoutValue(self): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index f696fe17cd..67ff1beede 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7,7 +7,6 @@ import io import socket import select import tempfile -import _testcapi import time import traceback import queue @@ -1274,7 +1273,10 @@ class GeneralModuleTests(unittest.TestCase): srv.listen(backlog) srv.close() + @support.cpython_only + def test_listen_backlog_overflow(self): # Issue 15989 + import _testcapi srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.bind((HOST, 0)) self.assertRaises(OverflowError, srv.listen, _testcapi.INT_MAX + 1) @@ -1593,6 +1595,14 @@ class BasicTCPTest(SocketConnectedTest): def _testShutdown(self): self.serv_conn.send(MSG) + self.serv_conn.shutdown(2) + + testShutdown_overflow = support.cpython_only(testShutdown) + + @support.cpython_only + def _testShutdown_overflow(self): + import _testcapi + self.serv_conn.send(MSG) # Issue 15989 self.assertRaises(OverflowError, self.serv_conn.shutdown, _testcapi.INT_MAX + 1) @@ -2361,7 +2371,12 @@ class CmsgMacroTests(unittest.TestCase): # code with these functions. # Match the definition in socketmodule.c - socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + try: + import _testcapi + except ImportError: + socklen_t_limit = 0x7fffffff + else: + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) @requireAttrs(socket, "CMSG_LEN") def testCMSG_LEN(self): @@ -3490,12 +3505,12 @@ class InterruptedSendTimeoutTest(InterruptedTimeoutBase, self.assertNotIsInstance(cm.exception, socket.timeout) self.assertEqual(cm.exception.errno, errno.EINTR) - # Issue #12958: The following tests have problems on Mac OS X - @support.anticipate_failure(sys.platform == "darwin") + # Issue #12958: The following tests have problems on OS X prior to 10.7 + @support.requires_mac_ver(10, 7) def testInterruptedSendTimeout(self): self.checkInterruptedSend(self.serv_conn.send, b"a"*512) - @support.anticipate_failure(sys.platform == "darwin") + @support.requires_mac_ver(10, 7) def testInterruptedSendtoTimeout(self): # Passing an actual address here as Python's wrapper for # sendto() doesn't allow passing a zero-length one; POSIX @@ -3504,7 +3519,7 @@ class InterruptedSendTimeoutTest(InterruptedTimeoutBase, self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, self.serv_addr) - @support.anticipate_failure(sys.platform == "darwin") + @support.requires_mac_ver(10, 7) @requireAttrs(socket.socket, "sendmsg") def testInterruptedSendmsgTimeout(self): self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) @@ -3586,14 +3601,23 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): pass end = time.time() self.assertTrue((end - start) < 1.0, "Error setting non-blocking mode.") - # Issue 15989 - if _testcapi.UINT_MAX < _testcapi.ULONG_MAX: - self.serv.setblocking(_testcapi.UINT_MAX + 1) - self.assertIsNone(self.serv.gettimeout()) def _testSetBlocking(self): pass + @support.cpython_only + def testSetBlocking_overflow(self): + # Issue 15989 + import _testcapi + if _testcapi.UINT_MAX >= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = support.cpython_only(_testSetBlocking) + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'test needs socket.SOCK_NONBLOCK') @support.requires_linux_version(2, 6, 28) diff --git a/Lib/test/test_structmembers.py b/Lib/test/test_structmembers.py index 18fecc3829..1c931ae778 100644 --- a/Lib/test/test_structmembers.py +++ b/Lib/test/test_structmembers.py @@ -1,3 +1,8 @@ +import unittest +from test import support + +# Skip this test if the _testcapi module isn't available. +support.import_module('_testcapi') from _testcapi import _test_structmembersType, \ CHAR_MAX, CHAR_MIN, UCHAR_MAX, \ SHRT_MAX, SHRT_MIN, USHRT_MAX, \ @@ -6,9 +11,6 @@ from _testcapi import _test_structmembersType, \ LLONG_MAX, LLONG_MIN, ULLONG_MAX, \ PY_SSIZE_T_MAX, PY_SSIZE_T_MIN -import unittest -from test import support - ts=_test_structmembersType(False, # T_BOOL 1, # T_BYTE 2, # T_UBYTE diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index c6cca269b8..8808c47edd 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -42,7 +42,6 @@ class TestUntestedModules(unittest.TestCase): import encodings import formatter import html.entities - import imghdr import keyword import mailcap import nturl2path diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index acf6d364f2..c1523dbe3a 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -610,6 +610,7 @@ class SysModuleTest(unittest.TestCase): ret, out, err = assert_python_ok(*args) self.assertIn(b"free PyDictObjects", err) +@test.support.cpython_only class SizeofTest(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index f64964dd1d..ad6f320894 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -217,6 +217,84 @@ class LzmaUstarReadTest(LzmaTest, UstarReadTest): pass +class ListTest(ReadTest, unittest.TestCase): + + # Override setUp to use default encoding (UTF-8) + def setUp(self): + self.tar = tarfile.open(self.tarname, mode=self.mode) + + def test_list(self): + tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') + with support.swap_attr(sys, 'stdout', tio): + self.tar.list(verbose=False) + out = tio.detach().getvalue() + self.assertIn(b'ustar/conttype', out) + self.assertIn(b'ustar/regtype', out) + self.assertIn(b'ustar/lnktype', out) + self.assertIn(b'ustar' + (b'/12345' * 40) + b'67/longname', out) + self.assertIn(b'./ustar/linktest2/symtype', out) + self.assertIn(b'./ustar/linktest2/lnktype', out) + # Make sure it puts trailing slash for directory + self.assertIn(b'ustar/dirtype/', out) + self.assertIn(b'ustar/dirtype-with-size/', out) + # Make sure it is able to print unencodable characters + self.assertIn(br'ustar/umlauts-' + br'\udcc4\udcd6\udcdc\udce4\udcf6\udcfc\udcdf', out) + self.assertIn(br'misc/regtype-hpux-signed-chksum-' + br'\udcc4\udcd6\udcdc\udce4\udcf6\udcfc\udcdf', out) + self.assertIn(br'misc/regtype-old-v7-signed-chksum-' + br'\udcc4\udcd6\udcdc\udce4\udcf6\udcfc\udcdf', out) + self.assertIn(br'pax/bad-pax-\udce4\udcf6\udcfc', out) + self.assertIn(br'pax/hdrcharset-\udce4\udcf6\udcfc', out) + # Make sure it prints files separated by one newline without any + # 'ls -l'-like accessories if verbose flag is not being used + # ... + # ustar/conttype + # ustar/regtype + # ... + self.assertRegex(out, br'ustar/conttype ?\r?\n' + br'ustar/regtype ?\r?\n') + # Make sure it does not print the source of link without verbose flag + self.assertNotIn(b'link to', out) + self.assertNotIn(b'->', out) + + def test_list_verbose(self): + tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') + with support.swap_attr(sys, 'stdout', tio): + self.tar.list(verbose=True) + out = tio.detach().getvalue() + # Make sure it prints files separated by one newline with 'ls -l'-like + # accessories if verbose flag is being used + # ... + # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/conttype + # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype + # ... + self.assertRegex(out, (br'-rw-r--r-- tarfile/tarfile\s+7011 ' + br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d ' + br'ustar/\w+type ?\r?\n') * 2) + # Make sure it prints the source of link with verbose flag + self.assertIn(b'ustar/symtype -> regtype', out) + self.assertIn(b'./ustar/linktest2/symtype -> ../linktest1/regtype', out) + self.assertIn(b'./ustar/linktest2/lnktype link to ' + b'./ustar/linktest1/regtype', out) + self.assertIn(b'gnu' + (b'/123' * 125) + b'/longlink link to gnu' + + (b'/123' * 125) + b'/longname', out) + self.assertIn(b'pax' + (b'/123' * 125) + b'/longlink link to pax' + + (b'/123' * 125) + b'/longname', out) + + +class GzipListTest(GzipTest, ListTest): + pass + + +class Bz2ListTest(Bz2Test, ListTest): + pass + + +class LzmaListTest(LzmaTest, ListTest): + pass + + class CommonReadTest(ReadTest): def test_empty_tarfile(self): diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 0520c0944a..dda0c7ac79 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -1,7 +1,6 @@ import unittest import sys import os -import _testcapi from test import support # Skip this test if the _tkinter module wasn't built. @@ -13,6 +12,11 @@ support.import_fresh_module('tkinter') from tkinter import Tcl from _tkinter import TclError +try: + from _testcapi import INT_MAX, PY_SSIZE_T_MAX +except ImportError: + INT_MAX = PY_SSIZE_T_MAX = sys.maxsize + tcl_version = _tkinter.TCL_VERSION.split('.') try: for i in range(len(tcl_version)): @@ -21,6 +25,21 @@ except ValueError: pass tcl_version = tuple(tcl_version) +_tk_patchlevel = None +def get_tk_patchlevel(): + global _tk_patchlevel + if _tk_patchlevel is None: + tcl = Tcl() + patchlevel = [] + for x in tcl.call('info', 'patchlevel').split('.'): + try: + x = int(x, 10) + except ValueError: + x = -1 + patchlevel.append(x) + _tk_patchlevel = tuple(patchlevel) + return _tk_patchlevel + class TkinterTest(unittest.TestCase): @@ -40,6 +59,10 @@ class TclTest(unittest.TestCase): tcl.eval('set a 1') self.assertEqual(tcl.eval('set a'),'1') + def test_eval_null_in_result(self): + tcl = self.interp + self.assertEqual(tcl.eval('set a "a\\0b"'), 'a\x00b') + def testEvalException(self): tcl = self.interp self.assertRaises(TclError,tcl.eval,'set a') @@ -112,20 +135,29 @@ class TclTest(unittest.TestCase): def testEvalFile(self): tcl = self.interp - filename = "testEvalFile.tcl" - fd = open(filename,'w') - script = """set a 1 - set b 2 - set c [ expr $a + $b ] - """ - fd.write(script) - fd.close() - tcl.evalfile(filename) - os.remove(filename) + with open(support.TESTFN, 'w') as f: + self.addCleanup(support.unlink, support.TESTFN) + f.write("""set a 1 + set b 2 + set c [ expr $a + $b ] + """) + tcl.evalfile(support.TESTFN) self.assertEqual(tcl.eval('set a'),'1') self.assertEqual(tcl.eval('set b'),'2') self.assertEqual(tcl.eval('set c'),'3') + def test_evalfile_null_in_result(self): + tcl = self.interp + with open(support.TESTFN, 'w') as f: + self.addCleanup(support.unlink, support.TESTFN) + f.write(""" + set a "a\0b" + set b "a\\0b" + """) + tcl.evalfile(support.TESTFN) + self.assertEqual(tcl.eval('set a'), 'a\x00b') + self.assertEqual(tcl.eval('set b'), 'a\x00b') + def testEvalFileException(self): tcl = self.interp filename = "doesnotexists" @@ -162,6 +194,146 @@ class TclTest(unittest.TestCase): # exit code must be zero self.assertEqual(f.close(), None) + def test_exprstring(self): + tcl = self.interp + tcl.call('set', 'a', 3) + tcl.call('set', 'b', 6) + def check(expr, expected): + result = tcl.exprstring(expr) + self.assertEqual(result, expected) + self.assertIsInstance(result, str) + + self.assertRaises(TypeError, tcl.exprstring) + self.assertRaises(TypeError, tcl.exprstring, '8.2', '+6') + self.assertRaises(TypeError, tcl.exprstring, b'8.2 + 6') + self.assertRaises(TclError, tcl.exprstring, 'spam') + check('', '0') + check('8.2 + 6', '14.2') + check('3.1 + $a', '6.1') + check('2 + "$a.$b"', '5.6') + check('4*[llength "6 2"]', '8') + check('{word one} < "word $a"', '0') + check('4*2 < 7', '0') + check('hypot($a, 4)', '5.0') + check('5 / 4', '1') + check('5 / 4.0', '1.25') + check('5 / ( [string length "abcd"] + 0.0 )', '1.25') + check('20.0/5.0', '4.0') + check('"0x03" > "2"', '1') + check('[string length "a\xbd\u20ac"]', '3') + check(r'[string length "a\xbd\u20ac"]', '3') + check('"abc"', 'abc') + check('"a\xbd\u20ac"', 'a\xbd\u20ac') + check(r'"a\xbd\u20ac"', 'a\xbd\u20ac') + check(r'"a\0b"', 'a\x00b') + if tcl_version >= (8, 5): + check('2**64', str(2**64)) + + def test_exprdouble(self): + tcl = self.interp + tcl.call('set', 'a', 3) + tcl.call('set', 'b', 6) + def check(expr, expected): + result = tcl.exprdouble(expr) + self.assertEqual(result, expected) + self.assertIsInstance(result, float) + + self.assertRaises(TypeError, tcl.exprdouble) + self.assertRaises(TypeError, tcl.exprdouble, '8.2', '+6') + self.assertRaises(TypeError, tcl.exprdouble, b'8.2 + 6') + self.assertRaises(TclError, tcl.exprdouble, 'spam') + check('', 0.0) + check('8.2 + 6', 14.2) + check('3.1 + $a', 6.1) + check('2 + "$a.$b"', 5.6) + check('4*[llength "6 2"]', 8.0) + check('{word one} < "word $a"', 0.0) + check('4*2 < 7', 0.0) + check('hypot($a, 4)', 5.0) + check('5 / 4', 1.0) + check('5 / 4.0', 1.25) + check('5 / ( [string length "abcd"] + 0.0 )', 1.25) + check('20.0/5.0', 4.0) + check('"0x03" > "2"', 1.0) + check('[string length "a\xbd\u20ac"]', 3.0) + check(r'[string length "a\xbd\u20ac"]', 3.0) + self.assertRaises(TclError, tcl.exprdouble, '"abc"') + if tcl_version >= (8, 5): + check('2**64', float(2**64)) + + def test_exprlong(self): + tcl = self.interp + tcl.call('set', 'a', 3) + tcl.call('set', 'b', 6) + def check(expr, expected): + result = tcl.exprlong(expr) + self.assertEqual(result, expected) + self.assertIsInstance(result, int) + + self.assertRaises(TypeError, tcl.exprlong) + self.assertRaises(TypeError, tcl.exprlong, '8.2', '+6') + self.assertRaises(TypeError, tcl.exprlong, b'8.2 + 6') + self.assertRaises(TclError, tcl.exprlong, 'spam') + check('', 0) + check('8.2 + 6', 14) + check('3.1 + $a', 6) + check('2 + "$a.$b"', 5) + check('4*[llength "6 2"]', 8) + check('{word one} < "word $a"', 0) + check('4*2 < 7', 0) + check('hypot($a, 4)', 5) + check('5 / 4', 1) + check('5 / 4.0', 1) + check('5 / ( [string length "abcd"] + 0.0 )', 1) + check('20.0/5.0', 4) + check('"0x03" > "2"', 1) + check('[string length "a\xbd\u20ac"]', 3) + check(r'[string length "a\xbd\u20ac"]', 3) + self.assertRaises(TclError, tcl.exprlong, '"abc"') + if tcl_version >= (8, 5): + self.assertRaises(TclError, tcl.exprlong, '2**64') + + def test_exprboolean(self): + tcl = self.interp + tcl.call('set', 'a', 3) + tcl.call('set', 'b', 6) + def check(expr, expected): + result = tcl.exprboolean(expr) + self.assertEqual(result, expected) + self.assertIsInstance(result, int) + self.assertNotIsInstance(result, bool) + + self.assertRaises(TypeError, tcl.exprboolean) + self.assertRaises(TypeError, tcl.exprboolean, '8.2', '+6') + self.assertRaises(TypeError, tcl.exprboolean, b'8.2 + 6') + self.assertRaises(TclError, tcl.exprboolean, 'spam') + check('', False) + for value in ('0', 'false', 'no', 'off'): + check(value, False) + check('"%s"' % value, False) + check('{%s}' % value, False) + for value in ('1', 'true', 'yes', 'on'): + check(value, True) + check('"%s"' % value, True) + check('{%s}' % value, True) + check('8.2 + 6', True) + check('3.1 + $a', True) + check('2 + "$a.$b"', True) + check('4*[llength "6 2"]', True) + check('{word one} < "word $a"', False) + check('4*2 < 7', False) + check('hypot($a, 4)', True) + check('5 / 4', True) + check('5 / 4.0', True) + check('5 / ( [string length "abcd"] + 0.0 )', True) + check('20.0/5.0', True) + check('"0x03" > "2"', True) + check('[string length "a\xbd\u20ac"]', True) + check(r'[string length "a\xbd\u20ac"]', True) + self.assertRaises(TclError, tcl.exprboolean, '"abc"') + if tcl_version >= (8, 5): + check('2**64', True) + def test_passing_values(self): def passValue(value): return self.interp.call('set', '_', value) @@ -170,6 +342,11 @@ class TclTest(unittest.TestCase): self.assertEqual(passValue(False), False if self.wantobjects else '0') self.assertEqual(passValue('string'), 'string') self.assertEqual(passValue('string\u20ac'), 'string\u20ac') + self.assertEqual(passValue('str\x00ing'), 'str\x00ing') + self.assertEqual(passValue('str\x00ing\xbd'), 'str\x00ing\xbd') + self.assertEqual(passValue('str\x00ing\u20ac'), 'str\x00ing\u20ac') + self.assertEqual(passValue(b'str\x00ing'), 'str\x00ing') + self.assertEqual(passValue(b'str\xc0\x80ing'), 'str\x00ing') for i in (0, 1, -1, 2**31-1, -2**31): self.assertEqual(passValue(i), i if self.wantobjects else str(i)) for f in (0.0, 1.0, -1.0, 1/3, @@ -218,6 +395,13 @@ class TclTest(unittest.TestCase): check('string', 'string') check('string\xbd', 'string\xbd') check('string\u20ac', 'string\u20ac') + check(b'string', 'string') + check(b'string\xe2\x82\xac', 'string\u20ac') + check('str\x00ing', 'str\x00ing') + check('str\x00ing\xbd', 'str\x00ing\xbd') + check('str\x00ing\u20ac', 'str\x00ing\u20ac') + check(b'str\xc0\x80ing', 'str\x00ing') + check(b'str\xc0\x80ing\xe2\x82\xac', 'str\x00ing\u20ac') for i in (0, 1, -1, 2**31-1, -2**31): check(i, str(i)) for f in (0.0, 1.0, -1.0): @@ -246,6 +430,7 @@ class TclTest(unittest.TestCase): (b'a\n b\t\r c\n ', ('a', 'b', 'c')), ('a \u20ac', ('a', '\u20ac')), (b'a \xe2\x82\xac', ('a', '\u20ac')), + (b'a\xc0\x80b c\xc0\x80d', ('a\x00b', 'c\x00d')), ('a {b c}', ('a', 'b c')), (r'a b\ c', ('a', 'b c')), (('a', 'b c'), ('a', 'b c')), @@ -259,10 +444,14 @@ class TclTest(unittest.TestCase): ('1', '2', '3.4')), ] if tcl_version >= (8, 5): + if not self.wantobjects or get_tk_patchlevel() < (8, 5, 5): + # Before 8.5.5 dicts were converted to lists through string + expected = ('12', '\u20ac', '\u20ac', '3.4') + else: + expected = (12, '\u20ac', '\u20ac', (3.4,)) testcases += [ - (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (1, '\u20ac', '\u20ac', (3.4,)) if self.wantobjects else - ('1', '\u20ac', '\u20ac', '3.4')), + (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), + expected), ] for arg, res in testcases: self.assertEqual(splitlist(arg), res, msg=arg) @@ -284,6 +473,9 @@ class TclTest(unittest.TestCase): (b'a\n b\t\r c\n ', ('a', 'b', 'c')), ('a \u20ac', ('a', '\u20ac')), (b'a \xe2\x82\xac', ('a', '\u20ac')), + (b'a\xc0\x80b', 'a\x00b'), + (b'a\xc0\x80b c\xc0\x80d', ('a\x00b', 'c\x00d')), + (b'{a\xc0\x80b c\xc0\x80d', '{a\x00b c\x00d'), ('a {b c}', ('a', ('b', 'c'))), (r'a b\ c', ('a', ('b', 'c'))), (('a', b'b c'), ('a', ('b', 'c'))), @@ -299,10 +491,14 @@ class TclTest(unittest.TestCase): ('1', '2', '3.4')), ] if tcl_version >= (8, 5): + if not self.wantobjects or get_tk_patchlevel() < (8, 5, 5): + # Before 8.5.5 dicts were converted to lists through string + expected = ('12', '\u20ac', '\u20ac', '3.4') + else: + expected = (12, '\u20ac', '\u20ac', (3.4,)) testcases += [ (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (12, '\u20ac', '\u20ac', (3.4,)) if self.wantobjects else - ('12', '\u20ac', '\u20ac', '3.4')), + expected), ] for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) @@ -347,9 +543,9 @@ class BigmemTclTest(unittest.TestCase): def setUp(self): self.interp = Tcl() - @unittest.skipUnless(_testcapi.INT_MAX < _testcapi.PY_SSIZE_T_MAX, - "needs UINT_MAX < SIZE_MAX") - @support.bigmemtest(size=_testcapi.INT_MAX + 1, memuse=5, dry_run=False) + @support.cpython_only + @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, "needs UINT_MAX < SIZE_MAX") + @support.bigmemtest(size=INT_MAX + 1, memuse=5, dry_run=False) def test_huge_string(self, size): value = ' ' * size self.assertRaises(OverflowError, self.interp.call, 'set', '_', value) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index e708ce88ce..5a970355e6 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -11,7 +11,7 @@ import contextlib import weakref import unittest -from test import support +from test import support, script_helper if hasattr(os, 'stat'): @@ -1073,7 +1073,8 @@ class TestTemporaryDirectory(BaseTestCase): self.nameCheck(tmp.name, dir, pre, suf) # Create a subdirectory and some files if recurse: - self.do_create(tmp.name, pre, suf, recurse-1) + d1 = self.do_create(tmp.name, pre, suf, recurse-1) + d1.name = None with open(os.path.join(tmp.name, "test.txt"), "wb") as f: f.write(b"Hello world!") return tmp @@ -1105,7 +1106,7 @@ class TestTemporaryDirectory(BaseTestCase): def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) d1 = self.do_create() - d2 = self.do_create() + d2 = self.do_create(recurse=0) # Symlink d1/foo -> d2 os.symlink(d2.name, os.path.join(d1.name, "foo")) @@ -1135,60 +1136,49 @@ class TestTemporaryDirectory(BaseTestCase): finally: os.rmdir(dir) - @unittest.expectedFailure # See issue #10188 def test_del_on_shutdown(self): # A TemporaryDirectory may be cleaned up during shutdown - # Make sure it works with the relevant modules nulled out with self.do_create() as dir: - d = self.do_create(dir=dir) - # Mimic the nulling out of modules that - # occurs during system shutdown - modules = [os, os.path] - if has_stat: - modules.append(stat) - # Currently broken, so suppress the warning - # that is otherwise emitted on stdout - with support.captured_stderr() as err: - with NulledModules(*modules): - d.cleanup() - # Currently broken, so stop spurious exception by - # indicating the object has already been closed - d._closed = True - # And this assert will fail, as expected by the - # unittest decorator... - self.assertFalse(os.path.exists(d.name), - "TemporaryDirectory %s exists after cleanup" % d.name) + for mod in ('os', 'shutil', 'sys', 'tempfile', 'warnings'): + code = """if True: + import os + import shutil + import sys + import tempfile + import warnings + + tmp = tempfile.TemporaryDirectory(dir={dir!r}) + sys.stdout.buffer.write(tmp.name.encode()) + + tmp2 = os.path.join(tmp.name, 'test_dir') + os.mkdir(tmp2) + with open(os.path.join(tmp2, "test.txt"), "w") as f: + f.write("Hello world!") + + {mod}.tmp = tmp + + warnings.filterwarnings("always", category=ResourceWarning) + """.format(dir=dir, mod=mod) + rc, out, err = script_helper.assert_python_ok("-c", code) + tmp_name = out.decode().strip() + self.assertFalse(os.path.exists(tmp_name), + "TemporaryDirectory %s exists after cleanup" % tmp_name) + err = err.decode('utf-8', 'backslashreplace') + self.assertNotIn("Exception ", err) def test_warnings_on_cleanup(self): - # Two kinds of warning on shutdown - # Issue 10888: may write to stderr if modules are nulled out - # ResourceWarning will be triggered by __del__ + # ResourceWarning will be triggered by __del__ with self.do_create() as dir: - if os.sep != '\\': - # Embed a backslash in order to make sure string escaping - # in the displayed error message is dealt with correctly - suffix = '\\check_backslash_handling' - else: - suffix = '' - d = self.do_create(dir=dir, suf=suffix) - - #Check for the Issue 10888 message - modules = [os, os.path] - if has_stat: - modules.append(stat) - with support.captured_stderr() as err: - with NulledModules(*modules): - d.cleanup() - message = err.getvalue().replace('\\\\', '\\') - self.assertIn("while cleaning up", message) - self.assertIn(d.name, message) + d = self.do_create(dir=dir, recurse=3) + name = d.name # Check for the resource warning with support.check_warnings(('Implicitly', ResourceWarning), quiet=False): warnings.filterwarnings("always", category=ResourceWarning) - d.__del__() - self.assertFalse(os.path.exists(d.name), - "TemporaryDirectory %s exists after __del__" % d.name) + del d + support.gc_collect() + self.assertFalse(os.path.exists(name), + "TemporaryDirectory %s exists after __del__" % name) def test_multiple_close(self): # Can be cleaned-up many times without error diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index fee48a3910..11c8979517 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,7 +3,7 @@ Tests for the threading module. """ import test.support -from test.support import verbose, strip_python_stderr, import_module +from test.support import verbose, strip_python_stderr, import_module, cpython_only from test.script_helper import assert_python_ok import random @@ -773,6 +773,7 @@ class ThreadJoinOnShutdown(BaseTestCase): for t in threads: t.join() + @cpython_only @unittest.skipIf(_testcapi is None, "need _testcapi module") def test_frame_tstate_tracing(self): # Issue #14432: Crash when a generator is created in a C thread that is diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index ae3113a04a..1a4d873a57 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -573,6 +573,7 @@ class TestPytime(unittest.TestCase): -(2.0 ** 100.0), 2.0 ** 100.0, ) + @support.cpython_only def test_time_t(self): from _testcapi import pytime_object_to_time_t for obj, time_t in ( @@ -588,6 +589,7 @@ class TestPytime(unittest.TestCase): for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_time_t, invalid) + @support.cpython_only def test_timeval(self): from _testcapi import pytime_object_to_timeval for obj, timeval in ( @@ -607,6 +609,7 @@ class TestPytime(unittest.TestCase): for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_timeval, invalid) + @support.cpython_only def test_timespec(self): from _testcapi import pytime_object_to_timespec for obj, timespec in ( diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 9af3b924a1..c38c65bb31 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1,12 +1,11 @@ """Test cases for traceback module""" -from _testcapi import traceback_print, exception_print from io import StringIO import sys import unittest import re from test.support import run_unittest, Error, captured_output -from test.support import TESTFN, unlink +from test.support import TESTFN, unlink, cpython_only import traceback @@ -173,7 +172,9 @@ class SyntaxTracebackCases(unittest.TestCase): class TracebackFormatTests(unittest.TestCase): + @cpython_only def test_traceback_format(self): + from _testcapi import traceback_print try: raise KeyError('blah') except KeyError: @@ -360,7 +361,9 @@ class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # This checks built-in reporting by the interpreter. # + @cpython_only def get_report(self, e): + from _testcapi import exception_print e = self.get_exception(e) with captured_output("stderr") as s: exception_print(e) diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 2e6374561f..de71680c63 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -9,12 +9,16 @@ Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com) import unittest import unicodedata -import _testcapi from test import support from http.client import HTTPException from test.test_normalization import check_version +try: + from _testcapi import INT_MAX, PY_SSIZE_T_MAX, UINT_MAX +except ImportError: + INT_MAX = PY_SSIZE_T_MAX = UINT_MAX = 2**64 - 1 + class UnicodeNamesTest(unittest.TestCase): def checkletter(self, name, code): @@ -216,15 +220,13 @@ class UnicodeNamesTest(unittest.TestCase): str, b"\\NSPACE", 'unicode-escape', 'strict' ) - @unittest.skipUnless(_testcapi.INT_MAX < _testcapi.PY_SSIZE_T_MAX, - "needs UINT_MAX < SIZE_MAX") - @support.bigmemtest(size=_testcapi.UINT_MAX + 1, - memuse=2 + 1, dry_run=False) + @support.cpython_only + @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, "needs UINT_MAX < SIZE_MAX") + @support.bigmemtest(size=UINT_MAX + 1, memuse=2 + 1, dry_run=False) def test_issue16335(self, size): # very very long bogus character name - x = b'\\N{SPACE' + b'x' * (_testcapi.UINT_MAX + 1) + b'}' - self.assertEqual(len(x), len(b'\\N{SPACE}') + - (_testcapi.UINT_MAX + 1)) + x = b'\\N{SPACE' + b'x' * (UINT_MAX + 1) + b'}' + self.assertEqual(len(x), len(b'\\N{SPACE}') + (UINT_MAX + 1)) self.assertRaisesRegex(UnicodeError, 'unknown Unicode character name', x.decode, 'unicode-escape' diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index f6b2a3fd71..c2ede07a83 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1108,8 +1108,13 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('%.1s' % "a\xe9\u20ac", 'a') self.assertEqual('%.2s' % "a\xe9\u20ac", 'a\xe9') - @support.cpython_only def test_formatting_huge_precision(self): + format_string = "%.{}f".format(sys.maxsize + 1) + with self.assertRaises(ValueError): + result = format_string % 2.34 + + @support.cpython_only + def test_formatting_huge_precision_c_limits(self): from _testcapi import INT_MAX format_string = "%.{}f".format(INT_MAX + 1) with self.assertRaises(ValueError): @@ -2090,6 +2095,7 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual(PyUnicode_FromFormat(b'%.%s', b'abc'), '%.%s') # Test PyUnicode_AsWideChar() + @support.cpython_only def test_aswidechar(self): from _testcapi import unicode_aswidechar support.import_module('ctypes') @@ -2127,6 +2133,7 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual(wchar, nonbmp + '\0') # Test PyUnicode_AsWideCharString() + @support.cpython_only def test_aswidecharstring(self): from _testcapi import unicode_aswidecharstring support.import_module('ctypes') @@ -2161,6 +2168,7 @@ class UnicodeTest(string_tests.CommonTest, s += "4" self.assertEqual(s, "3") + @support.cpython_only def test_encode_decimal(self): from _testcapi import unicode_encodedecimal self.assertEqual(unicode_encodedecimal('123'), @@ -2176,6 +2184,7 @@ class UnicodeTest(string_tests.CommonTest, "^'decimal' codec can't encode character", unicode_encodedecimal, "123\u20ac", "replace") + @support.cpython_only def test_transform_decimal(self): from _testcapi import unicode_transformdecimaltoascii as transform_decimal self.assertEqual(transform_decimal('123'), diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 7a34a05d05..2dec4e9f88 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1264,7 +1264,7 @@ class URLopener_Tests(unittest.TestCase): # def testTimeoutNone(self): # # global default timeout is ignored # import socket -# self.assertTrue(socket.getdefaulttimeout() is None) +# self.assertIsNone(socket.getdefaulttimeout()) # socket.setdefaulttimeout(30) # try: # ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) @@ -1276,7 +1276,7 @@ class URLopener_Tests(unittest.TestCase): # def testTimeoutDefault(self): # # global default timeout is used # import socket -# self.assertTrue(socket.getdefaulttimeout() is None) +# self.assertIsNone(socket.getdefaulttimeout()) # socket.setdefaulttimeout(30) # try: # ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index d4c894534a..3559b1b789 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -594,8 +594,8 @@ class OpenerDirectorTests(unittest.TestCase): self.assertIsInstance(args[0], Request) # response from opener.open is None, because there's no # handler that defines http_open to handle it - self.assertTrue(args[1] is None or - isinstance(args[1], MockResponse)) + if args[1] is not None: + self.assertIsInstance(args[1], MockResponse) def test_method_deprecations(self): req = Request("http://www.example.com") @@ -1000,7 +1000,8 @@ class HandlerTests(unittest.TestCase): MockHeaders({"location": to_url})) except urllib.error.HTTPError: # 307 in response to POST requires user OK - self.assertTrue(code == 307 and data is not None) + self.assertEqual(code, 307) + self.assertIsNotNone(data) self.assertEqual(o.req.get_full_url(), to_url) try: self.assertEqual(o.req.get_method(), "GET") diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index a4c16224fb..4981c2410b 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -85,7 +85,7 @@ class CloseSocketTest(unittest.TestCase): with support.transient_internet(url): response = _urlopen_with_retry(url) sock = response.fp - self.assertTrue(not sock.closed) + self.assertFalse(sock.closed) response.close() self.assertTrue(sock.closed) @@ -252,15 +252,15 @@ class OtherNetworkTests(unittest.TestCase): class TimeoutTest(unittest.TestCase): def test_http_basic(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) url = "http://www.python.org" with support.transient_internet(url, timeout=None): u = _urlopen_with_retry(url) self.addCleanup(u.close) - self.assertTrue(u.fp.raw._sock.gettimeout() is None) + self.assertIsNone(u.fp.raw._sock.gettimeout()) def test_http_default_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) url = "http://www.python.org" with support.transient_internet(url): socket.setdefaulttimeout(60) @@ -272,7 +272,7 @@ class TimeoutTest(unittest.TestCase): self.assertEqual(u.fp.raw._sock.gettimeout(), 60) def test_http_no_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) url = "http://www.python.org" with support.transient_internet(url): socket.setdefaulttimeout(60) @@ -281,7 +281,7 @@ class TimeoutTest(unittest.TestCase): self.addCleanup(u.close) finally: socket.setdefaulttimeout(None) - self.assertTrue(u.fp.raw._sock.gettimeout() is None) + self.assertIsNone(u.fp.raw._sock.gettimeout()) def test_http_timeout(self): url = "http://www.python.org" @@ -293,14 +293,14 @@ class TimeoutTest(unittest.TestCase): FTP_HOST = "ftp://ftp.mirror.nl/pub/gnu/" def test_ftp_basic(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) with support.transient_internet(self.FTP_HOST, timeout=None): u = _urlopen_with_retry(self.FTP_HOST) self.addCleanup(u.close) - self.assertTrue(u.fp.fp.raw._sock.gettimeout() is None) + self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) def test_ftp_default_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) with support.transient_internet(self.FTP_HOST): socket.setdefaulttimeout(60) try: @@ -311,7 +311,7 @@ class TimeoutTest(unittest.TestCase): self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) def test_ftp_no_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) with support.transient_internet(self.FTP_HOST): socket.setdefaulttimeout(60) try: @@ -319,7 +319,7 @@ class TimeoutTest(unittest.TestCase): self.addCleanup(u.close) finally: socket.setdefaulttimeout(None) - self.assertTrue(u.fp.fp.raw._sock.gettimeout() is None) + self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) def test_ftp_timeout(self): with support.transient_internet(self.FTP_HOST): diff --git a/Lib/test/test_warnings.py b/Lib/test/test_warnings.py index 6ce0b23474..10076af3aa 100644 --- a/Lib/test/test_warnings.py +++ b/Lib/test/test_warnings.py @@ -4,7 +4,6 @@ import os from io import StringIO import sys import unittest -import subprocess from test import support from test.script_helper import assert_python_ok @@ -719,47 +718,34 @@ class PyCatchWarningTests(CatchWarningTests, unittest.TestCase): class EnvironmentVariableTests(BaseTest): def test_single_warning(self): - newenv = os.environ.copy() - newenv["PYTHONWARNINGS"] = "ignore::DeprecationWarning" - p = subprocess.Popen([sys.executable, - "-c", "import sys; sys.stdout.write(str(sys.warnoptions))"], - stdout=subprocess.PIPE, env=newenv) - self.assertEqual(p.communicate()[0], b"['ignore::DeprecationWarning']") - self.assertEqual(p.wait(), 0) + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning") + self.assertEqual(stdout, b"['ignore::DeprecationWarning']") def test_comma_separated_warnings(self): - newenv = os.environ.copy() - newenv["PYTHONWARNINGS"] = ("ignore::DeprecationWarning," - "ignore::UnicodeWarning") - p = subprocess.Popen([sys.executable, - "-c", "import sys; sys.stdout.write(str(sys.warnoptions))"], - stdout=subprocess.PIPE, env=newenv) - self.assertEqual(p.communicate()[0], - b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") - self.assertEqual(p.wait(), 0) + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning,ignore::UnicodeWarning") + self.assertEqual(stdout, + b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") def test_envvar_and_command_line(self): - newenv = os.environ.copy() - newenv["PYTHONWARNINGS"] = "ignore::DeprecationWarning" - p = subprocess.Popen([sys.executable, "-W" "ignore::UnicodeWarning", - "-c", "import sys; sys.stdout.write(str(sys.warnoptions))"], - stdout=subprocess.PIPE, env=newenv) - self.assertEqual(p.communicate()[0], - b"['ignore::UnicodeWarning', 'ignore::DeprecationWarning']") - self.assertEqual(p.wait(), 0) + rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning") + self.assertEqual(stdout, + b"['ignore::UnicodeWarning', 'ignore::DeprecationWarning']") @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii', 'requires non-ascii filesystemencoding') def test_nonascii(self): - newenv = os.environ.copy() - newenv["PYTHONWARNINGS"] = "ignore:DeprecaciónWarning" - newenv["PYTHONIOENCODING"] = "utf-8" - p = subprocess.Popen([sys.executable, - "-c", "import sys; sys.stdout.write(str(sys.warnoptions))"], - stdout=subprocess.PIPE, env=newenv) - self.assertEqual(p.communicate()[0], - "['ignore:DeprecaciónWarning']".encode('utf-8')) - self.assertEqual(p.wait(), 0) + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONIOENCODING="utf-8", + PYTHONWARNINGS="ignore:DeprecaciónWarning") + self.assertEqual(stdout, + "['ignore:DeprecaciónWarning']".encode('utf-8')) class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): module = c_warnings @@ -774,18 +760,11 @@ class BootstrapTest(unittest.TestCase): # or not completely loaded (warnings imports indirectly encodings by # importing linecache) yet with support.temp_cwd() as cwd, support.temp_cwd('encodings'): - env = os.environ.copy() - env['PYTHONPATH'] = cwd - # encodings loaded by initfsencoding() - retcode = subprocess.call([sys.executable, '-c', 'pass'], env=env) - self.assertEqual(retcode, 0) + assert_python_ok('-c', 'pass', PYTHONPATH=cwd) # Use -W to load warnings module at startup - retcode = subprocess.call( - [sys.executable, '-c', 'pass', '-W', 'always'], - env=env) - self.assertEqual(retcode, 0) + assert_python_ok('-c', 'pass', '-W', 'always', PYTHONPATH=cwd) def setUpModule(): diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index 4107664429..901f3c99c6 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -196,7 +196,7 @@ class UtilityTests(TestCase): # Check existing value env = {key:alt} util.setup_testing_defaults(env) - self.assertTrue(env[key] is alt) + self.assertIs(env[key], alt) def checkCrossDefault(self,key,value,**kw): util.setup_testing_defaults(kw) @@ -343,7 +343,7 @@ class HeaderTests(TestCase): self.assertEqual(Headers(test[:]).keys(), ['x']) self.assertEqual(Headers(test[:]).values(), ['y']) self.assertEqual(Headers(test[:]).items(), test) - self.assertFalse(Headers(test).items() is test) # must be copy! + self.assertIsNot(Headers(test).items(), test) # must be copy! h=Headers([]) del h['foo'] # should not raise an error diff --git a/Lib/tkinter/test/test_tkinter/test_variables.py b/Lib/tkinter/test/test_tkinter/test_variables.py index fa1f89842f..9d910ac233 100644 --- a/Lib/tkinter/test/test_tkinter/test_variables.py +++ b/Lib/tkinter/test/test_tkinter/test_variables.py @@ -68,6 +68,18 @@ class TestVariable(TestBase): with self.assertRaises(TypeError): Variable(self.root, name=123) + def test_null_in_name(self): + with self.assertRaises(ValueError): + Variable(self.root, name='var\x00name') + with self.assertRaises(ValueError): + self.root.globalsetvar('var\x00name', "value") + with self.assertRaises(ValueError): + self.root.globalsetvar(b'var\x00name', "value") + with self.assertRaises(ValueError): + self.root.setvar('var\x00name', "value") + with self.assertRaises(ValueError): + self.root.setvar(b'var\x00name', "value") + def test_initialize(self): v = Var() self.assertFalse(v.side_effect) @@ -87,6 +99,12 @@ class TestStringVar(TestBase): self.root.globalsetvar("name", "value") self.assertEqual("value", v.get()) + def test_get_null(self): + v = StringVar(self.root, "abc\x00def", "name") + self.assertEqual("abc\x00def", v.get()) + self.root.globalsetvar("name", "val\x00ue") + self.assertEqual("val\x00ue", v.get()) + class TestIntVar(TestBase): diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index 2c4cb0540a..6ef7750f33 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -329,10 +329,11 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): self.checkColorParam(widget, 'disabledbackground') def test_insertborderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'insertborderwidth', 0, 1.3, -2) - self.checkParam(widget, 'insertborderwidth', 2, expected=1) - self.checkParam(widget, 'insertborderwidth', '10p', expected=1) + widget = self.create(insertwidth=100) + self.checkPixelsParam(widget, 'insertborderwidth', + 0, 1.3, 2.6, 6, -2, '10p') + # insertborderwidth is bounded above by a half of insertwidth. + self.checkParam(widget, 'insertborderwidth', 60, expected=100//2) def test_insertwidth(self): widget = self.create() diff --git a/Lib/tkinter/test/widget_tests.py b/Lib/tkinter/test/widget_tests.py index ac194a8995..a9820a7f68 100644 --- a/Lib/tkinter/test/widget_tests.py +++ b/Lib/tkinter/test/widget_tests.py @@ -6,6 +6,7 @@ import tkinter from tkinter.ttk import setup_master, Scale from tkinter.test.support import (tcl_version, requires_tcl, get_tk_patchlevel, pixels_conv, tcl_obj_eq) +import test.support noconv = False @@ -234,8 +235,14 @@ class StandardOptionsTests: widget = self.create() self.checkParam(widget, 'bitmap', 'questhead') self.checkParam(widget, 'bitmap', 'gray50') - self.checkInvalidParam(widget, 'bitmap', 'spam', - errmsg='bitmap "spam" not defined') + filename = test.support.findfile('python.xbm', subdir='imghdrdata') + self.checkParam(widget, 'bitmap', '@' + filename) + # Cocoa Tk widgets don't detect invalid -bitmap values + # See https://core.tcl.tk/tk/info/31cd33dbf0 + if not ('aqua' in self.root.tk.call('tk', 'windowingsystem') and + 'AppKit' in self.root.winfo_server()): + self.checkInvalidParam(widget, 'bitmap', 'spam', + errmsg='bitmap "spam" not defined') def test_borderwidth(self): widget = self.create() @@ -495,7 +502,6 @@ def add_standard_options(*source_classes): return decorator def setUpModule(): - import test.support if test.support.verbose: tcl = tkinter.Tcl() print('patchlevel =', tcl.call('info', 'patchlevel')) diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 6cc1b44e95..71eeb05a6e 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -76,14 +76,13 @@ class FatalIncludeError(SyntaxError): def default_loader(href, parse, encoding=None): if parse == "xml": - file = open(href, 'rb') - data = ElementTree.parse(file).getroot() + with open(href, 'rb') as file: + data = ElementTree.parse(file).getroot() else: if not encoding: encoding = 'UTF-8' - file = open(href, 'r', encoding=encoding) - data = file.read() - file.close() + with open(href, 'r', encoding=encoding) as file: + data = file.read() return data ## diff --git a/Makefile.pre.in b/Makefile.pre.in index e97b07b24b..fdca38e327 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1019,6 +1019,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ test/audiodata \ test/capath test/data \ test/cjkencodings test/decimaltestdata test/xmltestdata \ + test/imghdrdata \ test/subprocessdata test/sndhdrdata test/support \ test/tracedmodules test/encoded_modules \ test/namespace_pkgs \ @@ -703,6 +703,7 @@ Ronan Lamy Torsten Landschoff Łukasz Langa Tino Lange +Glenn Langford Andrew Langmead Detlef Lannert Soren Larsen @@ -10,9 +10,105 @@ What's New in Python 3.3.5 release candidate 1? Core and Builtins ----------------- +- Issue #20437: Fixed 21 potential bugs when deleting objects references. + +- Issue #20538: UTF-7 incremental decoder produced inconsistant string when + input was truncated in BASE64 section. + Library ------- +- Issue #14983: email.generator now always adds a line end after each MIME + boundary marker, instead of doing so only when there is an epilogue. This + fixes an RFC compliance bug and solves an issue with signed MIME parts. + +- Issue #20540: Fix a performance regression (vs. Python 3.2) when layering + a multiprocessing Connection over a TCP socket. For small payloads, Nagle's + algorithm would introduce idle delays before the entire transmission of a + message. + +- Issue #16983: the new email header parsing code will now decode encoded words + that are (incorrectly) surrounded by quotes, and register a defect. + +- Issue #19772: email.generator no longer mutates the message object when + doing a down-transform from 8bit to 7bit CTEs. + +- Issue #18805: the netmask/hostmask parsing in ipaddress now more reliably + filters out illegal values and correctly allows any valid prefix length. + +- Issue #17369: get_filename was raising an exception if the filename + parameter's RFC2231 encoding was broken in certain ways. This was + a regression relative to python2. + +- Issue #20013: Some imap servers disconnect if the current mailbox is + deleted, and imaplib did not handle that case gracefully. Now it + handles the 'bye' correctly. + +- Issue #19920: TarFile.list() no longer fails when outputs a listing + containing non-encodable characters. Based on patch by Vajrasky Kok. + +- Issue #20515: Fix NULL pointer dereference introduced by issue #20368. + +- Issue #19186: Restore namespacing of expat symbols inside the pyexpat module. + +- Issue #20426: When passing the re.DEBUG flag, re.compile() displays the + debug output every time it is called, regardless of the compilation cache. + +- Issue #20368: The null character now correctly passed from Tcl to Python. + Improved error handling in variables-related commands. + +- Issue #20435: Fix _pyio.StringIO.getvalue() to take into account newline + translation settings. + +- Issue #20288: fix handling of invalid numeric charrefs in HTMLParser. + +- Issue #20424: Python implementation of io.StringIO now supports lone surrogates. + +- Issue #19456: ntpath.join() now joins relative paths correctly when a drive + is present. + +- Issue #19077: tempfile.TemporaryDirectory cleanup is now most likely + successful when called during nulling out of modules during shutdown. + Misleading exception no longer raised when resource warning is emitted + during shutdown. + +- Issue #20367: Fix behavior of concurrent.futures.as_completed() for + duplicate arguments. Patch by Glenn Langford. + +- Issue #8260: The read(), readline() and readlines() methods of + codecs.StreamReader returned incomplete data when were called after + readline() or read(size). Based on patch by Amaury Forgeot d'Arc. + +IDLE +---- + +- Issue #20406: Use Python application icons for Idle window title bars. + Patch mostly by Serhiy Storchaka. + +- Update the python.gif icon for the Idle classbrowser and pathbowser + from the old green snake to the new new blue and yellow snakes. + +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Simões. + +Tests +----- + +- Issue #20532: Tests which use _testcapi are now marked as CPython only. + +- Issue #19920: Added tests for TarFile.list(). Based on patch by Vajrasky Kok. + +- Issue #19990: Added tests for the imghdr module. Based on patch by + Claudiu Popa. + +- Issue #20474: Fix test_socket "unexpected success" failures on OS X 10.7+. + +Documentation +------------- + +- Issue #20488: Importlib is no longer *an* implementation of import, it's *the* + implementation. + What's New in Python 3.3.4? =========================== @@ -47,11 +143,6 @@ Core and Builtins source encoding declarations, and can be used to make Python batch files on Windows. -- Issue #19081: When a zipimport .zip file in sys.path being imported from - is modified during the lifetime of the Python process after zipimport has - already cached the zip's table of contents we detect this and recover - rather than read bad data from the .zip (causing odd import errors). - - Issue #17432: Drop UCS2 from names of Unicode functions in python3.def. - Issue #19969: PyBytes_FromFormatV() now raises an OverflowError if "%c" @@ -300,7 +391,7 @@ IDLE - Issue #17390: Add Python version to Idle editor window title bar. Original patches by Edmond Burnett and Kent Johnson. - + - Issue #18960: IDLE now ignores the source encoding declaration on the second line if the first line contains anything except a comment. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 7d0d258b7a..48351bee03 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -159,10 +159,8 @@ _DictRemover_call(PyObject *myself, PyObject *args, PyObject *kw) if (-1 == PyDict_DelItem(self->dict, self->key)) /* XXX Error context */ PyErr_WriteUnraisable(Py_None); - Py_DECREF(self->key); - self->key = NULL; - Py_DECREF(self->dict); - self->dict = NULL; + Py_CLEAR(self->key); + Py_CLEAR(self->dict); } Py_INCREF(Py_None); return Py_None; @@ -2930,10 +2928,8 @@ static int PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob) { if (ob == NULL) { - Py_XDECREF(self->restype); - self->restype = NULL; - Py_XDECREF(self->checker); - self->checker = NULL; + Py_CLEAR(self->restype); + Py_CLEAR(self->checker); return 0; } if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { @@ -2976,10 +2972,8 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob) PyObject *converters; if (ob == NULL || ob == Py_None) { - Py_XDECREF(self->converters); - self->converters = NULL; - Py_XDECREF(self->argtypes); - self->argtypes = NULL; + Py_CLEAR(self->converters); + Py_CLEAR(self->argtypes); } else { converters = converters_from_argtypes(ob); if (!converters) diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 49994158db..09c13d4dbd 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -230,8 +230,7 @@ int pysqlite_build_row_cast_map(pysqlite_Cursor* self) if (converter != Py_None) { Py_DECREF(converter); } - Py_XDECREF(self->row_cast_map); - self->row_cast_map = NULL; + Py_CLEAR(self->row_cast_map); return -1; } @@ -443,8 +442,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* self->locked = 1; self->reset = 0; - Py_XDECREF(self->next_row); - self->next_row = NULL; + Py_CLEAR(self->next_row); if (multiple) { /* executemany() */ @@ -860,8 +858,7 @@ PyObject* pysqlite_cursor_iternext(pysqlite_Cursor *self) if (!self->next_row) { if (self->statement) { (void)pysqlite_statement_reset(self->statement); - Py_DECREF(self->statement); - self->statement = NULL; + Py_CLEAR(self->statement); } return NULL; } diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 6bf0b69164..262d679055 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -419,6 +419,51 @@ Merge(PyObject *args) static PyObject * +unicodeFromTclStringAndSize(const char *s, Py_ssize_t size) +{ + PyObject *r = PyUnicode_DecodeUTF8(s, size, NULL); + if (!r && PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { + /* Tcl encodes null character as \xc0\x80 */ + if (memchr(s, '\xc0', size)) { + char *buf, *q; + const char *e = s + size; + PyErr_Clear(); + q = buf = (char *)PyMem_Malloc(size); + if (buf == NULL) + return NULL; + while (s != e) { + if (s + 1 != e && s[0] == '\xc0' && s[1] == '\x80') { + *q++ = '\0'; + s += 2; + } + else + *q++ = *s++; + } + s = buf; + size = q - s; + r = PyUnicode_DecodeUTF8(s, size, NULL); + PyMem_Free(buf); + } + } + return r; +} + +static PyObject * +unicodeFromTclString(const char *s) +{ + return unicodeFromTclStringAndSize(s, strlen(s)); +} + +static PyObject * +unicodeFromTclObj(Tcl_Obj *value) +{ + int len; + char *s = Tcl_GetStringFromObj(value, &len); + return unicodeFromTclStringAndSize(s, len); +} + + +static PyObject * Split(char *list) { int argc; @@ -435,13 +480,13 @@ Split(char *list) * Could be a quoted string containing funnies, e.g. {"}. * Return the string itself. */ - return PyUnicode_FromString(list); + return unicodeFromTclString(list); } if (argc == 0) v = PyUnicode_FromString(""); else if (argc == 1) - v = PyUnicode_FromString(argv[0]); + v = unicodeFromTclString(argv[0]); else if ((v = PyTuple_New(argc)) != NULL) { int i; PyObject *w; @@ -780,11 +825,8 @@ PyDoc_STRVAR(PyTclObject_string__doc__, static PyObject * PyTclObject_string(PyTclObject *self, void *ignored) { - char *s; - int len; if (!self->string) { - s = Tcl_GetStringFromObj(self->value, &len); - self->string = PyUnicode_FromStringAndSize(s, len); + self->string = unicodeFromTclObj(self->value); if (!self->string) return NULL; } @@ -795,15 +837,12 @@ PyTclObject_string(PyTclObject *self, void *ignored) static PyObject * PyTclObject_str(PyTclObject *self, void *ignored) { - char *s; - int len; - if (self->string && PyUnicode_Check(self->string)) { + if (self->string) { Py_INCREF(self->string); return self->string; } /* XXX Could chache result if it is non-ASCII. */ - s = Tcl_GetStringFromObj(self->value, &len); - return PyUnicode_DecodeUTF8(s, len, "strict"); + return unicodeFromTclObj(self->value); } static PyObject * @@ -873,7 +912,7 @@ PyDoc_STRVAR(get_typename__doc__, "name of the Tcl type"); static PyObject* get_typename(PyTclObject* obj, void* ignored) { - return PyUnicode_FromString(obj->value->typePtr->name); + return unicodeFromTclString(obj->value->typePtr->name); } @@ -985,6 +1024,8 @@ AsObj(PyObject *value) return NULL; } kind = PyUnicode_KIND(value); + if (kind == sizeof(Tcl_UniChar)) + return Tcl_NewUnicodeObj(inbuf, size); allocsize = ((size_t)size) * sizeof(Tcl_UniChar); outbuf = (Tcl_UniChar*)ckalloc(allocsize); /* Else overflow occurred, and we take the next exit */ @@ -1035,8 +1076,7 @@ FromObj(PyObject* tkapp, Tcl_Obj *value) TkappObject *app = (TkappObject*)tkapp; if (value->typePtr == NULL) { - return PyUnicode_FromStringAndSize(value->bytes, - value->length); + return unicodeFromTclStringAndSize(value->bytes, value->length); } if (value->typePtr == app->BooleanType) { @@ -1093,15 +1133,9 @@ FromObj(PyObject* tkapp, Tcl_Obj *value) } if (value->typePtr == app->StringType) { -#if TCL_UTF_MAX==3 - return PyUnicode_FromKindAndData( - PyUnicode_2BYTE_KIND, Tcl_GetUnicode(value), - Tcl_GetCharLength(value)); -#else return PyUnicode_FromKindAndData( - PyUnicode_4BYTE_KIND, Tcl_GetUnicode(value), + sizeof(Tcl_UniChar), Tcl_GetUnicode(value), Tcl_GetCharLength(value)); -#endif } return newPyTclObject(value); @@ -1195,8 +1229,8 @@ static PyObject* Tkapp_CallResult(TkappObject *self) { PyObject *res = NULL; + Tcl_Obj *value = Tcl_GetObjResult(self->interp); if(self->wantobjects) { - Tcl_Obj *value = Tcl_GetObjResult(self->interp); /* Not sure whether the IncrRef is necessary, but something may overwrite the interpreter result while we are converting it. */ @@ -1204,7 +1238,7 @@ Tkapp_CallResult(TkappObject *self) res = FromObj((PyObject*)self, value); Tcl_DecrRefCount(value); } else { - res = PyUnicode_FromString(Tcl_GetStringResult(self->interp)); + res = unicodeFromTclObj(value); } return res; } @@ -1395,7 +1429,7 @@ Tkapp_Eval(PyObject *self, PyObject *args) if (err == TCL_ERROR) res = Tkinter_Error(self); else - res = PyUnicode_FromString(Tkapp_Result(self)); + res = unicodeFromTclString(Tkapp_Result(self)); LEAVE_OVERLAP_TCL return res; } @@ -1445,9 +1479,8 @@ Tkapp_EvalFile(PyObject *self, PyObject *args) ENTER_OVERLAP if (err == TCL_ERROR) res = Tkinter_Error(self); - else - res = PyUnicode_FromString(Tkapp_Result(self)); + res = unicodeFromTclString(Tkapp_Result(self)); LEAVE_OVERLAP_TCL return res; } @@ -1470,7 +1503,7 @@ Tkapp_Record(PyObject *self, PyObject *args) if (err == TCL_ERROR) res = Tkinter_Error(self); else - res = PyUnicode_FromString(Tkapp_Result(self)); + res = unicodeFromTclString(Tkapp_Result(self)); LEAVE_OVERLAP_TCL return res; } @@ -1517,20 +1550,45 @@ typedef struct VarEvent { static int varname_converter(PyObject *in, void *_out) { + char *s; char **out = (char**)_out; if (PyBytes_Check(in)) { - *out = PyBytes_AsString(in); + if (PyBytes_Size(in) > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "bytes object is too long"); + return 0; + } + s = PyBytes_AsString(in); + if (strlen(s) != PyBytes_Size(in)) { + PyErr_SetString(PyExc_ValueError, "null byte in bytes object"); + return 0; + } + *out = s; return 1; } if (PyUnicode_Check(in)) { - *out = _PyUnicode_AsString(in); + Py_ssize_t size; + s = PyUnicode_AsUTF8AndSize(in, &size); + if (s == NULL) { + return 0; + } + if (size > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "string is too long"); + return 0; + } + if (strlen(s) != size) { + PyErr_SetString(PyExc_ValueError, "null character in string"); + return 0; + } + *out = s; return 1; } if (PyTclObject_Check(in)) { *out = PyTclObject_TclString(in); return 1; } - /* XXX: Should give diagnostics. */ + PyErr_Format(PyExc_TypeError, + "must be str, bytes or Tcl_Obj, not %.50s", + in->ob_type->tp_name); return 0; } @@ -1616,8 +1674,11 @@ SetVar(PyObject *self, PyObject *args, int flags) PyObject *res = NULL; Tcl_Obj *newval, *ok; - if (PyArg_ParseTuple(args, "O&O:setvar", - varname_converter, &name1, &newValue)) { + switch (PyTuple_GET_SIZE(args)) { + case 2: + if (!PyArg_ParseTuple(args, "O&O:setvar", + varname_converter, &name1, &newValue)) + return NULL; /* XXX Acquire tcl lock??? */ newval = AsObj(newValue); if (newval == NULL) @@ -1633,27 +1694,27 @@ SetVar(PyObject *self, PyObject *args, int flags) Py_INCREF(res); } LEAVE_OVERLAP_TCL - } - else { - PyErr_Clear(); - if (PyArg_ParseTuple(args, "ssO:setvar", - &name1, &name2, &newValue)) { - /* XXX must hold tcl lock already??? */ - newval = AsObj(newValue); - ENTER_TCL - ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); - ENTER_OVERLAP - if (!ok) - Tkinter_Error(self); - else { - res = Py_None; - Py_INCREF(res); - } - LEAVE_OVERLAP_TCL - } - else { + break; + case 3: + if (!PyArg_ParseTuple(args, "ssO:setvar", + &name1, &name2, &newValue)) return NULL; + /* XXX must hold tcl lock already??? */ + newval = AsObj(newValue); + ENTER_TCL + ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); + ENTER_OVERLAP + if (!ok) + Tkinter_Error(self); + else { + res = Py_None; + Py_INCREF(res); } + LEAVE_OVERLAP_TCL + break; + default: + PyErr_SetString(PyExc_TypeError, "setvar requires 2 to 3 arguments"); + return NULL; } return res; } @@ -1693,7 +1754,7 @@ GetVar(PyObject *self, PyObject *args, int flags) res = FromObj(self, tres); } else { - res = PyUnicode_FromString(Tcl_GetString(tres)); + res = unicodeFromTclObj(tres); } } LEAVE_OVERLAP_TCL @@ -1831,7 +1892,7 @@ Tkapp_ExprString(PyObject *self, PyObject *args) if (retval == TCL_ERROR) res = Tkinter_Error(self); else - res = Py_BuildValue("s", Tkapp_Result(self)); + res = unicodeFromTclString(Tkapp_Result(self)); LEAVE_OVERLAP_TCL return res; } @@ -1956,7 +2017,7 @@ Tkapp_SplitList(PyObject *self, PyObject *args) goto finally; for (i = 0; i < argc; i++) { - PyObject *s = PyUnicode_FromString(argv[i]); + PyObject *s = unicodeFromTclString(argv[i]); if (!s || PyTuple_SetItem(v, i, s)) { Py_DECREF(v); v = NULL; @@ -2075,20 +2136,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, char *argv[]) return PythonCmd_Error(interp); for (i = 0; i < (argc - 1); i++) { - PyObject *s = PyUnicode_FromString(argv[i + 1]); - if (!s) { - /* Is Tk leaking 0xC080 in %A - a "modified" utf-8 null? */ - if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError) && - !strcmp(argv[i + 1], "\xC0\x80")) { - PyErr_Clear(); - /* Convert to "strict" utf-8 null */ - s = PyUnicode_FromString("\0"); - } else { - Py_DECREF(arg); - return PythonCmd_Error(interp); - } - } - if (PyTuple_SetItem(arg, i, s)) { + PyObject *s = unicodeFromTclString(argv[i + 1]); + if (!s || PyTuple_SetItem(arg, i, s)) { Py_DECREF(arg); return PythonCmd_Error(interp); } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 2c03284ea2..f337e1c562 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -7,6 +7,10 @@ /* External API definitions */ +/* Namespace external symbols to allow multiple libexpat version to + co-exist. */ +#include "pyexpatns.h" + #if defined(_MSC_EXTENSIONS) && !defined(__BEOS__) && !defined(__CYGWIN__) #define XML_USE_MSC_EXTENSIONS 1 #endif diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4b077a0188..e72fbcf11b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -693,8 +693,7 @@ typedef struct { static void path_cleanup(path_t *path) { if (path->cleanup) { - Py_DECREF(path->cleanup); - path->cleanup = NULL; + Py_CLEAR(path->cleanup); } } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 7e51d35e62..45680ae962 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -314,8 +314,7 @@ call_with_frame(PyCodeObject *c, PyObject* func, PyObject* args, } else { if (trace_frame(tstate, f, PyTrace_RETURN, res) < 0) { - Py_XDECREF(res); - res = NULL; + Py_CLEAR(res); } } #else diff --git a/Modules/readline.c b/Modules/readline.c index bcd34f70c1..8fac526be3 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -231,8 +231,7 @@ set_hook(const char *funcname, PyObject **hook_var, PyObject *args) if (!PyArg_ParseTuple(args, buf, &function)) return NULL; if (function == Py_None) { - Py_XDECREF(*hook_var); - *hook_var = NULL; + Py_CLEAR(*hook_var); } else if (PyCallable_Check(function)) { PyObject *tmp = *hook_var; @@ -827,7 +826,7 @@ on_completion_display_matches_hook(char **matches, (r != Py_None && PyLong_AsLong(r) == -1 && PyErr_Occurred())) { goto error; } - Py_XDECREF(r); r=NULL; + Py_CLEAR(r); if (0) { error: diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index c492224ecb..952a0919d4 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -74,8 +74,7 @@ reap_obj(pylist fd2obj[FD_SETSIZE + 1]) { int i; for (i = 0; i < FD_SETSIZE + 1 && fd2obj[i].sentinel >= 0; i++) { - Py_XDECREF(fd2obj[i].obj); - fd2obj[i].obj = NULL; + Py_CLEAR(fd2obj[i].obj); } fd2obj[0].sentinel = -1; } diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index fbe1bb7a8c..704c9f591b 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1305,12 +1305,9 @@ finisignal(void) Py_XDECREF(func); } - Py_XDECREF(IntHandler); - IntHandler = NULL; - Py_XDECREF(DefaultHandler); - DefaultHandler = NULL; - Py_XDECREF(IgnoreHandler); - IgnoreHandler = NULL; + Py_CLEAR(IntHandler); + Py_CLEAR(DefaultHandler); + Py_CLEAR(IgnoreHandler); } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 8c8b08ea9f..32b4eba2f2 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1226,7 +1226,7 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto) } #endif -#ifdef AF_CAN +#ifdef HAVE_LINUX_CAN_H case AF_CAN: { struct sockaddr_can *a = (struct sockaddr_can *)addr; @@ -1654,7 +1654,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } #endif -#ifdef AF_CAN +#ifdef HAVE_LINUX_CAN_H case AF_CAN: switch (s->sock_proto) { case CAN_RAW: @@ -1859,7 +1859,7 @@ getsockaddrlen(PySocketSockObject *s, socklen_t *len_ret) } #endif -#ifdef AF_CAN +#ifdef HAVE_LINUX_CAN_H case AF_CAN: { *len_ret = sizeof (struct sockaddr_can); diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 8b877cfdbc..fc3f37595b 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -197,8 +197,7 @@ syslog_closelog(PyObject *self, PyObject *unused) { if (S_log_open) { closelog(); - Py_XDECREF(S_ident_o); - S_ident_o = NULL; + Py_CLEAR(S_ident_o); S_log_open = 0; } Py_INCREF(Py_None); diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index e718795fa7..1d1072dcea 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -392,8 +392,7 @@ PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs) } error: - Py_XDECREF(self); - self = NULL; + Py_CLEAR(self); success: if (zdict.buf != NULL) PyBuffer_Release(&zdict); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 808e595157..b31213098a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -952,8 +952,7 @@ void PyFrame_Fini(void) { (void)PyFrame_ClearFreeList(); - Py_XDECREF(builtin_object); - builtin_object = NULL; + Py_CLEAR(builtin_object); } /* Print summary info about the state of the optimized allocator */ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index ec3f91b2c6..123df8cef3 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -866,8 +866,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) _Py_ForgetReference((PyObject *) v); /* DECREF items deleted by shrinkage */ for (i = newsize; i < oldsize; i++) { - Py_XDECREF(v->ob_item[i]); - v->ob_item[i] = NULL; + Py_CLEAR(v->ob_item[i]); } sv = PyObject_GC_Resize(PyTupleObject, v, newsize); if (sv == NULL) { @@ -913,8 +912,7 @@ PyTuple_Fini(void) #if PyTuple_MAXSAVESIZE > 0 /* empty tuples are used all over the place and applications may * rely on the fact that an empty tuple is a singleton. */ - Py_XDECREF(free_list[0]); - free_list[0] = NULL; + Py_CLEAR(free_list[0]); (void)PyTuple_ClearFreeList(); #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a149177a09..e1ff999e13 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1846,8 +1846,7 @@ _PyUnicode_ClearStaticStrings() { _Py_Identifier *tmp, *s = static_strings; while (s) { - Py_DECREF(s->object); - s->object = NULL; + Py_CLEAR(s->object); tmp = s->next; s->next = NULL; s = tmp; @@ -4082,8 +4081,7 @@ make_decode_exception(PyObject **exceptionObject, return; onError: - Py_DECREF(*exceptionObject); - *exceptionObject = NULL; + Py_CLEAR(*exceptionObject); } /* error handling callback helper: @@ -4474,8 +4472,16 @@ utf7Error: /* return state */ if (consumed) { if (inShift) { - outpos = shiftOutStart; /* back off output */ *consumed = startinpos; + if (outpos != shiftOutStart && + PyUnicode_MAX_CHAR_VALUE(unicode) > 127) { + PyObject *result = PyUnicode_FromKindAndData( + PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), + shiftOutStart); + Py_DECREF(unicode); + unicode = result; + } + outpos = shiftOutStart; /* back off output */ } else { *consumed = s-starts; @@ -6216,8 +6222,7 @@ make_encode_exception(PyObject **exceptionObject, goto onError; return; onError: - Py_DECREF(*exceptionObject); - *exceptionObject = NULL; + Py_CLEAR(*exceptionObject); } } @@ -8217,8 +8222,7 @@ make_translate_exception(PyObject **exceptionObject, goto onError; return; onError: - Py_DECREF(*exceptionObject); - *exceptionObject = NULL; + Py_CLEAR(*exceptionObject); } } diff --git a/Python/ceval.c b/Python/ceval.c index 73925dc413..2b1619163b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3049,8 +3049,7 @@ fast_yield: if (call_trace(tstate->c_tracefunc, tstate->c_traceobj, f, PyTrace_RETURN, retval)) { - Py_XDECREF(retval); - retval = NULL; + Py_CLEAR(retval); why = WHY_EXCEPTION; } } @@ -3068,8 +3067,7 @@ fast_yield: else if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, f, PyTrace_RETURN, retval)) { - Py_XDECREF(retval); - retval = NULL; + Py_CLEAR(retval); /* why = WHY_EXCEPTION; */ } } @@ -3426,8 +3424,7 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, if (co->co_flags & CO_GENERATOR) { /* Don't need to keep the reference to f_back, it will be set * when the generator is resumed. */ - Py_XDECREF(f->f_back); - f->f_back = NULL; + Py_CLEAR(f->f_back); PCALL(PCALL_GENERATOR); diff --git a/Python/import.c b/Python/import.c index e91cef83ff..26f82cf29b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -253,8 +253,7 @@ imp_release_lock(PyObject *self, PyObject *noargs) void _PyImport_Fini(void) { - Py_XDECREF(extensions); - extensions = NULL; + Py_CLEAR(extensions); #ifdef WITH_THREAD if (import_lock != NULL) { PyThread_free_lock(import_lock); @@ -497,8 +496,7 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, /* Somebody already imported the module, likely under a different name. XXX this should really not happen. */ - Py_DECREF(def->m_base.m_copy); - def->m_base.m_copy = NULL; + Py_CLEAR(def->m_base.m_copy); } dict = PyModule_GetDict(mod); if (dict == NULL) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 222630c169..2f700e6507 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -395,8 +395,7 @@ trace_trampoline(PyObject *self, PyFrameObject *frame, result = call_trampoline(tstate, callback, frame, what, arg); if (result == NULL) { PyEval_SetTrace(NULL, NULL); - Py_XDECREF(frame->f_trace); - frame->f_trace = NULL; + Py_CLEAR(frame->f_trace); return -1; } if (result != Py_None) { |
