From 5c3d52405b647bc69185f657ed4c180c02ac14f7 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Thu, 12 Apr 2018 00:27:11 -0700 Subject: TST: Extract a helper function to test for reference cycles This also means we can now test that our test is actually able to detect the type of failure we expect Trying to give myself some tools to debug the failure at https://github.com/numpy/numpy/pull/10882/files#r180813166 --- numpy/testing/_private/utils.py | 64 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) (limited to 'numpy/testing/_private/utils.py') diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 507ecb1e2..4a113f12e 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -7,6 +7,7 @@ from __future__ import division, absolute_import, print_function import os import sys import re +import gc import operator import warnings from functools import partial, wraps @@ -35,7 +36,7 @@ __all__ = [ 'assert_allclose', 'IgnoreException', 'clear_and_catch_warnings', 'SkipTest', 'KnownFailureException', 'temppath', 'tempdir', 'IS_PYPY', 'HAS_REFCOUNT', 'suppress_warnings', 'assert_array_compare', - '_assert_valid_refcount', '_gen_alignment_data', + '_assert_valid_refcount', '_gen_alignment_data', 'assert_no_gc_cycles', ] @@ -2272,3 +2273,64 @@ class suppress_warnings(object): return func(*args, **kwargs) return new_func + + +@contextlib.contextmanager +def _assert_no_gc_cycles_context(name=None): + __tracebackhide__ = True # Hide traceback for py.test + + # not meaningful to test if there is no refcounting + if not HAS_REFCOUNT: + return + + assert_(gc.isenabled()) + gc.disable() + try: + gc.collect() + yield + # gc.collect returns the number of unreachable objects in cycles that + # were found -- we are checking that no cycles were created in the context + n_objects_in_cycles = gc.collect() + finally: + gc.enable() + + if n_objects_in_cycles: + name_str = " when calling %s" % name if name is not None else "" + raise AssertionError( + "Reference cycles were found{}: {} objects were collected" + .format(name_str, n_objects_in_cycles)) + + +def assert_no_gc_cycles(*args, **kwargs): + """ + Fail if the given callable produces any reference cycles. + + If called with all arguments omitted, may be used as a context manager: + + with assert_no_gc_cycles(): + do_something() + + .. versionadded:: 1.15.0 + + Parameters + ---------- + func : callable + The callable to test. + \\*args : Arguments + Arguments passed to `func`. + \\*\\*kwargs : Kwargs + Keyword arguments passed to `func`. + + Returns + ------- + Nothing. The result is deliberately discarded to ensure that all cycles + are found. + + """ + if not args: + return _assert_no_gc_cycles_context() + + func = args[0] + args = args[1:] + with _assert_no_gc_cycles_context(name=func.__name__): + func(*args, **kwargs) -- cgit v1.2.1 From d21ec05eb006c072e4fd8c5fe1bd63619378aded Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Thu, 12 Apr 2018 00:42:18 -0700 Subject: ENH: Show the full list of leaked objects An example output for the test added in the previous commit is: AssertionError: Reference cycles were found when calling make_cycle: 1 objects were collected, of which 1 are shown below: list object with id=2279664872136: [, ] --- numpy/testing/_private/utils.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'numpy/testing/_private/utils.py') diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 4a113f12e..0c9fd644c 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -15,6 +15,7 @@ import shutil import contextlib from tempfile import mkdtemp, mkstemp from unittest.case import SkipTest +import pprint from numpy.core import( float32, empty, arange, array_repr, ndarray, isnat, array) @@ -2285,20 +2286,38 @@ def _assert_no_gc_cycles_context(name=None): assert_(gc.isenabled()) gc.disable() + gc_debug = gc.get_debug() try: gc.collect() + gc.set_debug(gc.DEBUG_SAVEALL) yield # gc.collect returns the number of unreachable objects in cycles that # were found -- we are checking that no cycles were created in the context n_objects_in_cycles = gc.collect() + objects_in_cycles = gc.garbage[:] finally: + del gc.garbage[:] + gc.set_debug(gc_debug) gc.enable() if n_objects_in_cycles: name_str = " when calling %s" % name if name is not None else "" raise AssertionError( - "Reference cycles were found{}: {} objects were collected" - .format(name_str, n_objects_in_cycles)) + "Reference cycles were found{}: {} objects were collected, " + "of which {} are shown below:{}" + .format( + name_str, + n_objects_in_cycles, + len(objects_in_cycles), + ''.join( + "\n {} object with id={}:\n {}".format( + type(o).__name__, + id(o), + pprint.pformat(o).replace('\n', '\n ') + ) for o in objects_in_cycles + ) + ) + ) def assert_no_gc_cycles(*args, **kwargs): -- cgit v1.2.1 From 3ff0c5c82b8abc4c94b1801a13f488778631f38a Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Thu, 12 Apr 2018 22:07:58 -0700 Subject: BUG: Ensure the garbage is clear first in assert_no_gc_cycles It's not always possible to guarantee this, so also adds a test to verify that we don't hang --- numpy/testing/_private/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'numpy/testing/_private/utils.py') diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 0c9fd644c..b0c0b0c48 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -2288,7 +2288,14 @@ def _assert_no_gc_cycles_context(name=None): gc.disable() gc_debug = gc.get_debug() try: - gc.collect() + for i in range(100): + if gc.collect() == 0: + break + else: + raise RuntimeError( + "Unable to fully collect garbage - perhaps a __del__ method is " + "creating more reference cycles?") + gc.set_debug(gc.DEBUG_SAVEALL) yield # gc.collect returns the number of unreachable objects in cycles that -- cgit v1.2.1