summaryrefslogtreecommitdiffstats
path: root/Lib/test
diff options
context:
space:
mode:
authorVictor Stinner <victor.stinner@gmail.com>2017-06-27 00:02:04 (GMT)
committerGitHub <noreply@github.com>2017-06-27 00:02:04 (GMT)
commit48b5c422ffb03affb00c184b9a99e5537be92732 (patch)
treef1d8701a3dc6cc371c247a31ba72aed56dd5af1a /Lib/test
parentbac7d3363b099d0cdef3e541f8581859edfddc85 (diff)
downloadcpython-48b5c422ffb03affb00c184b9a99e5537be92732.zip
cpython-48b5c422ffb03affb00c184b9a99e5537be92732.tar.gz
cpython-48b5c422ffb03affb00c184b9a99e5537be92732.tar.bz2
bpo-30776: reduce regrtest -R false positives (#2422)
* Change the regrtest --huntrleaks checker to decide if a test file leaks or not. Require that each run leaks at least 1 reference. * Warmup runs are now completely ignored: ignored in the checker test and not used anymore to compute the sum. * Add an unit test for a reference leak. Example of reference differences previously considered a failure (leak) and now considered as success (success, no leak): [3, 0, 0] [0, 1, 0] [8, -8, 1]
Diffstat (limited to 'Lib/test')
-rw-r--r--Lib/test/libregrtest/refleak.py25
-rw-r--r--Lib/test/test_regrtest.py53
2 files changed, 57 insertions, 21 deletions
diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py
index c69fc0f..8e18d75 100644
--- a/Lib/test/libregrtest/refleak.py
+++ b/Lib/test/libregrtest/refleak.py
@@ -93,9 +93,21 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
rc_before = rc_after
fd_before = fd_after
print(file=sys.stderr)
+
# These checkers return False on success, True on failure
def check_rc_deltas(deltas):
- return any(deltas)
+ # bpo-30776: Try to ignore false positives:
+ #
+ # [3, 0, 0]
+ # [0, 1, 0]
+ # [8, -8, 1]
+ #
+ # Expected leaks:
+ #
+ # [5, 5, 6]
+ # [10, 1, 1]
+ return all(delta >= 1 for delta in deltas)
+
def check_alloc_deltas(deltas):
# At least 1/3rd of 0s
if 3 * deltas.count(0) < len(deltas):
@@ -104,14 +116,21 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
if not set(deltas) <= {1,0,-1}:
return True
return False
+
+ def check_fd_deltas(deltas):
+ return any(deltas)
+
failed = False
for deltas, item_name, checker in [
(rc_deltas, 'references', check_rc_deltas),
(alloc_deltas, 'memory blocks', check_alloc_deltas),
- (fd_deltas, 'file descriptors', check_rc_deltas)]:
+ (fd_deltas, 'file descriptors', check_fd_deltas)
+ ]:
+ # ignore warmup runs
+ deltas = deltas[nwarmup:]
if checker(deltas):
msg = '%s leaked %s %s, sum=%s' % (
- test, deltas[nwarmup:], item_name, sum(deltas))
+ test, deltas, item_name, sum(deltas))
print(msg, file=sys.stderr, flush=True)
with open(fname, "a") as refrep:
print(msg, file=refrep)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 0e676ee..a544f88 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -783,6 +783,40 @@ class ArgsTestCase(BaseTestCase):
output = self.run_tests('--forever', test, exitcode=2)
self.check_executed_tests(output, [test]*3, failed=test)
+ def check_leak(self, code, what):
+ test = self.create_test('huntrleaks', code=code)
+
+ filename = 'reflog.txt'
+ self.addCleanup(support.unlink, filename)
+ output = self.run_tests('--huntrleaks', '3:3:', test,
+ exitcode=2,
+ stderr=subprocess.STDOUT)
+ self.check_executed_tests(output, [test], failed=test)
+
+ line = 'beginning 6 repetitions\n123456\n......\n'
+ self.check_line(output, re.escape(line))
+
+ line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what)
+ self.assertIn(line2, output)
+
+ with open(filename) as fp:
+ reflog = fp.read()
+ self.assertIn(line2, reflog)
+
+ @unittest.skipUnless(Py_DEBUG, 'need a debug build')
+ def test_huntrleaks(self):
+ # test --huntrleaks
+ code = textwrap.dedent("""
+ import unittest
+
+ GLOBAL_LIST = []
+
+ class RefLeakTest(unittest.TestCase):
+ def test_leak(self):
+ GLOBAL_LIST.append(object())
+ """)
+ self.check_leak(code, 'references')
+
@unittest.skipUnless(Py_DEBUG, 'need a debug build')
def test_huntrleaks_fd_leak(self):
# test --huntrleaks for file descriptor leak
@@ -807,24 +841,7 @@ class ArgsTestCase(BaseTestCase):
fd = os.open(__file__, os.O_RDONLY)
# bug: never cloes the file descriptor
""")
- test = self.create_test('huntrleaks', code=code)
-
- filename = 'reflog.txt'
- self.addCleanup(support.unlink, filename)
- output = self.run_tests('--huntrleaks', '3:3:', test,
- exitcode=2,
- stderr=subprocess.STDOUT)
- self.check_executed_tests(output, [test], failed=test)
-
- line = 'beginning 6 repetitions\n123456\n......\n'
- self.check_line(output, re.escape(line))
-
- line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test
- self.assertIn(line2, output)
-
- with open(filename) as fp:
- reflog = fp.read()
- self.assertIn(line2, reflog)
+ self.check_leak(code, 'file descriptors')
def test_list_tests(self):
# test --list-tests