Skip to content

Commit d063dee

Browse files
tacaswelltacaswell
committed
Merge pull request #5360 from mdboom/new-memleak-script
TST: Add a new memleak script that does everything
2 parents c457f01 + b59627b commit d063dee

File tree

6 files changed

+143
-342
lines changed

6 files changed

+143
-342
lines changed

‎lib/matplotlib/cbook.py

Lines changed: 0 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1425,51 +1425,6 @@ def restrict_dict(d, keys):
14251425
return dict([(k, v) for (k, v) in six.iteritems(d) if k in keys])
14261426

14271427

1428-
def report_memory(i=0): # argument may go away
1429-
'return the memory consumed by process'
1430-
from matplotlib.compat.subprocess import Popen, PIPE
1431-
pid = os.getpid()
1432-
if sys.platform == 'sunos5':
1433-
try:
1434-
a2 = Popen('ps -p %d -o osz' % pid, shell=True,
1435-
stdout=PIPE).stdout.readlines()
1436-
except OSError:
1437-
raise NotImplementedError(
1438-
"report_memory works on Sun OS only if "
1439-
"the 'ps' program is found")
1440-
mem = int(a2[-1].strip())
1441-
elif sys.platform.startswith('linux'):
1442-
try:
1443-
a2 = Popen('ps -p %d -o rss,sz' % pid, shell=True,
1444-
stdout=PIPE).stdout.readlines()
1445-
except OSError:
1446-
raise NotImplementedError(
1447-
"report_memory works on Linux only if "
1448-
"the 'ps' program is found")
1449-
mem = int(a2[1].split()[1])
1450-
elif sys.platform.startswith('darwin'):
1451-
try:
1452-
a2 = Popen('ps -p %d -o rss,vsz' % pid, shell=True,
1453-
stdout=PIPE).stdout.readlines()
1454-
except OSError:
1455-
raise NotImplementedError(
1456-
"report_memory works on Mac OS only if "
1457-
"the 'ps' program is found")
1458-
mem = int(a2[1].split()[0])
1459-
elif sys.platform.startswith('win'):
1460-
try:
1461-
a2 = Popen(["tasklist", "/nh", "/fi", "pid eq %d" % pid],
1462-
stdout=PIPE).stdout.read()
1463-
except OSError:
1464-
raise NotImplementedError(
1465-
"report_memory works on Windows only if "
1466-
"the 'tasklist' program is found")
1467-
mem = int(a2.strip().split()[-2].replace(',', ''))
1468-
else:
1469-
raise NotImplementedError(
1470-
"We don't have a memory monitor for %s" % sys.platform)
1471-
return mem
1472-
14731428
_safezip_msg = 'In safezip, len(args[0])=%d but len(args[%d])=%d'
14741429

14751430

@@ -1501,58 +1456,6 @@ def safe_masked_invalid(x):
15011456
return xm
15021457

15031458

1504-
class MemoryMonitor(object):
1505-
def __init__(self, nmax=20000):
1506-
self._nmax = nmax
1507-
self._mem = np.zeros((self._nmax,), np.int32)
1508-
self.clear()
1509-
1510-
def clear(self):
1511-
self._n = 0
1512-
self._overflow = False
1513-
1514-
def __call__(self):
1515-
mem = report_memory()
1516-
if self._n < self._nmax:
1517-
self._mem[self._n] = mem
1518-
self._n += 1
1519-
else:
1520-
self._overflow = True
1521-
return mem
1522-
1523-
def report(self, segments=4):
1524-
n = self._n
1525-
segments = min(n, segments)
1526-
dn = int(n / segments)
1527-
ii = list(xrange(0, n, dn))
1528-
ii[-1] = n - 1
1529-
print()
1530-
print('memory report: i, mem, dmem, dmem/nloops')
1531-
print(0, self._mem[0])
1532-
for i in range(1, len(ii)):
1533-
di = ii[i] - ii[i - 1]
1534-
if di == 0:
1535-
continue
1536-
dm = self._mem[ii[i]] - self._mem[ii[i - 1]]
1537-
print('%5d %5d %3d %8.3f' % (ii[i], self._mem[ii[i]],
1538-
dm, dm / float(di)))
1539-
if self._overflow:
1540-
print("Warning: array size was too small for the number of calls.")
1541-
1542-
def xy(self, i0=0, isub=1):
1543-
x = np.arange(i0, self._n, isub)
1544-
return x, self._mem[i0:self._n:isub]
1545-
1546-
def plot(self, i0=0, isub=1, fig=None):
1547-
if fig is None:
1548-
from .pylab import figure
1549-
fig = figure()
1550-
1551-
ax = fig.add_subplot(111)
1552-
ax.plot(*self.xy(i0, isub))
1553-
fig.canvas.draw()
1554-
1555-
15561459
def print_cycles(objects, outstream=sys.stdout, show_progress=False):
15571460
"""
15581461
*objects*

‎unit/agg_memleak.py

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎unit/memleak.py

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
#!/usr/bin/env python
2+
3+
from __future__ import print_function
4+
5+
import gc
6+
7+
try:
8+
import tracemalloc
9+
except ImportError:
10+
raise ImportError("This script requires Python 3.4 or later")
11+
12+
try:
13+
import psutil
14+
except ImportError:
15+
raise ImportError("This script requires psutil")
16+
17+
import numpy as np
18+
19+
20+
def run_memleak_test(bench, iterations, report):
21+
tracemalloc.start()
22+
23+
starti = min(50, iterations / 2)
24+
endi = iterations
25+
26+
malloc_arr = np.empty((endi,), dtype=np.int64)
27+
rss_arr = np.empty((endi,), dtype=np.int64)
28+
rss_peaks = np.empty((endi,), dtype=np.int64)
29+
nobjs_arr = np.empty((endi,), dtype=np.int64)
30+
garbage_arr = np.empty((endi,), dtype=np.int64)
31+
open_files_arr = np.empty((endi,), dtype=np.int64)
32+
rss_peak = 0
33+
34+
p = psutil.Process()
35+
36+
for i in range(endi):
37+
bench()
38+
39+
gc.collect()
40+
41+
rss = p.memory_info().rss
42+
malloc, peak = tracemalloc.get_traced_memory()
43+
nobjs = len(gc.get_objects())
44+
garbage = len(gc.garbage)
45+
open_files = len(p.open_files())
46+
print("{0: 4d}: pymalloc {1: 10d}, rss {2: 10d}, nobjs {3: 10d}, garbage {4: 4d}, files: {5: 4d}".format(
47+
i, malloc, rss, nobjs, garbage, open_files))
48+
49+
malloc_arr[i] = malloc
50+
rss_arr[i] = rss
51+
if rss > rss_peak:
52+
rss_peak = rss
53+
rss_peaks[i] = rss_peak
54+
nobjs_arr[i] = nobjs
55+
garbage_arr[i] = garbage
56+
open_files_arr[i] = open_files
57+
58+
print('Average memory consumed per loop: %1.4f bytes\n' %
59+
(np.sum(rss_peaks[starti+1:] - rss_peaks[starti:-1]) / float(endi - starti)))
60+
61+
from matplotlib import pyplot as plt
62+
fig, (ax1, ax2, ax3) = plt.subplots(3)
63+
ax1b = ax1.twinx()
64+
ax1.plot(malloc_arr, 'r')
65+
ax1b.plot(rss_arr, 'b')
66+
ax1.set_ylabel('pymalloc', color='r')
67+
ax1b.set_ylabel('rss', color='b')
68+
69+
ax2b = ax2.twinx()
70+
ax2.plot(nobjs_arr, 'r')
71+
ax2b.plot(garbage_arr, 'b')
72+
ax2.set_ylabel('total objects', color='r')
73+
ax2b.set_ylabel('garbage objects', color='b')
74+
75+
ax3.plot(open_files_arr)
76+
ax3.set_ylabel('open file handles')
77+
78+
if not report.endswith('.pdf'):
79+
report = report + '.pdf'
80+
fig.savefig(report, format='pdf')
81+
82+
83+
class MemleakTest(object):
84+
def __init__(self, empty):
85+
self.empty = empty
86+
87+
def __call__(self):
88+
import matplotlib.pyplot as plt
89+
90+
fig = plt.figure(1)
91+
92+
if not self.empty:
93+
t1 = np.arange(0.0, 2.0, 0.01)
94+
y1 = np.sin(2 * np.pi * t1)
95+
y2 = np.random.rand(len(t1))
96+
X = np.random.rand(50, 50)
97+
98+
ax = fig.add_subplot(221)
99+
ax.plot(t1, y1, '-')
100+
ax.plot(t1, y2, 's')
101+
102+
ax = fig.add_subplot(222)
103+
ax.imshow(X)
104+
105+
ax = fig.add_subplot(223)
106+
ax.scatter(np.random.rand(50), np.random.rand(50),
107+
s=100 * np.random.rand(50), c=np.random.rand(50))
108+
109+
ax = fig.add_subplot(224)
110+
ax.pcolor(10 * np.random.rand(50, 50))
111+
112+
fig.savefig('tmp', dpi=75)
113+
plt.close(1)
114+
115+
116+
if __name__ == '__main__':
117+
import argparse
118+
119+
parser = argparse.ArgumentParser('Run memory leak tests')
120+
parser.add_argument('backend', type=str, nargs=1,
121+
help='backend to test')
122+
parser.add_argument('iterations', type=int, nargs=1,
123+
help='number of iterations')
124+
parser.add_argument('report', type=str, nargs=1,
125+
help='filename to save report')
126+
parser.add_argument('--empty', action='store_true',
127+
help="Don't plot any content, just test creating "
128+
"and destroying figures")
129+
parser.add_argument('--interactive', action='store_true',
130+
help="Turn on interactive mode to actually open "
131+
"windows. Only works with some GUI backends.")
132+
133+
134+
args = parser.parse_args()
135+
136+
import matplotlib
137+
matplotlib.use(args.backend[0])
138+
139+
if args.interactive:
140+
from matplotlib import pyplot as plt
141+
plt.ion()
142+
143+
run_memleak_test(MemleakTest(args.empty), args.iterations[0], args.report[0])

0 commit comments

Comments
 (0)