diff --git a/lib/matplotlib/cbook.py b/lib/matplotlib/cbook.py index 58cb04dc960d..1499f6435165 100644 --- a/lib/matplotlib/cbook.py +++ b/lib/matplotlib/cbook.py @@ -1438,51 +1438,6 @@ def restrict_dict(d, keys): return dict([(k, v) for (k, v) in six.iteritems(d) if k in keys]) -def report_memory(i=0): # argument may go away - 'return the memory consumed by process' - from matplotlib.compat.subprocess import Popen, PIPE - pid = os.getpid() - if sys.platform == 'sunos5': - try: - a2 = Popen('ps -p %d -o osz' % pid, shell=True, - stdout=PIPE).stdout.readlines() - except OSError: - raise NotImplementedError( - "report_memory works on Sun OS only if " - "the 'ps' program is found") - mem = int(a2[-1].strip()) - elif sys.platform.startswith('linux'): - try: - a2 = Popen('ps -p %d -o rss,sz' % pid, shell=True, - stdout=PIPE).stdout.readlines() - except OSError: - raise NotImplementedError( - "report_memory works on Linux only if " - "the 'ps' program is found") - mem = int(a2[1].split()[1]) - elif sys.platform.startswith('darwin'): - try: - a2 = Popen('ps -p %d -o rss,vsz' % pid, shell=True, - stdout=PIPE).stdout.readlines() - except OSError: - raise NotImplementedError( - "report_memory works on Mac OS only if " - "the 'ps' program is found") - mem = int(a2[1].split()[0]) - elif sys.platform.startswith('win'): - try: - a2 = Popen(["tasklist", "/nh", "/fi", "pid eq %d" % pid], - stdout=PIPE).stdout.read() - except OSError: - raise NotImplementedError( - "report_memory works on Windows only if " - "the 'tasklist' program is found") - mem = int(a2.strip().split()[-2].replace(',', '')) - else: - raise NotImplementedError( - "We don't have a memory monitor for %s" % sys.platform) - return mem - _safezip_msg = 'In safezip, len(args[0])=%d but len(args[%d])=%d' @@ -1514,58 +1469,6 @@ def safe_masked_invalid(x): return xm -class MemoryMonitor(object): - def __init__(self, nmax=20000): - self._nmax = nmax - self._mem = np.zeros((self._nmax,), np.int32) - self.clear() - - def clear(self): - self._n = 0 - self._overflow = False - - def __call__(self): - mem = report_memory() - if self._n < self._nmax: - self._mem[self._n] = mem - self._n += 1 - else: - self._overflow = True - return mem - - def report(self, segments=4): - n = self._n - segments = min(n, segments) - dn = int(n / segments) - ii = list(xrange(0, n, dn)) - ii[-1] = n - 1 - print() - print('memory report: i, mem, dmem, dmem/nloops') - print(0, self._mem[0]) - for i in range(1, len(ii)): - di = ii[i] - ii[i - 1] - if di == 0: - continue - dm = self._mem[ii[i]] - self._mem[ii[i - 1]] - print('%5d %5d %3d %8.3f' % (ii[i], self._mem[ii[i]], - dm, dm / float(di))) - if self._overflow: - print("Warning: array size was too small for the number of calls.") - - def xy(self, i0=0, isub=1): - x = np.arange(i0, self._n, isub) - return x, self._mem[i0:self._n:isub] - - def plot(self, i0=0, isub=1, fig=None): - if fig is None: - from .pylab import figure - fig = figure() - - ax = fig.add_subplot(111) - ax.plot(*self.xy(i0, isub)) - fig.canvas.draw() - - def print_cycles(objects, outstream=sys.stdout, show_progress=False): """ *objects* diff --git a/unit/agg_memleak.py b/unit/agg_memleak.py deleted file mode 100644 index c257da1f1f64..000000000000 --- a/unit/agg_memleak.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -And another broken test... -""" - -from __future__ import print_function -import os -from matplotlib.ft2font import FT2Font -from numpy.random import rand -from matplotlib.backend_bases import GraphicsContextBase -from matplotlib.backends._backend_agg import RendererAgg - - -def report_memory(i): - pid = os.getpid() - a2 = os.popen('ps -p %d -o rss,sz' % pid).readlines() - print(i, ' ', a2[1], end='') - return int(a2[1].split()[0]) - -fname = '/usr/local/share/matplotlib/Vera.ttf' - -N = 200 -for i in range(N): - gc = GraphicsContextBase() - gc.set_clip_rectangle([20, 20, 20, 20]) - o = RendererAgg(400, 400, 72) - - for j in range(50): - xs = [400 * int(rand()) for k in range(8)] - ys = [400 * int(rand()) for k in range(8)] - rgb = (1, 0, 0) - pnts = zip(xs, ys) - o.draw_polygon(gc, rgb, pnts) # no such method?? - o.draw_polygon(gc, None, pnts) - - for j in range(50): - x = [400 * int(rand()) for k in range(4)] - y = [400 * int(rand()) for k in range(4)] - o.draw_lines(gc, x, y) - - for j in range(50): - args = [400 * int(rand()) for k in range(4)] - rgb = (1, 0, 0) - o.draw_rectangle(gc, rgb, *args) - - if 1: # add text - font = FT2Font(fname) - font.clear() - font.set_text('hi mom', 60) - font.set_size(12, 72) - o.draw_text_image(font.get_image(), 30, 40, gc) - - o.write_png('aggtest%d.png' % i) - val = report_memory(i) - if i == 1: - start = val - -end = val -print('Average memory consumed per loop: %1.4f\n' % ((end - start) / float(N))) - -# w/o text and w/o write_png: Average memory consumed per loop: 0.02 -# w/o text and w/ write_png : Average memory consumed per loop: 0.3400 -# w/ text and w/ write_png : Average memory consumed per loop: 0.32 diff --git a/unit/memleak.py b/unit/memleak.py new file mode 100755 index 000000000000..12af02ceac84 --- /dev/null +++ b/unit/memleak.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import gc + +try: + import tracemalloc +except ImportError: + raise ImportError("This script requires Python 3.4 or later") + +try: + import psutil +except ImportError: + raise ImportError("This script requires psutil") + +import numpy as np + + +def run_memleak_test(bench, iterations, report): + tracemalloc.start() + + starti = min(50, iterations / 2) + endi = iterations + + malloc_arr = np.empty((endi,), dtype=np.int64) + rss_arr = np.empty((endi,), dtype=np.int64) + rss_peaks = np.empty((endi,), dtype=np.int64) + nobjs_arr = np.empty((endi,), dtype=np.int64) + garbage_arr = np.empty((endi,), dtype=np.int64) + open_files_arr = np.empty((endi,), dtype=np.int64) + rss_peak = 0 + + p = psutil.Process() + + for i in range(endi): + bench() + + gc.collect() + + rss = p.memory_info().rss + malloc, peak = tracemalloc.get_traced_memory() + nobjs = len(gc.get_objects()) + garbage = len(gc.garbage) + open_files = len(p.open_files()) + print("{0: 4d}: pymalloc {1: 10d}, rss {2: 10d}, nobjs {3: 10d}, garbage {4: 4d}, files: {5: 4d}".format( + i, malloc, rss, nobjs, garbage, open_files)) + + malloc_arr[i] = malloc + rss_arr[i] = rss + if rss > rss_peak: + rss_peak = rss + rss_peaks[i] = rss_peak + nobjs_arr[i] = nobjs + garbage_arr[i] = garbage + open_files_arr[i] = open_files + + print('Average memory consumed per loop: %1.4f bytes\n' % + (np.sum(rss_peaks[starti+1:] - rss_peaks[starti:-1]) / float(endi - starti))) + + from matplotlib import pyplot as plt + fig, (ax1, ax2, ax3) = plt.subplots(3) + ax1b = ax1.twinx() + ax1.plot(malloc_arr, 'r') + ax1b.plot(rss_arr, 'b') + ax1.set_ylabel('pymalloc', color='r') + ax1b.set_ylabel('rss', color='b') + + ax2b = ax2.twinx() + ax2.plot(nobjs_arr, 'r') + ax2b.plot(garbage_arr, 'b') + ax2.set_ylabel('total objects', color='r') + ax2b.set_ylabel('garbage objects', color='b') + + ax3.plot(open_files_arr) + ax3.set_ylabel('open file handles') + + if not report.endswith('.pdf'): + report = report + '.pdf' + fig.savefig(report, format='pdf') + + +class MemleakTest(object): + def __init__(self, empty): + self.empty = empty + + def __call__(self): + import matplotlib.pyplot as plt + + fig = plt.figure(1) + + if not self.empty: + t1 = np.arange(0.0, 2.0, 0.01) + y1 = np.sin(2 * np.pi * t1) + y2 = np.random.rand(len(t1)) + X = np.random.rand(50, 50) + + ax = fig.add_subplot(221) + ax.plot(t1, y1, '-') + ax.plot(t1, y2, 's') + + ax = fig.add_subplot(222) + ax.imshow(X) + + ax = fig.add_subplot(223) + ax.scatter(np.random.rand(50), np.random.rand(50), + s=100 * np.random.rand(50), c=np.random.rand(50)) + + ax = fig.add_subplot(224) + ax.pcolor(10 * np.random.rand(50, 50)) + + fig.savefig('tmp', dpi=75) + plt.close(1) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser('Run memory leak tests') + parser.add_argument('backend', type=str, nargs=1, + help='backend to test') + parser.add_argument('iterations', type=int, nargs=1, + help='number of iterations') + parser.add_argument('report', type=str, nargs=1, + help='filename to save report') + parser.add_argument('--empty', action='store_true', + help="Don't plot any content, just test creating " + "and destroying figures") + parser.add_argument('--interactive', action='store_true', + help="Turn on interactive mode to actually open " + "windows. Only works with some GUI backends.") + + + args = parser.parse_args() + + import matplotlib + matplotlib.use(args.backend[0]) + + if args.interactive: + from matplotlib import pyplot as plt + plt.ion() + + run_memleak_test(MemleakTest(args.empty), args.iterations[0], args.report[0]) diff --git a/unit/memleak_gui.py b/unit/memleak_gui.py deleted file mode 100755 index 1467846f12ad..000000000000 --- a/unit/memleak_gui.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python - -''' -This illustrates a leak that occurs with any interactive backend. -Run with : - - > python memleak_gui.py -dGTKAgg # or TkAgg, etc.. - -use --help option to see all options - -The default number of loops typically will not yield a stable -estimate--for that you may need many hundreds of loops and some patience. - -You may need to edit cbook.report_memory to support your platform - -''' -from __future__ import print_function -import gc -from optparse import OptionParser - -parser = OptionParser() -parser.add_option("-q", "--quiet", default=True, - action="store_false", dest="verbose") -parser.add_option("-s", "--start", dest="start", - default="30", - help="first index of averaging interval") -parser.add_option("-e", "--end", dest="end", - default="100", - help="last index of averaging interval") -parser.add_option("-t", "--toolbar", dest="toolbar", - default="toolbar2", - help="toolbar: None, classic, toolbar2") -# The following overrides matplotlib's version of the -d option -# uses it if found -parser.add_option("-d", "--backend", dest="backend", - default='', - help="backend") -parser.add_option("-c", "--cycles", dest="cycles", - default=False, action="store_true") - -options, args = parser.parse_args() - -indStart = int(options.start) -indEnd = int(options.end) -import matplotlib -matplotlib.rcParams['toolbar'] = matplotlib.validate_toolbar(options.toolbar) -if options.backend: - matplotlib.use(options.backend) -import pylab -import matplotlib.cbook as cbook - -print('# columns are: iteration, OS memory (k), number of python objects') -print('#') -for i in range(indEnd + 1): - - fig = pylab.figure() - fig.savefig('test') # This seems to just slow down the testing. - fig.clf() - pylab.close(fig) - gc.collect() - val = cbook.report_memory(i) - if options.verbose: - if i % 10 == 0: - #print ("iter: %4d OS memory: %8d Python objects: %8d" % - print("%4d %8d %8d" % (i, val, len(gc.get_objects()))) - if i == indStart: - start = val # wait a few cycles for memory usage to stabilize - -gc.collect() -end = val - -print('# columns above are: iteration, OS memory (k), number of python objects') -print('#') -print('# uncollectable list:', gc.garbage) -print('#') - -if i > indStart: - print('# Backend %(backend)s, toolbar %(toolbar)s' % matplotlib.rcParams) - backend = options.backend.lower() - if backend.startswith("gtk"): - import gtk - import gobject - print("# pygtk version: %s, gtk version: %s, pygobject version: %s, glib version: %s" % \ - (gtk.pygtk_version, gtk.gtk_version, - gobject.pygobject_version, gobject.glib_version)) - elif backend.startswith("qt4"): - import PyQt4.pyqtconfig - print("# PyQt4 version: %s, Qt version %x" % \ - (PyQt4.pyqtconfig.Configuration().pyqt_version_str, - PyQt4.pyqtconfig.Configuration().qt_version)) - elif backend.startswith("wx"): - import wx - print("# wxPython version: %s" % wx.__version__) - elif backend.startswith("tk"): - import Tkinter - print("# Tkinter version: %s, Tk version: %s, Tcl version: %s" % (Tkinter.__version__, Tkinter.TkVersion, Tkinter.TclVersion)) - - print('# Averaging over loops %d to %d' % (indStart, indEnd)) - print('# Memory went from %dk to %dk' % (start, end)) - print('# Average memory consumed per loop: %1.4fk bytes\n' % ((end - start) / float(indEnd - indStart))) - -if options.cycles: - cbook.print_cycles(gc.garbage) diff --git a/unit/memleak_hawaii3.py b/unit/memleak_hawaii3.py deleted file mode 100755 index e139abac6981..000000000000 --- a/unit/memleak_hawaii3.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function - -import gc -import matplotlib -matplotlib.use('PDF') - -from matplotlib.cbook import report_memory -import numpy as np -import matplotlib.pyplot as plt -# take a memory snapshot on indStart and compare it with indEnd - -rand = np.random.rand - -indStart, indEnd = 200, 401 -mem_size, coll_count = [], [] -for i in range(indEnd): - - fig = plt.figure(1) - fig.clf() - - t1 = np.arange(0.0, 2.0, 0.01) - y1 = np.sin(2 * np.pi * t1) - y2 = rand(len(t1)) - X = rand(50, 50) - - ax = fig.add_subplot(221) - ax.plot(t1, y1, '-') - ax.plot(t1, y2, 's') - - ax = fig.add_subplot(222) - ax.imshow(X) - - ax = fig.add_subplot(223) - ax.scatter(rand(50), rand(50), s=100 * rand(50), c=rand(50)) - - ax = fig.add_subplot(224) - ax.pcolor(10 * rand(50, 50)) - - fig.savefig('tmp%d' % i, dpi=75) - plt.close(1) - - coll = gc.collect() - val = report_memory(i) - print(i, val) - if i == indStart: - start = val # wait a few cycles for memory usage to stabilize - mem_size.append(val) - coll_count.append(coll) - -end = val -print('Average memory consumed per loop: %1.4fk bytes\n' % - ((end - start) / float(indEnd - indStart))) -fig, ax = plt.subplots() -ax2 = ax.twinx() -ax.plot(mem_size, 'r') -ax.set_ylabel('memory size', color='r') -ax2.plot(coll_count, 'k') -ax2.set_ylabel('collect count', color='k') -fig.savefig('report') diff --git a/unit/memleak_nongui.py b/unit/memleak_nongui.py deleted file mode 100644 index 01cdb4d831bc..000000000000 --- a/unit/memleak_nongui.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import print_function -import matplotlib -matplotlib.use('Agg') -from matplotlib.figure import Figure -from matplotlib.cbook import report_memory - - -def plot(): - fig = Figure() - i = 0 - while True: - print(i, report_memory(i)) - fig.clf() - ax = fig.add_axes([0.1, 0.1, 0.7, 0.7]) - ax.plot([1, 2, 3]) - i += 1 - -if __name__ == '__main__': - plot()