Skip to content

Make hexbin much faster #859

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 22, 2012
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
2012-05-22 Collections now have a setting "offset_position" to select whether
the offsets are given in "screen" coordinates (default,
following the old behavior) or "data" coordinates. This is currently
used internally to improve the performance of hexbin.

As a result, the "draw_path_collection" backend methods have grown
a new argument "offset_position". - MGD

2012-05-03 symlog scale now obeys the logarithmic base. Previously, it was
completely ignored and always treated as base e. - MGD

Expand Down
2 changes: 1 addition & 1 deletion examples/pylab_examples/hexbin_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import matplotlib.cm as cm
import matplotlib.pyplot as plt

np.random.seed(0)
n = 100000
x = np.random.standard_normal(n)
y = 2.0 + 3.0 * x + 4.0 * np.random.standard_normal(n)
Expand All @@ -33,4 +34,3 @@
cb.set_label('log10(N)')

plt.show()

47 changes: 24 additions & 23 deletions lib/matplotlib/axes.py
Original file line number Diff line number Diff line change
Expand Up @@ -2729,7 +2729,7 @@ def set_yticks(self, ticks, minor=False):

def get_ymajorticklabels(self):
"""
Get the major y tick labels as a list of
Get the major y tick labels as a list of
:class:`~matplotlib.text.Text` instances.
"""
return cbook.silent_list('Text yticklabel',
Expand Down Expand Up @@ -5207,7 +5207,7 @@ def errorbar(self, x, y, yerr=None, xerr=None,
only upper/lower limits. In that case a caret symbol is
used to indicate this. lims-arguments may be of the same
type as *xerr* and *yerr*.

*errorevery*: positive integer
subsamples the errorbars. Eg if everyerror=5, errorbars for every
5-th datapoint will be plotted. The data plot itself still shows
Expand Down Expand Up @@ -5361,7 +5361,7 @@ def xywhere(xs, ys, mask):
leftlo, ylo = xywhere(left, y, xlolims & everymask)
caplines.extend( self.plot(leftlo, ylo, 'k|', **plot_kw) )
else:

leftlo, ylo = xywhere(left, y, everymask)
caplines.extend( self.plot(leftlo, ylo, 'k|', **plot_kw) )

Expand Down Expand Up @@ -6175,43 +6175,44 @@ def hexbin(self, x, y, C = None, gridsize = 100, bins = None,
lattice1.astype(float).ravel(), lattice2.astype(float).ravel()))
good_idxs = ~np.isnan(accum)

px = xmin + sx * np.array([ 0.5, 0.5, 0.0, -0.5, -0.5, 0.0])
py = ymin + sy * np.array([-0.5, 0.5, 1.0, 0.5, -0.5, -1.0]) / 3.0

polygons = np.zeros((6, n, 2), float)
polygons[:,:nx1*ny1,0] = np.repeat(np.arange(nx1), ny1)
polygons[:,:nx1*ny1,1] = np.tile(np.arange(ny1), nx1)
polygons[:,nx1*ny1:,0] = np.repeat(np.arange(nx2) + 0.5, ny2)
polygons[:,nx1*ny1:,1] = np.tile(np.arange(ny2), nx2) + 0.5

offsets = np.zeros((n, 2), float)
offsets[:nx1*ny1,0] = np.repeat(np.arange(nx1), ny1)
offsets[:nx1*ny1,1] = np.tile(np.arange(ny1), nx1)
offsets[nx1*ny1:,0] = np.repeat(np.arange(nx2) + 0.5, ny2)
offsets[nx1*ny1:,1] = np.tile(np.arange(ny2), nx2) + 0.5
offsets[:,0] *= sx
offsets[:,1] *= sy
offsets[:,0] += xmin
offsets[:,1] += ymin
# remove accumulation bins with no data
polygons = polygons[:,good_idxs,:]
offsets = offsets[good_idxs,:]
accum = accum[good_idxs]

polygons = np.transpose(polygons, axes=[1,0,2])
polygons[:,:,0] *= sx
polygons[:,:,1] *= sy
polygons[:,:,0] += px
polygons[:,:,1] += py

if xscale=='log':
polygons[:,:,0] = 10**(polygons[:,:,0])
offsets[:,0] = 10**(offsets[:,0])
xmin = 10**xmin
xmax = 10**xmax
self.set_xscale('log')
if yscale=='log':
polygons[:,:,1] = 10**(polygons[:,:,1])
offsets[:,1] = 10**(offsets[:,1])
ymin = 10**ymin
ymax = 10**ymax
self.set_yscale('log')

polygon = np.zeros((6, 2), float)
polygon[:,0] = sx * np.array([ 0.5, 0.5, 0.0, -0.5, -0.5, 0.0])
polygon[:,1] = sy * np.array([-0.5, 0.5, 1.0, 0.5, -0.5, -1.0]) / 3.0

if edgecolors=='none':
edgecolors = 'face'

collection = mcoll.PolyCollection(
polygons,
[polygon],
edgecolors = edgecolors,
linewidths = linewidths,
transOffset = self.transData,
offsets = offsets,
transOffset = mtransforms.IdentityTransform(),
offset_position = "data"
)

if isinstance(norm, mcolors.LogNorm):
Expand Down
30 changes: 22 additions & 8 deletions lib/matplotlib/backend_bases.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,14 +186,16 @@ def draw_markers(self, gc, marker_path, marker_trans, path, trans, rgbFace=None)

def draw_path_collection(self, gc, master_transform, paths, all_transforms,
offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls):
linewidths, linestyles, antialiaseds, urls,
offset_position):
"""
Draws a collection of paths selecting drawing properties from
the lists *facecolors*, *edgecolors*, *linewidths*,
*linestyles* and *antialiaseds*. *offsets* is a list of
offsets to apply to each of the paths. The offsets in
*offsets* are first transformed by *offsetTrans* before being
applied.
applied. *offset_position* may be either "screen" or "data"
depending on the space that the offsets are in.

This provides a fallback implementation of
:meth:`draw_path_collection` that makes multiple calls to
Expand All @@ -213,8 +215,9 @@ def draw_path_collection(self, gc, master_transform, paths, all_transforms,
path_ids.append((path, transform))

for xo, yo, path_id, gc0, rgbFace in self._iter_collection(
gc, path_ids, offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls):
gc, master_transform, all_transforms, path_ids, offsets,
offsetTrans, facecolors, edgecolors, linewidths, linestyles,
antialiaseds, urls, offset_position):
path, transform = path_id
transform = transforms.Affine2D(transform.get_matrix()).translate(xo, yo)
self.draw_path(gc0, path, transform, rgbFace)
Expand All @@ -240,7 +243,7 @@ def draw_quad_mesh(self, gc, master_transform, meshWidth, meshHeight,

return self.draw_path_collection(
gc, master_transform, paths, [], offsets, offsetTrans, facecolors,
edgecolors, linewidths, [], [antialiased], [None])
edgecolors, linewidths, [], [antialiased], [None], 'screen')

def draw_gouraud_triangle(self, gc, points, colors, transform):
"""
Expand Down Expand Up @@ -302,9 +305,10 @@ def _iter_collection_raw_paths(self, master_transform, paths,
transform = all_transforms[i % Ntransforms]
yield path, transform + master_transform

def _iter_collection(self, gc, path_ids, offsets, offsetTrans, facecolors,
edgecolors, linewidths, linestyles, antialiaseds,
urls):
def _iter_collection(self, gc, master_transform, all_transforms,
path_ids, offsets, offsetTrans, facecolors,
edgecolors, linewidths, linestyles,
antialiaseds, urls, offset_position):
"""
This is a helper method (along with
:meth:`_iter_collection_raw_paths`) to make it easier to write
Expand All @@ -330,6 +334,7 @@ def _iter_collection(self, gc, path_ids, offsets, offsetTrans, facecolors,
*path_ids*; *gc* is a graphics context and *rgbFace* is a color to
use for filling the path.
"""
Ntransforms = len(all_transforms)
Npaths = len(path_ids)
Noffsets = len(offsets)
N = max(Npaths, Noffsets)
Expand Down Expand Up @@ -359,6 +364,15 @@ def _iter_collection(self, gc, path_ids, offsets, offsetTrans, facecolors,
path_id = path_ids[i % Npaths]
if Noffsets:
xo, yo = toffsets[i % Noffsets]
if offset_position == 'data':
if Ntransforms:
transform = all_transforms[i % Ntransforms] + master_transform
else:
transform = master_transform
xo, yo = transform.transform_point((xo, yo))
xp, yp = transform.transform_point((0, 0))
xo = -(xp - xo)
yo = -(yp - yo)
if Nfacecolors:
rgbFace = facecolors[i % Nfacecolors]
if Nedgecolors:
Expand Down
3 changes: 2 additions & 1 deletion lib/matplotlib/backends/backend_macosx.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ def draw_markers(self, gc, marker_path, marker_trans, path, trans, rgbFace=None)

def draw_path_collection(self, gc, master_transform, paths, all_transforms,
offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls):
linewidths, linestyles, antialiaseds, urls,
offset_position):
cliprect = gc.get_clip_rectangle()
clippath, clippath_transform = gc.get_clip_path()
if all_transforms:
Expand Down
53 changes: 53 additions & 0 deletions lib/matplotlib/backends/backend_pdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,8 @@ def __init__(self, filename):
self.markers = {}
self.multi_byte_charprocs = {}

self.paths = []

# The PDF spec recommends to include every procset
procsets = [ Name(x)
for x in "PDF Text ImageB ImageC ImageI".split() ]
Expand Down Expand Up @@ -505,9 +507,12 @@ def close(self):
xobjects[tup[0]] = tup[1]
for name, value in self.multi_byte_charprocs.iteritems():
xobjects[name] = value
for name, path, trans, ob, join, cap, padding in self.paths:
xobjects[name] = ob
self.writeObject(self.XObjectObject, xobjects)
self.writeImages()
self.writeMarkers()
self.writePathCollectionTemplates()
self.writeObject(self.pagesObject,
{ 'Type': Name('Pages'),
'Kids': self.pageList,
Expand Down Expand Up @@ -1259,6 +1264,28 @@ def writeMarkers(self):
self.output(Op.paint_path(False, fillp, strokep))
self.endStream()

def pathCollectionObject(self, gc, path, trans, padding):
name = Name('P%d' % len(self.paths))
ob = self.reserveObject('path %d' % len(self.paths))
self.paths.append(
(name, path, trans, ob, gc.get_joinstyle(), gc.get_capstyle(), padding))
return name

def writePathCollectionTemplates(self):
for (name, path, trans, ob, joinstyle, capstyle, padding) in self.paths:
pathops = self.pathOperations(path, trans, simplify=False)
bbox = path.get_extents(trans)
bbox = bbox.padded(padding)
self.beginStream(
ob.id, None,
{'Type': Name('XObject'), 'Subtype': Name('Form'),
'BBox': list(bbox.extents)})
self.output(GraphicsContextPdf.joinstyles[joinstyle], Op.setlinejoin)
self.output(GraphicsContextPdf.capstyles[capstyle], Op.setlinecap)
self.output(*pathops)
self.output(Op.paint_path(False, True, True))
self.endStream()

@staticmethod
def pathOperations(path, transform, clip=None, simplify=None):
cmds = []
Expand Down Expand Up @@ -1466,6 +1493,32 @@ def draw_path(self, gc, path, transform, rgbFace=None):
rgbFace is None and gc.get_hatch_path() is None)
self.file.output(self.gc.paint())

def draw_path_collection(self, gc, master_transform, paths, all_transforms,
offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls,
offset_position):

padding = np.max(linewidths)
path_codes = []
for i, (path, transform) in enumerate(self._iter_collection_raw_paths(
master_transform, paths, all_transforms)):
name = self.file.pathCollectionObject(gc, path, transform, padding)
path_codes.append(name)

output = self.file.output
output(Op.gsave)
lastx, lasty = 0, 0
for xo, yo, path_id, gc0, rgbFace in self._iter_collection(
gc, master_transform, all_transforms, path_codes, offsets,
offsetTrans, facecolors, edgecolors, linewidths, linestyles,
antialiaseds, urls, offset_position):

self.check_gc(gc0, rgbFace)
dx, dy = xo - lastx, yo - lasty
output(1, 0, 0, 1, dx, dy, Op.concat_matrix, path_id, Op.use_xobject)
lastx, lasty = xo, yo
output(Op.grestore)

def draw_markers(self, gc, marker_path, marker_trans, path, trans, rgbFace=None):
# For simple paths or small numbers of markers, don't bother
# making an XObject
Expand Down
8 changes: 5 additions & 3 deletions lib/matplotlib/backends/backend_ps.py
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,8 @@ def draw_markers(self, gc, marker_path, marker_trans, path, trans, rgbFace=None)

def draw_path_collection(self, gc, master_transform, paths, all_transforms,
offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls):
linewidths, linestyles, antialiaseds, urls,
offset_position):
write = self._pswriter.write

path_codes = []
Expand All @@ -640,8 +641,9 @@ def draw_path_collection(self, gc, master_transform, paths, all_transforms,
path_codes.append(name)

for xo, yo, path_id, gc0, rgbFace in self._iter_collection(
gc, path_codes, offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls):
gc, master_transform, all_transforms, path_codes, offsets,
offsetTrans, facecolors, edgecolors, linewidths, linestyles,
antialiaseds, urls, offset_position):
ps = "%g %g %s" % (xo, yo, path_id)
self._draw_ps(ps, gc0, rgbFace)

Expand Down
8 changes: 5 additions & 3 deletions lib/matplotlib/backends/backend_svg.py
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,8 @@ def draw_markers(self, gc, marker_path, marker_trans, path, trans, rgbFace=None)

def draw_path_collection(self, gc, master_transform, paths, all_transforms,
offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls):
linewidths, linestyles, antialiaseds, urls,
offset_position):
writer = self.writer
path_codes = []
writer.start(u'defs')
Expand All @@ -592,8 +593,9 @@ def draw_path_collection(self, gc, master_transform, paths, all_transforms,
writer.end(u'defs')

for xo, yo, path_id, gc0, rgbFace in self._iter_collection(
gc, path_codes, offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls):
gc, master_transform, all_transforms, path_codes, offsets,
offsetTrans, facecolors, edgecolors, linewidths, linestyles,
antialiaseds, urls, offset_position):
clipid = self._get_clip(gc0)
url = gc0.get_url()
if url is not None:
Expand Down
Loading