Skip to content

Commit 976db37

Browse files
jcupittkleisauke
jcupitt
and
kleisauke
authored
Revised threading system (#3105)
* reimplement threadpool just a set of threads that get recycled "ninja test" passes and dzsave seems to work, though pytest fails for some reason * clean up threading code a bit move base stuff into thread.c, so we have a simple thread -> threadset -> threadpool layering * start trying to revert g_threadpool based on the original commit * working! nice low systime again still need to repply cnages to threadpool.c since 80e0cc3 * reapply fixes from master so threadpool.c is now up to date * rename VipsThread as VipsWorker a bit less confusing * use a semaphore to count workers in a pool * tidy up * formatting * dynamic threadpool sizing based on counting the number of blocked threads in each pool it works, but the improvement is not great :( * add "concurrency" metadata item so operators can hint threadpool size (dzsave especially) * don't use thinstrip for small images * add RGB mode to openslide since flatten was taking 20% of CPU time for dzsave * fix up rgb mode now actually works * make the tile buffer per thread in the new openslideload rgb mode * fix dynamic pool downsize * mild refactoring * fix the buffer system oops, turned it off by mistake * all done! * revise changelog * Update libvips/iofuncs/threadset.c Co-authored-by: Kleis Auke Wolthuizen <github@kleisauke.nl> * Update libvips/iofuncs/threadset.c Co-authored-by: Kleis Auke Wolthuizen <github@kleisauke.nl> * LSan: add libMagickCore to suppression file * Revert "Remove mutex lock for VipsThreadStartFn" This reverts commit 4144049. * add VIPS_MAX_THREADS to set a hard limit on the threadset size * Revert "Revert "Remove mutex lock for VipsThreadStartFn"" This reverts commit 77e8520. * remove sslock from sink.c * move fixed threadpool build to init not first use * add some doc comments * revert test suite threshold change * add a test for MAX_THREADS and move the test tmp/ area into the builddir * limit VIPS_MAX_THREADS to sane values * use tabs rather than spaces Co-authored-by: Kleis Auke Wolthuizen <github@kleisauke.nl>
1 parent ea0912f commit 976db37

25 files changed

+1369
-733
lines changed

‎ChangeLog

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ master
1313
- add ".pnm" save [ewelot]
1414
- threaded tiff jp2k and jpeg decompress
1515
- improve speed and efficiency of animated WebP write [dloebl]
16+
- add rgb mode to openslideload
17+
- new thread recycler
18+
- threadpools size dynamically with load
19+
- operations can hint threadpool size
1620
- support for N-colour ICC profiles
1721

1822
11/10/22 started 8.13.3

‎libvips/colour/colour.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,13 @@ vips_colour_gen( VipsRegion *or,
234234
int i, y;
235235
VipsPel *p[MAX_INPUT_IMAGES], *q;
236236

237+
/*
238+
printf( "vips_colour_gen: %s, "
239+
"left = %d, top = %d, width = %d, height = %d\n",
240+
VIPS_OBJECT_CLASS( class )->nickname,
241+
r->left, r->top, r->width, r->height );
242+
*/
243+
237244
if( vips_reorder_prepare_many( or->im, ir, r ) )
238245
return( -1 );
239246

‎libvips/conversion/sequential.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ vips_sequential_generate( VipsRegion *or,
126126

127127
VIPS_GATE_START( "vips_sequential_generate: wait" );
128128

129-
g_mutex_lock( sequential->lock );
129+
vips__worker_lock( sequential->lock );
130130

131131
VIPS_GATE_STOP( "vips_sequential_generate: wait" );
132132

‎libvips/conversion/tilecache.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -620,7 +620,7 @@ vips_tile_cache_gen( VipsRegion *or,
620620

621621
VIPS_GATE_START( "vips_tile_cache_gen: wait1" );
622622

623-
g_mutex_lock( cache->lock );
623+
vips__worker_lock( cache->lock );
624624

625625
VIPS_GATE_STOP( "vips_tile_cache_gen: wait1" );
626626

@@ -895,7 +895,7 @@ vips_line_cache_gen( VipsRegion *or,
895895

896896
VIPS_GATE_START( "vips_line_cache_gen: wait" );
897897

898-
g_mutex_lock( block_cache->lock );
898+
vips__worker_lock( block_cache->lock );
899899

900900
VIPS_GATE_STOP( "vips_line_cache_gen: wait" );
901901

‎libvips/create/black.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,14 @@ static int
8383
vips_black_gen( VipsRegion *or, void *seq, void *a, void *b,
8484
gboolean *stop )
8585
{
86+
/*
87+
VipsRect *r = &or->valid;
88+
89+
printf( "vips_black_gen: "
90+
"left = %d, top = %d, width = %d, height = %d\n",
91+
r->left, r->top, r->width, r->height );
92+
*/
93+
8694
vips_region_black( or );
8795

8896
return( 0 );

‎libvips/foreign/dzsave.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1900,6 +1900,8 @@ strip_work( VipsThreadState *state, void *a )
19001900

19011901
g_mutex_unlock( vips__global_lock );
19021902

1903+
vips_image_set_int( x, VIPS_META_CONCURRENCY, 1 );
1904+
19031905
if( write_image( dz, out, x, dz->suffix ) ) {
19041906
g_object_unref( out );
19051907
g_object_unref( x );

‎libvips/foreign/fits.c

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -450,11 +450,13 @@ fits2vips_generate( VipsRegion *out,
450450

451451
q = VIPS_REGION_ADDR( out, r->left, r->top );
452452

453-
g_mutex_lock( fits->lock );
453+
vips__worker_lock( fits->lock );
454+
454455
if( vips_fits_read_subset( fits, fpixel, lpixel, inc, q ) ) {
455456
g_mutex_unlock( fits->lock );
456457
return( -1 );
457458
}
459+
458460
g_mutex_unlock( fits->lock );
459461
}
460462
else {
@@ -478,12 +480,14 @@ fits2vips_generate( VipsRegion *out,
478480

479481
q = VIPS_REGION_ADDR( out, r->left, y );
480482

481-
g_mutex_lock( fits->lock );
483+
vips__worker_lock( fits->lock );
484+
482485
if( vips_fits_read_subset( fits,
483486
fpixel, lpixel, inc, q ) ) {
484487
g_mutex_unlock( fits->lock );
485488
return( -1 );
486489
}
490+
487491
g_mutex_unlock( fits->lock );
488492
}
489493
}

‎libvips/foreign/magick2vips.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -754,9 +754,11 @@ magick_fill_region( VipsRegion *out,
754754

755755
PixelPacket *pixels;
756756

757-
g_mutex_lock( read->lock );
757+
vips__worker_lock( read->lock );
758+
758759
pixels = get_pixels( read->frames[frame],
759760
r->left, line, r->width, 1 );
761+
760762
g_mutex_unlock( read->lock );
761763

762764
if( !pixels ) {

‎libvips/foreign/magick7load.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -676,10 +676,12 @@ vips_foreign_load_magick7_fill_region( VipsRegion *or,
676676
Quantum * restrict p;
677677
VipsPel * restrict q;
678678

679-
g_mutex_lock( magick7->lock );
679+
vips__worker_lock( magick7->lock );
680+
680681
p = GetCacheViewAuthenticPixels( magick7->cache_view[frame],
681682
r->left, line, r->width, 1,
682683
magick7->exception );
684+
683685
g_mutex_unlock( magick7->lock );
684686

685687
if( !p )

0 commit comments

Comments
 (0)