Skip to content

Commit 051e56e

Browse files
jbo-adsjbo-ads
authored
Merge pull request #221 from jbo-ads/subtile-thread-optimization
Fetch subtiles from second level dimension using multiple threads
2 parents 643a7e1 + 02ec060 commit 051e56e

File tree

3 files changed

+107
-2
lines changed

3 files changed

+107
-2
lines changed

‎include/mapcache.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1148,6 +1148,12 @@ struct mapcache_tileset {
11481148

11491149
mapcache_dimension_assembly_type dimension_assembly_type;
11501150

1151+
/**
1152+
* Maximum zoom level for activating multithreaded subtile retrieval
1153+
* -1 means 'not activated'
1154+
*/
1155+
int assembly_threaded_fetching_maxzoom;
1156+
11511157
/**
11521158
* image to be used as a watermark
11531159
*/

‎lib/configuration_xml.c‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,31 @@ void parseDimensions(mapcache_context *ctx, ezxml_t node, mapcache_tileset *tile
158158
}
159159
}
160160

161+
tileset->assembly_threaded_fetching_maxzoom = -1;
162+
dimension_node = ezxml_child(node,"assembly_threaded_fetching");
163+
if (dimension_node) {
164+
if (dimension_node && dimension_node->txt) {
165+
if (!strcmp(dimension_node->txt,"true")) {
166+
int maxzoom = INT_MAX;
167+
char * smaxzoom = (char*)ezxml_attr(dimension_node,"maxzoom");;
168+
if (smaxzoom && *smaxzoom) {
169+
char *endptr;
170+
maxzoom = (int)strtol(smaxzoom,&endptr,10);
171+
if(*endptr != 0 || maxzoom < 0) {
172+
ctx->set_error(ctx, 400, "failed to parse assembly_threaded_fetching"
173+
" maxzoom %s (expecting a positive integer)", smaxzoom);
174+
return;
175+
}
176+
}
177+
tileset->assembly_threaded_fetching_maxzoom = maxzoom;
178+
} else if (strcmp(dimension_node->txt,"false")) {
179+
ctx->set_error(ctx,400,"failed to parse <assembly_threaded_fetching>"
180+
" (%s), expecting \"true\" or \"false\"",dimension_node->txt);
181+
return;
182+
}
183+
}
184+
}
185+
161186
/* should we create subdimensions from source if not found in cache.
162187
e.g. if dimension=mosaic returns dimension=val1,val2,val3 should we
163188
query the wms source with dimension=val1 , dimension=val2 and/or

‎lib/tileset.c‎

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -795,11 +795,42 @@ int mapcache_tileset_tile_get_readonly(mapcache_context *ctx, mapcache_tile *til
795795

796796
typedef struct {
797797
mapcache_tile *tile;
798-
int cache_status;
798+
int isFetched;
799799
} mapcache_subtile;
800800

801801
static void mapcache_tileset_tile_get_without_subdimensions(mapcache_context *ctx, mapcache_tile *tile, int read_only);
802802

803+
#if APR_HAS_THREADS
804+
/* use a thread pool if using 1.3.12 or higher apu */
805+
#include "apu_version.h"
806+
#if (APU_MAJOR_VERSION <= 1 && APU_MINOR_VERSION <= 3)
807+
#define USE_THREADPOOL 0
808+
#include <apr_thread_proc.h>
809+
#else
810+
#define USE_THREADPOOL 1
811+
#include <apr_thread_pool.h>
812+
#endif
813+
814+
typedef struct {
815+
mapcache_subtile *subtile;
816+
mapcache_tile *tile;
817+
mapcache_context *ctx;
818+
} _thread_subtile;
819+
static void* APR_THREAD_FUNC _thread_get_subtile(apr_thread_t *thread, void *data)
820+
{
821+
_thread_subtile * t = (_thread_subtile *)data;
822+
/* creates the tile from the source, takes care of metatiling */
823+
mapcache_tileset_tile_get_without_subdimensions(t->ctx, t->subtile->tile,
824+
(t->tile->tileset->subdimension_read_only||!t->tile->tileset->source)?1:0);
825+
t->subtile->isFetched = MAPCACHE_TRUE;
826+
#if !USE_THREADPOOL
827+
apr_thread_exit(thread, APR_SUCCESS);
828+
#endif
829+
return NULL;
830+
}
831+
#endif // APR_HAS_THREADS
832+
833+
803834
void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, mapcache_tile *tile) {
804835
apr_array_header_t *subtiles;
805836
mapcache_extent extent;
@@ -813,6 +844,7 @@ void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, map
813844
*/
814845
subtiles = apr_array_make(ctx->pool,1,sizeof(mapcache_subtile));
815846
st.tile = tile;
847+
st.isFetched = MAPCACHE_FALSE;
816848
APR_ARRAY_PUSH(subtiles,mapcache_subtile) = st;
817849
mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent);
818850
if(GC_HAS_ERROR(ctx)) goto cleanup;
@@ -856,6 +888,7 @@ void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, map
856888
/* clone the existing subtiles if we have more than one sub-dimension to assemble for the the current dimension */
857889
for(k=1;k<single_subdimension->nelts;k++) {
858890
st.tile = mapcache_tileset_tile_clone(ctx->pool,APR_ARRAY_IDX(subtiles,j,mapcache_subtile).tile);
891+
st.isFetched = MAPCACHE_FALSE;
859892
APR_ARRAY_PUSH(subtiles,mapcache_subtile)=st;
860893
}
861894
}
@@ -872,10 +905,51 @@ void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, map
872905

873906
/* our subtiles array now contains a list of tiles with subdimensions split up, we now need to fetch them from the cache */
874907
/* note that subtiles[0].tile == tile */
908+
#if APR_HAS_THREADS
909+
if (tile->tileset->assembly_threaded_fetching_maxzoom != -1
910+
&& tile->z <= tile->tileset->assembly_threaded_fetching_maxzoom) {
911+
apr_thread_t **threads;
912+
apr_threadattr_t *thread_attrs;
913+
int nthreads;
914+
_thread_subtile * thread_subtiles;
915+
916+
nthreads = 0;
917+
apr_threadattr_create(&thread_attrs, ctx->pool);
918+
thread_subtiles = (_thread_subtile*)apr_pcalloc(ctx->pool,subtiles->nelts*sizeof(_thread_subtile));
919+
threads = (apr_thread_t**)apr_pcalloc(ctx->pool,subtiles->nelts*sizeof(apr_thread_t*));
920+
for(i=subtiles->nelts-1; i>=0; i--) {
921+
int rv;
922+
thread_subtiles[i].subtile = &(APR_ARRAY_IDX(subtiles,i,mapcache_subtile));
923+
thread_subtiles[i].tile = tile;
924+
thread_subtiles[i].ctx = ctx->clone(ctx);
925+
rv = apr_thread_create(&threads[i], thread_attrs, _thread_get_subtile,
926+
(void*)&(thread_subtiles[i]), thread_subtiles[i].ctx->pool);
927+
if (rv != APR_SUCCESS) {
928+
// In case of failure creating a thread, the subtile will be fetched in the main loop below
929+
break;
930+
}
931+
nthreads++;
932+
}
933+
for(i=0 ; i<nthreads ; i++) {
934+
int rv;
935+
apr_thread_join(&rv, threads[i]);
936+
if(rv != APR_SUCCESS) {
937+
ctx->set_error(ctx,500, "thread %d of %d failed on exit\n",i,nthreads);
938+
}
939+
if(GC_HAS_ERROR(thread_subtiles[i].ctx)) {
940+
/* transfer error message from child thread to main context */
941+
ctx->set_error(ctx,thread_subtiles[i].ctx->get_error(thread_subtiles[i].ctx),
942+
thread_subtiles[i].ctx->get_error_message(thread_subtiles[i].ctx));
943+
}
944+
}
945+
}
946+
#endif // APR_HAS_THREADS
875947

876948
for(i=subtiles->nelts-1; i>=0; i--) {
877949
mapcache_tile *subtile = APR_ARRAY_IDX(subtiles,i,mapcache_subtile).tile;
878-
mapcache_tileset_tile_get_without_subdimensions(ctx, subtile, (tile->tileset->subdimension_read_only||!tile->tileset->source)?1:0); /* creates the tile from the source, takes care of metatiling */
950+
if (!APR_ARRAY_IDX(subtiles,i,mapcache_subtile).isFetched) {
951+
mapcache_tileset_tile_get_without_subdimensions(ctx, subtile, (tile->tileset->subdimension_read_only||!tile->tileset->source)?1:0); /* creates the tile from the source, takes care of metatiling */
952+
}
879953
if(GC_HAS_ERROR(ctx))
880954
goto cleanup;
881955
if(!subtile->nodata) {

0 commit comments

Comments
 (0)