Merge pull request #13903 from rouault/GDALGetNumThreads

Add GDALGetNumThreads() and use it

Author: rouault

alg/gdal_tps.cpp

@@ -33,6 +33,7 @@
 #include "gdal_alg_priv.h"
 #include "gdal_priv.h"
 #include "gdalgenericinverse.h"
+#include "gdal_thread_pool.h"
 
 CPL_C_START
 CPLXMLNode *GDALSerializeTPSTransformer(void *pTransformArg);
@@ -243,15 +244,11 @@ void *GDALCreateTPSTransformerInt(int nGCPCount, const GDAL_GCP *pasGCPList,
     // and checking memory usage too...
     if (nGCPCount > 100)
     {
-        const char *pszWarpThreads =
-            CSLFetchNameValue(papszOptions, "NUM_THREADS");
-        if (pszWarpThreads == nullptr)
-            pszWarpThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "1");
-        if (EQUAL(pszWarpThreads, "ALL_CPUS"))
-            nThreads = CPLGetNumCPUs();
-        else
-            nThreads = atoi(pszWarpThreads);
-        nThreads = std::clamp(nThreads, 1, 2);
+        // We don't need more than 2 threads: one for forward transformation,
+        // and another one for reverse transformation.
+        nThreads = GDALGetNumThreads(papszOptions, "NUM_THREADS",
+                                     /* nMaxVal = */ 2,
+                                     /* bDefaultAllCPUs = */ false);
 
         // Do sanity checks w.r.t. available RAM
 

alg/gdalgrid.cpp

@@ -36,6 +36,7 @@
 #include "cpl_vsi.h"
 #include "cpl_worker_thread_pool.h"
 #include "gdal.h"
+#include "gdal_thread_pool.h"
 
 constexpr double TO_RADIANS = M_PI / 180.0;
 
@@ -3266,14 +3267,8 @@ GDALGridContext *GDALGridContextCreate(GDALGridAlgorithm eAlgorithm,
     /* -------------------------------------------------------------------- */
     /*  Start thread pool.                                                  */
     /* -------------------------------------------------------------------- */
-    const char *pszThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS");
-    int nThreads = 0;
-    if (EQUAL(pszThreads, "ALL_CPUS"))
-        nThreads = CPLGetNumCPUs();
-    else
-        nThreads = atoi(pszThreads);
-    if (nThreads > 128)
-        nThreads = 128;
+    const int nThreads = GDALGetNumThreads(GDAL_DEFAULT_MAX_THREAD_COUNT,
+                                           /* bDefaultToAllCPUs = */ true);
     if (nThreads > 1)
     {
         psContext->poWorkerThreadPool = new CPLWorkerThreadPool();

alg/gdalpansharpen.cpp

@@ -33,6 +33,7 @@
 #include "../frmts/vrt/vrtdataset.h"
 #include "gdal_priv.h"
 #include "gdal_priv_templates.hpp"
+#include "gdal_thread_pool.h"
 // #include "gdalsse_priv.h"
 
 // Limit types to practical use cases.
@@ -461,15 +462,8 @@ GDALPansharpenOperation::Initialize(const GDALPansharpenOptions *psOptionsIn)
         nThreads = CPLGetNumCPUs();
     else if (nThreads == 0)
     {
-        const char *pszNumThreads =
-            CPLGetConfigOption("GDAL_NUM_THREADS", nullptr);
-        if (pszNumThreads)
-        {
-            if (EQUAL(pszNumThreads, "ALL_CPUS"))
-                nThreads = CPLGetNumCPUs();
-            else
-                nThreads = std::max(0, std::min(128, atoi(pszNumThreads)));
-        }
+        nThreads = GDALGetNumThreads(GDAL_DEFAULT_MAX_THREAD_COUNT,
+                                     /* bDefaulAllCPUs = */ false);
     }
     if (nThreads > 1)
     {

alg/gdalwarpkernel.cpp

@@ -309,25 +309,13 @@ void *GWKThreadsCreate(char **papszWarpOptions,
                        GDALTransformerFunc /* pfnTransformer */,
                        void *pTransformerArg)
 {
-    const char *pszWarpThreads =
-        CSLFetchNameValue(papszWarpOptions, "NUM_THREADS");
-    if (pszWarpThreads == nullptr)
-        pszWarpThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "1");
-
-    int nThreads = 0;
-    if (EQUAL(pszWarpThreads, "ALL_CPUS"))
-        nThreads = CPLGetNumCPUs();
-    else
-        nThreads = atoi(pszWarpThreads);
-    if (nThreads <= 1)
-        nThreads = 0;
-    if (nThreads > 128)
-        nThreads = 128;
-
+    const int nThreads = GDALGetNumThreads(papszWarpOptions, "NUM_THREADS",
+                                           GDAL_DEFAULT_MAX_THREAD_COUNT,
+                                           /* bDefaultAllCPUs = */ false);
     GWKThreadData *psThreadData = new GWKThreadData();
     auto poThreadPool =
-        nThreads > 0 ? GDALGetGlobalThreadPool(nThreads) : nullptr;
-    if (nThreads && poThreadPool)
+        nThreads > 1 ? GDALGetGlobalThreadPool(nThreads) : nullptr;
+    if (poThreadPool)
     {
         psThreadData->nMaxThreads = nThreads;
         psThreadData->threadJobs.reset(new std::vector<GWKJobStruct>(

apps/ogr2ogr_lib.cpp

@@ -48,6 +48,7 @@
 #include "gdal_alg.h"
 #include "gdal_alg_priv.h"
 #include "gdal_priv.h"
+#include "gdal_thread_pool.h"
 #include "ogr_api.h"
 #include "ogr_core.h"
 #include "ogr_feature.h"
@@ -6240,26 +6241,13 @@ bool LayerTranslator::TranslateArrow(
     std::vector<GByte> abyModifiedWKB;
     const int nNumReprojectionThreads = []()
     {
-        const int nNumCPUs = CPLGetNumCPUs();
-        if (nNumCPUs <= 1)
-        {
-            return 1;
-        }
-        else
-        {
-            const char *pszNumThreads =
-                CPLGetConfigOption("GDAL_NUM_THREADS", nullptr);
-            if (pszNumThreads)
-            {
-                if (EQUAL(pszNumThreads, "ALL_CPUS"))
-                    return CPLGetNumCPUs();
-                return std::min(atoi(pszNumThreads), 1024);
-            }
-            else
-            {
-                return std::max(2, nNumCPUs / 2);
-            }
-        }
+        const char *pszNumThreads = nullptr;
+        int nVal =
+            GDALGetNumThreads(GDAL_DEFAULT_MAX_THREAD_COUNT,
+                              /* bDefaultToAllCPUs = */ false, &pszNumThreads);
+        if (!pszNumThreads)
+            nVal = std::max(1, CPLGetNumCPUs() / 2);
+        return nVal;
     }();
 
     // Somewhat arbitrary threshold (config option only/mostly for autotest purposes)

autotest/cpp/test_gdal_algorithm.cpp

@@ -4862,6 +4862,13 @@ TEST_F(test_gdal_algorithm, AddNumThreadsArg)
         EXPECT_EQ(alg.m_numThreads, 1);
     }
 
+    {
+        CPLConfigOptionSetter oSetter("GDAL_NUM_THREADS", "ALL_CPUS", false);
+        MyAlgorithm alg;
+        EXPECT_TRUE(alg.ParseCommandLineArguments({}));
+        EXPECT_EQ(alg.m_numThreads, CPLGetNumCPUs());
+    }
+
     {
         MyAlgorithm alg;
         EXPECT_TRUE(alg.ParseCommandLineArguments({"--num-threads=1"}));

frmts/avif/avifdataset.cpp

@@ -18,6 +18,7 @@
 #include "avifdrivercore.h"
 #include "gdalexif.h"
 #include "memdataset.h"
+#include "gdal_thread_pool.h"
 
 #include <avif/avif.h>
 
@@ -929,13 +930,9 @@ GDALAVIFDataset::CreateCopy(const char *pszFilename, GDALDataset *poSrcDS,
             avifCodecChoiceFromName(CPLString(pszCodec).tolower().c_str());
     }
 
-    const char *pszThreads = CSLFetchNameValueDef(
-        papszOptions, "NUM_THREADS",
-        CPLGetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS"));
-    if (pszThreads && !EQUAL(pszThreads, "ALL_CPUS"))
-        encoder->maxThreads = atoi(pszThreads);
-    else
-        encoder->maxThreads = CPLGetNumCPUs();
+    encoder->maxThreads = GDALGetNumThreads(papszOptions, "NUM_THREADS",
+                                            GDAL_DEFAULT_MAX_THREAD_COUNT,
+                                            /* bDefaultToAllCPUs = */ true);
 
 #if AVIF_VERSION_MAJOR >= 1
     encoder->quality = nQuality;

frmts/basisu_ktx2/common.cpp

@@ -14,6 +14,7 @@
 #include "gdal_frmts.h"
 #include "common.h"
 #include "include_basisu_sdk.h"
+#include "gdal_thread_pool.h"
 
 #include <algorithm>
 #include <mutex>
@@ -264,11 +265,8 @@ bool GDAL_KTX2_BASISU_CreateCopy(const char *pszFilename, GDALDataset *poSrcDS,
         params.m_mip_srgb = params.m_perceptual;
     }
 
-    const int nNumThreads = std::max(
-        1, atoi(CSLFetchNameValueDef(
-               papszOptions, "NUM_THREADS",
-               CPLGetConfigOption("GDAL_NUM_THREADS",
-                                  CPLSPrintf("%d", CPLGetNumCPUs())))));
+    const int nNumThreads = GDALGetNumThreads(GDAL_DEFAULT_MAX_THREAD_COUNT,
+                                              /* bDefaultAllCPUs = */ true);
     CPLDebug("KTX2", "Using %d threads", nNumThreads);
     if (params.m_uastc)
     {

frmts/gti/gdaltileindexdataset.cpp

@@ -29,6 +29,7 @@
 #include "cpl_mem_cache.h"
 #include "cpl_minixml.h"
 #include "cpl_quad_tree.h"
+#include "cpl_worker_thread_pool.h"
 #include "vrtdataset.h"
 #include "vrt_priv.h"
 #include "ogrsf_frmts.h"
@@ -3753,15 +3754,8 @@ int GDALTileIndexDataset::GetNumThreads() const
         CSLFetchNameValueDef(GetOpenOptions(), "NUM_THREADS", nullptr);
     if (!pszNumThreads)
         pszNumThreads = CPLGetConfigOption("GTI_NUM_THREADS", nullptr);
-    if (!pszNumThreads)
-        pszNumThreads = CPLGetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS");
-    if (EQUAL(pszNumThreads, "0") || EQUAL(pszNumThreads, "1"))
-        return atoi(pszNumThreads);
-    const int nMaxPoolSize = GDALGetMaxDatasetPoolSize();
-    const int nLimit = std::min(CPLGetNumCPUs(), nMaxPoolSize);
-    if (EQUAL(pszNumThreads, "ALL_CPUS"))
-        return nLimit;
-    return std::min(atoi(pszNumThreads), nLimit);
+    return GDALGetNumThreads(pszNumThreads, GDALGetMaxDatasetPoolSize(),
+                             /* bDefaultAllCPUs = */ true);
 }
 
 /************************************************************************/

frmts/gtiff/cogdriver.cpp

@@ -677,11 +677,12 @@ static std::unique_ptr<GDALDataset> CreateReprojectedDS(
     std::unique_ptr<CPLConfigOptionSetter> poWarpThreadSetter;
     if (pszNumThreads)
     {
-        poWarpThreadSetter.reset(new CPLConfigOptionSetter(
-            "GDAL_NUM_THREADS", pszNumThreads, false));
+        poWarpThreadSetter = std::make_unique<CPLConfigOptionSetter>(
+            "GDAL_NUM_THREADS", pszNumThreads, false);
     }
 
     auto hRet = GDALWarp(osTmpFile, nullptr, 1, &hSrcDS, psOptions, nullptr);
+    CPL_IGNORE_RET_VAL(poWarpThreadSetter);
     GDALWarpAppOptionsFree(psOptions);
     CPLDebug("COG", "Reprojecting source dataset: end");