Skip to content

Commit ed8bd4e

Browse files
rouaultrouault
committed
Add a 'materialize' step for pipelines
1 parent 43d3e55 commit ed8bd4e

15 files changed

+984
-0
lines changed

‎apps/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ target_sources(appslib PRIVATE
3737
gdalalg_dataset_copy.cpp
3838
gdalalg_dataset_rename.cpp
3939
gdalalg_dataset_delete.cpp
40+
gdalalg_materialize.cpp
4041
gdalalg_mdim.cpp
4142
gdalalg_mdim_info.cpp
4243
gdalalg_mdim_convert.cpp

‎apps/gdalalg_materialize.cpp‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
/******************************************************************************
2+
*
3+
* Project: GDAL
4+
* Purpose: gdal "materialize" pipeline step
5+
* Author: Even Rouault <even dot rouault at spatialys.com>
6+
*
7+
******************************************************************************
8+
* Copyright (c) 2025, Even Rouault <even dot rouault at spatialys.com>
9+
*
10+
* SPDX-License-Identifier: MIT
11+
****************************************************************************/
12+
13+
#include "gdalalg_materialize.h"
14+
#include "gdal_utils.h"
15+
#include "gdal_priv.h"
16+
#include "ogrsf_frmts.h"
17+
18+
//! @cond Doxygen_Suppress
19+
20+
#ifndef _
21+
#define _(x) (x)
22+
#endif
23+
24+
/************************************************************************/
25+
/* GDALMaterializeRasterAlgorithm() */
26+
/************************************************************************/
27+
28+
GDALMaterializeRasterAlgorithm::GDALMaterializeRasterAlgorithm()
29+
: GDALMaterializeStepAlgorithm<GDALRasterPipelineStepAlgorithm,
30+
GDAL_OF_RASTER>(HELP_URL)
31+
{
32+
AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER,
33+
/* positionalAndRequired = */ false,
34+
_("Materialized dataset name"))
35+
.SetDatasetInputFlags(GADV_NAME);
36+
37+
AddOutputFormatArg(&m_format)
38+
.AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
39+
{GDAL_DCAP_RASTER, GDAL_DCAP_CREATECOPY,
40+
GDAL_DCAP_OPEN, GDAL_DMD_EXTENSIONS})
41+
.AddMetadataItem(GAAMDI_ALLOWED_FORMATS, {"MEM", "COG"})
42+
.AddMetadataItem(GAAMDI_EXCLUDED_FORMATS, {"VRT"});
43+
44+
AddCreationOptionsArg(&m_creationOptions);
45+
AddOverwriteArg(&m_overwrite);
46+
}
47+
48+
/************************************************************************/
49+
/* GDALMaterializeRasterAlgorithm::RunStep() */
50+
/************************************************************************/
51+
52+
bool GDALMaterializeRasterAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
53+
{
54+
auto pfnProgress = ctxt.m_pfnProgress;
55+
auto pProgressData = ctxt.m_pProgressData;
56+
57+
auto poSrcDS = m_inputDataset[0].GetDatasetRef();
58+
CPLAssert(poSrcDS);
59+
CPLAssert(!m_outputDataset.GetDatasetRef());
60+
61+
if (m_format.empty())
62+
m_format = "GTiff";
63+
64+
auto poDrv = GetGDALDriverManager()->GetDriverByName(m_format.c_str());
65+
if (!poDrv)
66+
{
67+
ReportError(CE_Failure, CPLE_AppDefined, "Driver %s does not exist",
68+
m_format.c_str());
69+
return false;
70+
}
71+
72+
std::string filename = m_outputDataset.GetName();
73+
const bool autoDeleteFile =
74+
filename.empty() && !EQUAL(m_format.c_str(), "MEM");
75+
if (autoDeleteFile)
76+
{
77+
filename = CPLGenerateTempFilenameSafe(nullptr);
78+
79+
const char *pszExt = poDrv->GetMetadataItem(GDAL_DMD_EXTENSIONS);
80+
if (pszExt)
81+
{
82+
filename += '.';
83+
filename += CPLStringList(CSLTokenizeString(pszExt))[0];
84+
}
85+
}
86+
87+
CPLStringList aosOptions(m_creationOptions);
88+
if (EQUAL(m_format.c_str(), "GTiff"))
89+
{
90+
if (aosOptions.FetchNameValue("TILED") == nullptr)
91+
{
92+
aosOptions.SetNameValue("TILED", "YES");
93+
}
94+
if (aosOptions.FetchNameValue("COPY_SRC_OVERVIEWS") == nullptr)
95+
{
96+
aosOptions.SetNameValue("COPY_SRC_OVERVIEWS", "YES");
97+
}
98+
if (aosOptions.FetchNameValue("COMPRESS") == nullptr)
99+
{
100+
const char *pszCOList =
101+
poDrv->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST);
102+
aosOptions.SetNameValue(
103+
"COMPRESS",
104+
pszCOList && strstr(pszCOList, "ZSTD") ? "ZSTD" : "DEFLATE");
105+
}
106+
}
107+
108+
if (autoDeleteFile)
109+
{
110+
aosOptions.SetNameValue("@SUPPRESS_ASAP", "YES");
111+
}
112+
113+
auto poOutDS = std::unique_ptr<GDALDataset>(
114+
poDrv->CreateCopy(filename.c_str(), poSrcDS, false, aosOptions.List(),
115+
pfnProgress, pProgressData));
116+
bool ok = poOutDS != nullptr && poOutDS->FlushCache() == CE_None;
117+
if (poOutDS)
118+
{
119+
if (poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED))
120+
{
121+
ok = poOutDS->Close() == CE_None;
122+
poOutDS.reset();
123+
if (ok)
124+
{
125+
const char *const apszAllowedDrivers[] = {m_format.c_str(),
126+
nullptr};
127+
poOutDS.reset(GDALDataset::Open(
128+
filename.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR,
129+
apszAllowedDrivers));
130+
ok = poOutDS != nullptr;
131+
}
132+
}
133+
if (ok)
134+
{
135+
if (autoDeleteFile)
136+
{
137+
#if !defined(_WIN32)
138+
if (poDrv->GetMetadataItem(GDAL_DCAP_CAN_READ_AFTER_DELETE))
139+
{
140+
CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler);
141+
poDrv->Delete(poOutDS.get(),
142+
CPLStringList(poOutDS->GetFileList()).List());
143+
}
144+
#endif
145+
poOutDS->MarkSuppressOnClose();
146+
}
147+
148+
m_outputDataset.Set(std::move(poOutDS));
149+
}
150+
}
151+
return ok;
152+
}
153+
154+
/************************************************************************/
155+
/* GDALMaterializeVectorAlgorithm() */
156+
/************************************************************************/
157+
158+
GDALMaterializeVectorAlgorithm::GDALMaterializeVectorAlgorithm()
159+
: GDALMaterializeStepAlgorithm<GDALVectorPipelineStepAlgorithm,
160+
GDAL_OF_VECTOR>(HELP_URL)
161+
{
162+
AddOutputDatasetArg(&m_outputDataset, GDAL_OF_VECTOR,
163+
/* positionalAndRequired = */ false,
164+
_("Materialized dataset name"))
165+
.SetDatasetInputFlags(GADV_NAME);
166+
167+
AddOutputFormatArg(&m_format)
168+
.AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
169+
{GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE, GDAL_DCAP_OPEN,
170+
GDAL_DMD_EXTENSIONS})
171+
.AddMetadataItem(GAAMDI_ALLOWED_FORMATS, {"MEM"})
172+
.AddMetadataItem(GAAMDI_EXCLUDED_FORMATS,
173+
{"MBTiles", "MVT", "PMTiles", "JP2ECW"});
174+
175+
AddCreationOptionsArg(&m_creationOptions);
176+
AddLayerCreationOptionsArg(&m_layerCreationOptions);
177+
AddOverwriteArg(&m_overwrite);
178+
}
179+
180+
/************************************************************************/
181+
/* GDALMaterializeVectorAlgorithm::RunStep() */
182+
/************************************************************************/
183+
184+
bool GDALMaterializeVectorAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
185+
{
186+
auto pfnProgress = ctxt.m_pfnProgress;
187+
auto pProgressData = ctxt.m_pProgressData;
188+
189+
auto poSrcDS = m_inputDataset[0].GetDatasetRef();
190+
CPLAssert(poSrcDS);
191+
CPLAssert(!m_outputDataset.GetDatasetRef());
192+
193+
if (m_format.empty())
194+
{
195+
bool bSeveralGeomFields = false;
196+
for (const auto *poLayer : poSrcDS->GetLayers())
197+
{
198+
if (!bSeveralGeomFields)
199+
bSeveralGeomFields =
200+
poLayer->GetLayerDefn()->GetGeomFieldCount() > 1;
201+
if (!bSeveralGeomFields &&
202+
poLayer->GetLayerDefn()->GetGeomFieldCount() > 0)
203+
{
204+
for (const auto *poFieldDefn :
205+
poLayer->GetLayerDefn()->GetFields())
206+
{
207+
const auto eType = poFieldDefn->GetType();
208+
if (eType == OFTStringList || eType == OFTIntegerList ||
209+
eType == OFTRealList || eType == OFTInteger64List)
210+
{
211+
bSeveralGeomFields = true;
212+
}
213+
}
214+
}
215+
}
216+
m_format = bSeveralGeomFields ? "SQLite" : "GPKG";
217+
}
218+
219+
auto poDrv = GetGDALDriverManager()->GetDriverByName(m_format.c_str());
220+
if (!poDrv)
221+
{
222+
ReportError(CE_Failure, CPLE_AppDefined, "Driver %s does not exist",
223+
m_format.c_str());
224+
return false;
225+
}
226+
227+
std::string filename = m_outputDataset.GetName();
228+
const bool autoDeleteFile =
229+
filename.empty() && !EQUAL(m_format.c_str(), "MEM");
230+
if (autoDeleteFile)
231+
{
232+
filename = CPLGenerateTempFilenameSafe(nullptr);
233+
234+
const char *pszExt = poDrv->GetMetadataItem(GDAL_DMD_EXTENSIONS);
235+
if (pszExt)
236+
{
237+
filename += '.';
238+
filename += CPLStringList(CSLTokenizeString(pszExt))[0];
239+
}
240+
}
241+
242+
CPLStringList aosOptions;
243+
aosOptions.AddString("--invoked-from-gdal-vector-convert");
244+
if (!m_overwrite)
245+
{
246+
aosOptions.AddString("--no-overwrite");
247+
}
248+
249+
aosOptions.AddString("-of");
250+
aosOptions.AddString(m_format.c_str());
251+
for (const auto &co : m_creationOptions)
252+
{
253+
aosOptions.AddString("-dsco");
254+
aosOptions.AddString(co.c_str());
255+
}
256+
if (EQUAL(m_format.c_str(), "SQLite"))
257+
{
258+
const char *pszCOList =
259+
poDrv->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST);
260+
if (pszCOList && strstr(pszCOList, "SPATIALITE") &&
261+
CPLStringList(m_creationOptions).FetchNameValue("SPATIALITE") ==
262+
nullptr)
263+
{
264+
aosOptions.AddString("-dsco");
265+
aosOptions.AddString("SPATIALITE=YES");
266+
}
267+
}
268+
for (const auto &co : m_layerCreationOptions)
269+
{
270+
aosOptions.AddString("-lco");
271+
aosOptions.AddString(co.c_str());
272+
}
273+
if (pfnProgress && pfnProgress != GDALDummyProgress)
274+
{
275+
aosOptions.AddString("-progress");
276+
}
277+
278+
if (autoDeleteFile)
279+
{
280+
aosOptions.AddString("-dsco");
281+
aosOptions.AddString("@SUPPRESS_ASAP=YES");
282+
}
283+
284+
GDALVectorTranslateOptions *psOptions =
285+
GDALVectorTranslateOptionsNew(aosOptions.List(), nullptr);
286+
GDALVectorTranslateOptionsSetProgress(psOptions, pfnProgress,
287+
pProgressData);
288+
289+
GDALDatasetH hSrcDS = GDALDataset::ToHandle(poSrcDS);
290+
auto poOutDS = std::unique_ptr<GDALDataset>(
291+
GDALDataset::FromHandle(GDALVectorTranslate(
292+
filename.c_str(), nullptr, 1, &hSrcDS, psOptions, nullptr)));
293+
GDALVectorTranslateOptionsFree(psOptions);
294+
295+
bool ok = poOutDS != nullptr && poOutDS->FlushCache() == CE_None;
296+
if (poOutDS)
297+
{
298+
if (poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED))
299+
{
300+
ok = poOutDS->Close() == CE_None;
301+
poOutDS.reset();
302+
if (ok)
303+
{
304+
const char *const apszAllowedDrivers[] = {m_format.c_str(),
305+
nullptr};
306+
poOutDS.reset(GDALDataset::Open(
307+
filename.c_str(), GDAL_OF_VECTOR | GDAL_OF_VERBOSE_ERROR,
308+
apszAllowedDrivers));
309+
ok = poOutDS != nullptr;
310+
}
311+
}
312+
if (ok)
313+
{
314+
if (autoDeleteFile)
315+
{
316+
#if !defined(_WIN32)
317+
if (poDrv->GetMetadataItem(GDAL_DCAP_CAN_READ_AFTER_DELETE))
318+
{
319+
CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler);
320+
poDrv->Delete(poOutDS.get(),
321+
CPLStringList(poOutDS->GetFileList()).List());
322+
}
323+
#endif
324+
poOutDS->MarkSuppressOnClose();
325+
}
326+
327+
m_outputDataset.Set(std::move(poOutDS));
328+
}
329+
}
330+
return ok;
331+
}
332+
333+
//! @endcond

0 commit comments

Comments
 (0)