diff --git a/src/main.js b/src/main.js index f06cd3869..5203411f6 100644 --- a/src/main.js +++ b/src/main.js @@ -109,6 +109,10 @@ program '-f|--log_format ', 'define the log format: https://github.com/expressjs/morgan#morganformat-options', ) + .option( + '--ignore-missing-files', + 'Continue startup even if configured mbtiles/pmtiles files are missing', + ) .version(packageJson.version, '-v, --version'); program.parse(process.argv); const opts = program.opts(); @@ -132,6 +136,7 @@ const startServer = (configPath, config) => { logFormat: opts.log_format, fetchTimeout: opts.fetchTimeout, publicUrl, + ignoreMissingFiles: opts.ignoreMissingFiles, }); }; diff --git a/src/serve_data.js b/src/serve_data.js index 82de02c17..d26bed1cb 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -508,7 +508,7 @@ export const serve_data = { * @returns {Promise} */ add: async function (options, repo, params, id, programOpts) { - const { publicUrl, verbose } = programOpts; + const { publicUrl, verbose, ignoreMissingFiles } = programOpts; let inputFile; let inputType; if (params.pmtiles) { @@ -542,8 +542,18 @@ export const serve_data = { // Only check file stats for local files, not remote URLs if (!isValidRemoteUrl(inputFile)) { - const inputFileStats = await fsp.stat(inputFile); - if (!inputFileStats.isFile() || inputFileStats.size === 0) { + try { + const inputFileStats = await fsp.stat(inputFile); + if (!inputFileStats.isFile() || inputFileStats.size === 0) { + throw Error(`Not valid input file: "${inputFile}"`); + } + } catch (err) { + if (ignoreMissingFiles) { + console.log( + `WARN: Data source '${id}' file not found: "${inputFile}" - skipping`, + ); + return; + } throw Error(`Not valid input file: "${inputFile}"`); } } @@ -555,24 +565,34 @@ export const serve_data = { tileJSON['encoding'] = params['encoding']; tileJSON['tileSize'] = params['tileSize']; - if (inputType === 'pmtiles') { - source = openPMtiles( - inputFile, - params.s3Profile, - params.requestPayer, - params.s3Region, - params.s3UrlFormat, - verbose, - ); - sourceType = 'pmtiles'; - const metadata = await getPMtilesInfo(source, inputFile); - Object.assign(tileJSON, metadata); - } else if (inputType === 'mbtiles') { - sourceType = 'mbtiles'; - const mbw = await openMbTilesWrapper(inputFile); - const info = await mbw.getInfo(); - source = mbw.getMbTiles(); - Object.assign(tileJSON, info); + try { + if (inputType === 'pmtiles') { + source = openPMtiles( + inputFile, + params.s3Profile, + params.requestPayer, + params.s3Region, + params.s3UrlFormat, + verbose, + ); + sourceType = 'pmtiles'; + const metadata = await getPMtilesInfo(source, inputFile); + Object.assign(tileJSON, metadata); + } else if (inputType === 'mbtiles') { + sourceType = 'mbtiles'; + const mbw = await openMbTilesWrapper(inputFile); + const info = await mbw.getInfo(); + source = mbw.getMbTiles(); + Object.assign(tileJSON, info); + } + } catch (err) { + if (ignoreMissingFiles) { + console.log( + `WARN: Unable to open data source '${id}' from "${inputFile}": ${err.message} - skipping (requests will return 404)`, + ); + return; + } + throw err; } delete tileJSON['filesize']; diff --git a/src/serve_style.js b/src/serve_style.js index 694e5a024..34711a5fe 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -213,7 +213,7 @@ export const serve_style = { reportTiles, reportFont, ) { - const { publicUrl } = programOpts; + const { publicUrl, ignoreMissingFiles } = programOpts; const styleFile = path.resolve(options.paths.styles, params.style); const styleJSON = clone(style); @@ -247,6 +247,9 @@ export const serve_style = { return false; } + // Track missing sources + const missingSources = []; + for (const name of Object.keys(styleJSON.sources)) { // eslint-disable-next-line security/detect-object-injection -- name is from Object.keys of style sources const source = styleJSON.sources[name]; @@ -270,7 +273,9 @@ export const serve_style = { const identifier = reportTiles(dataId, protocol); if (!identifier) { - return false; + // This datasource is missing or invalid in some way + missingSources.push(name); + continue; } source.url = `local://data/${identifier}.json`; } @@ -286,6 +291,20 @@ export const serve_style = { } } + // Check if any sources are missing after processing all of them + if (missingSources.length > 0) { + if (ignoreMissingFiles) { + console.log( + `WARN: Style '${id}' references ${missingSources.length} missing data source(s): [${missingSources.join(', ')}] - not adding style`, + ); + } else { + console.log( + `ERROR: Style '${id}' references missing data source(s): [${missingSources.join(', ')}]`, + ); + } + return false; + } + for (const obj of styleJSON.layers) { if (obj['type'] === 'symbol') { const fonts = (obj['layout'] || {})['text-font']; diff --git a/src/server.js b/src/server.js index 126f357a4..fac226e95 100644 --- a/src/server.js +++ b/src/server.js @@ -241,7 +241,36 @@ async function start(opts) { } } if (dataItemId) { - // input files exists in the data config, return found id + // Data source exists in config, now validate file exists + // eslint-disable-next-line security/detect-object-injection -- dataItemId is validated above + const dataSource = data[dataItemId]; + const fileType = dataSource.pmtiles ? 'pmtiles' : 'mbtiles'; + const fileName = dataSource[fileType]; + + // Skip validation for remote URLs + if (fileName && !isValidRemoteUrl(fileName)) { + const filePath = path.resolve(options.paths[fileType], fileName); + try { + const stats = fs.statSync(filePath); + if (!stats.isFile() || stats.size === 0) { + if (opts.ignoreMissingFiles) { + // File doesn't exist or is empty - return undefined to skip + return undefined; + } + // File missing but flag not set - let it fail later + return dataItemId; + } + } catch (err) { + // File doesn't exist + if (opts.ignoreMissingFiles) { + return undefined; + } + // File missing but flag not set - let it fail later + return dataItemId; + } + } + + // File exists or is remote URL, return the id return dataItemId; } else { if (!allowMoreData) { @@ -257,6 +286,21 @@ async function start(opts) { if (isValidRemoteUrl(styleSourceId)) { id = fnv1a(styleSourceId) + '_' + id.replace(/^.*\/(.*)$/, '$1'); + } else { + try { + const stats = fs.statSync(styleSourceId); + if (!stats.isFile() || stats.size === 0) { + if (opts.ignoreMissingFiles) { + // File doesn't exist or is empty - return undefined to skip + return undefined; + } + } + } catch (err) { + // File doesn't exist + if (opts.ignoreMissingFiles) { + return undefined; + } + } } // eslint-disable-next-line security/detect-object-injection -- id is being checked for existence before modification while (data[id]) id += '_'; //if the data source id already exists, add a "_" untill it doesn't