feat: Add ignore-missing-files cli option to avoid crashing at startup

src/main.js

@@ -109,6 +109,10 @@ program
     '-f|--log_format <format>',
     'define the log format:  https://github.com/expressjs/morgan#morganformat-options',
   )
+  .option(
+    '--ignore-missing-files',
+    'Continue startup even if configured mbtiles/pmtiles files are missing',
+  )
   .version(packageJson.version, '-v, --version');
 program.parse(process.argv);
 const opts = program.opts();
@@ -132,6 +136,7 @@ const startServer = (configPath, config) => {
     logFormat: opts.log_format,
     fetchTimeout: opts.fetchTimeout,
     publicUrl,
+    ignoreMissingFiles: opts.ignoreMissingFiles,
   });
 };
 

src/serve_data.js

@@ -508,7 +508,7 @@ export const serve_data = {
    * @returns {Promise<void>}
    */
   add: async function (options, repo, params, id, programOpts) {
-    const { publicUrl, verbose } = programOpts;
+    const { publicUrl, verbose, ignoreMissingFiles } = programOpts;
     let inputFile;
     let inputType;
     if (params.pmtiles) {
@@ -542,8 +542,18 @@ export const serve_data = {
 
     // Only check file stats for local files, not remote URLs
     if (!isValidRemoteUrl(inputFile)) {
-      const inputFileStats = await fsp.stat(inputFile);
-      if (!inputFileStats.isFile() || inputFileStats.size === 0) {
+      try {
+        const inputFileStats = await fsp.stat(inputFile);
+        if (!inputFileStats.isFile() || inputFileStats.size === 0) {
+          throw Error(`Not valid input file: "${inputFile}"`);
+        }
+      } catch (err) {
+        if (ignoreMissingFiles) {
+          console.log(
+            `WARN: Data source '${id}' file not found: "${inputFile}" - skipping`,
+          );
+          return;
+        }
         throw Error(`Not valid input file: "${inputFile}"`);
       }
     }
@@ -555,24 +565,34 @@ export const serve_data = {
     tileJSON['encoding'] = params['encoding'];
     tileJSON['tileSize'] = params['tileSize'];
 
-    if (inputType === 'pmtiles') {
-      source = openPMtiles(
-        inputFile,
-        params.s3Profile,
-        params.requestPayer,
-        params.s3Region,
-        params.s3UrlFormat,
-        verbose,
-      );
-      sourceType = 'pmtiles';
-      const metadata = await getPMtilesInfo(source, inputFile);
-      Object.assign(tileJSON, metadata);
-    } else if (inputType === 'mbtiles') {
-      sourceType = 'mbtiles';
-      const mbw = await openMbTilesWrapper(inputFile);
-      const info = await mbw.getInfo();
-      source = mbw.getMbTiles();
-      Object.assign(tileJSON, info);
+    try {
+      if (inputType === 'pmtiles') {
+        source = openPMtiles(
+          inputFile,
+          params.s3Profile,
+          params.requestPayer,
+          params.s3Region,
+          params.s3UrlFormat,
+          verbose,
+        );
+        sourceType = 'pmtiles';
+        const metadata = await getPMtilesInfo(source, inputFile);
+        Object.assign(tileJSON, metadata);
+      } else if (inputType === 'mbtiles') {
+        sourceType = 'mbtiles';
+        const mbw = await openMbTilesWrapper(inputFile);
+        const info = await mbw.getInfo();
+        source = mbw.getMbTiles();
+        Object.assign(tileJSON, info);
+      }
+    } catch (err) {
+      if (ignoreMissingFiles) {
+        console.log(
+          `WARN: Unable to open data source '${id}' from "${inputFile}": ${err.message} - skipping (requests will return 404)`,
+        );
+        return;
+      }
+      throw err;
     }
 
     delete tileJSON['filesize'];

src/serve_style.js

@@ -213,7 +213,7 @@ export const serve_style = {
     reportTiles,
     reportFont,
   ) {
-    const { publicUrl } = programOpts;
+    const { publicUrl, ignoreMissingFiles } = programOpts;
     const styleFile = path.resolve(options.paths.styles, params.style);
     const styleJSON = clone(style);
 
@@ -247,6 +247,9 @@ export const serve_style = {
       return false;
     }
 
+    // Track missing sources
+    const missingSources = [];
+
     for (const name of Object.keys(styleJSON.sources)) {
       // eslint-disable-next-line security/detect-object-injection -- name is from Object.keys of style sources
       const source = styleJSON.sources[name];
@@ -270,7 +273,9 @@ export const serve_style = {
 
         const identifier = reportTiles(dataId, protocol);
         if (!identifier) {
-          return false;
+          // This datasource is missing or invalid in some way
+          missingSources.push(name);
+          continue;
         }
         source.url = `local://data/${identifier}.json`;
       }
@@ -286,6 +291,20 @@ export const serve_style = {
       }
     }
 
+    // Check if any sources are missing after processing all of them
+    if (missingSources.length > 0) {
+      if (ignoreMissingFiles) {
+        console.log(
+          `WARN: Style '${id}' references ${missingSources.length} missing data source(s): [${missingSources.join(', ')}] - not adding style`,
+        );
+      } else {
+        console.log(
+          `ERROR: Style '${id}' references missing data source(s): [${missingSources.join(', ')}]`,
+        );
+      }
+      return false;
+    }
+
     for (const obj of styleJSON.layers) {
       if (obj['type'] === 'symbol') {
         const fonts = (obj['layout'] || {})['text-font'];

src/server.js

@@ -241,7 +241,36 @@ async function start(opts) {
             }
           }
           if (dataItemId) {
-            // input files exists in the data config, return found id
+            // Data source exists in config, now validate file exists
+            // eslint-disable-next-line security/detect-object-injection -- dataItemId is validated above
+            const dataSource = data[dataItemId];
+            const fileType = dataSource.pmtiles ? 'pmtiles' : 'mbtiles';
+            const fileName = dataSource[fileType];
+
+            // Skip validation for remote URLs
+            if (fileName && !isValidRemoteUrl(fileName)) {
+              const filePath = path.resolve(options.paths[fileType], fileName);
+              try {
+                const stats = fs.statSync(filePath);
+                if (!stats.isFile() || stats.size === 0) {
+                  if (opts.ignoreMissingFiles) {
+                    // File doesn't exist or is empty - return undefined to skip
+                    return undefined;
+                  }
+                  // File missing but flag not set - let it fail later
+                  return dataItemId;
+                }
+              } catch (err) {
+                // File doesn't exist
+                if (opts.ignoreMissingFiles) {
+                  return undefined;
+                }
+                // File missing but flag not set - let it fail later
+                return dataItemId;
+              }
+            }
+
+            // File exists or is remote URL, return the id
             return dataItemId;
           } else {
             if (!allowMoreData) {
@@ -257,6 +286,21 @@ async function start(opts) {
               if (isValidRemoteUrl(styleSourceId)) {
                 id =
                   fnv1a(styleSourceId) + '_' + id.replace(/^.*\/(.*)$/, '$1');
+              } else {
+                try {
+                  const stats = fs.statSync(styleSourceId);
+                  if (!stats.isFile() || stats.size === 0) {
+                    if (opts.ignoreMissingFiles) {
+                      // File doesn't exist or is empty - return undefined to skip
+                      return undefined;
+                    }
+                  }
+                } catch (err) {
+                  // File doesn't exist
+                  if (opts.ignoreMissingFiles) {
+                    return undefined;
+                  }
+                }
               }
               // eslint-disable-next-line security/detect-object-injection -- id is being checked for existence before modification
               while (data[id]) id += '_'; //if the data source id already exists, add a "_" untill it doesn't