diff --git a/src/service/extract-load-service.ts b/src/service/extract-load-service.ts index c9b874e..a716848 100644 --- a/src/service/extract-load-service.ts +++ b/src/service/extract-load-service.ts @@ -91,15 +91,16 @@ export class ExtractLoadService { const promises = []; console.time(`processFiles ${tdei_dataset_id}`); for await (const entry of directory) { - if (entry.type === 'File' && entry.path.endsWith('.geojson')) { + if (entry.type === 'File' && entry.path.endsWith('.geojson') && !entry.path.includes('__MACOSX/')) { const content = await entry.buffer(); let jsonData; try { jsonData = JSON.parse(content.toString('utf8')); } catch (error) { - console.error("Unable to parse content as JSON:", content.toString('utf8')); - continue; + console.error("Unable to parse content as JSON:", entry.path, error); + throw new Error("Unable to parse content as JSON:" + entry.path + error); } + if (entry.path.includes('nodes')) { promises.push(this.bulkInsertNodes(client, tdei_dataset_id, user_id, jsonData)); } else if (entry.path.includes('edges')) { @@ -117,8 +118,6 @@ export class ExtractLoadService { promises.push(this.bulkInsertExtension(client, tdei_dataset_id, user_id, jsonData, entry)); } - } else { - entry.autodrain(); } } await Promise.all(promises);