Skip to content

Commit 6c2707e

Browse files
sefineksefinek
committed
Clarify dataset progress logging
1 parent 847a23a commit 6c2707e

File tree

1 file changed

+139
-50
lines changed

1 file changed

+139
-50
lines changed

‎tools/updatedb.js‎

Lines changed: 139 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,18 @@ const { Address6, Address4 } = require('ip-address');
3131
// ============================================================================
3232

3333
// Logging utility for consistent and readable output
34+
const numberFormatter = new Intl.NumberFormat('en-US');
35+
3436
const log = {
35-
info: (msg, ...logArgs) => console.log('[INFO]', msg, ...logArgs),
36-
success: (msg, ...logArgs) => console.log('[SUCCESS]', msg, ...logArgs),
37-
warn: (msg, ...logArgs) => console.warn('[WARN]', msg, ...logArgs),
38-
error: (msg, ...logArgs) => console.error('[ERROR]', msg, ...logArgs),
39-
progress: (msg) => process.stdout.write(`[INFO] ${msg}... `),
40-
done: () => console.log('Done'),
37+
info: (msg, ...logArgs) => console.log('[INFO]', msg, ...logArgs),
38+
success: (msg, ...logArgs) => console.log('[SUCCESS]', msg, ...logArgs),
39+
warn: (msg, ...logArgs) => console.warn('[WARN]', msg, ...logArgs),
40+
error: (msg, ...logArgs) => console.error('[ERROR]', msg, ...logArgs),
41+
progress: (msg) => process.stdout.write(`[INFO] ${msg}... `),
42+
done: () => console.log('Done'),
43+
progressCount: (msg, count, unit = 'entries processed') => {
44+
console.log('[INFO]', `${msg} (${numberFormatter.format(count)} ${unit})`);
45+
},
4146
};
4247

4348
// ============================================================================
@@ -333,7 +338,7 @@ function processLookupCountry(src, cb) {
333338
// Data Processing Functions
334339
// ============================================================================
335340

336-
async function processCountryData(src, dest) {
341+
async function processCountryData(src, dest, label = 'Country data') {
337342
let lines = 0;
338343
const dataFile = path.join(dataPath, dest);
339344
const tmpDataFile = path.join(tmpPath, src);
@@ -342,13 +347,13 @@ async function processCountryData(src, dest) {
342347
mkdir(dataFile);
343348

344349
process.stdout.write('\n');
345-
log.progress('Processing country data (this may take a moment)');
350+
log.progress(`Processing ${label} (this may take a moment)`);
346351
let tstart = Date.now();
347352
const datFile = fs.createWriteStream(dataFile);
348353

349-
function processLine(line) {
350-
const fields = CSVtoArray(line);
351-
if (!fields || fields.length < 6) return log.warn('Malformed line detected:', line);
354+
function processLine(line) {
355+
const fields = CSVtoArray(line);
356+
if (!fields || fields.length < 6) return log.warn('Malformed line detected:', line);
352357

353358
lines++;
354359

@@ -389,11 +394,12 @@ async function processCountryData(src, dest) {
389394
b.writeUInt32BE(eip, 4);
390395
}
391396

392-
b.write(cc, bsz - 2);
393-
if (Date.now() - tstart > 5000) {
394-
tstart = Date.now();
395-
process.stdout.write(`\nStill working (${lines})...`);
396-
}
397+
b.write(cc, bsz - 2);
398+
if (Date.now() - tstart > 5000) {
399+
tstart = Date.now();
400+
process.stdout.write('\n');
401+
log.progressCount(`Processing ${label}`, lines);
402+
}
397403

398404
if (datFile._writableState.needDrain) {
399405
return new Promise(resolve => {
@@ -405,26 +411,65 @@ async function processCountryData(src, dest) {
405411
}
406412
}
407413

408-
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
409-
let i = 0;
410-
for await (const line of rl) {
411-
i++;
412-
if (i === 1) continue;
413-
await processLine(line);
414-
}
415-
datFile.close();
416-
log.done();
414+
await new Promise((resolve, reject) => {
415+
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
416+
let settled = false;
417+
let i = 0;
418+
419+
function finish(err) {
420+
if (settled) return;
421+
settled = true;
422+
if (!rl.closed) rl.close();
423+
if (err) reject(err);
424+
else resolve();
425+
}
426+
427+
function resume() {
428+
if (!settled && !rl.closed) rl.resume();
429+
}
430+
431+
rl.on('line', line => {
432+
rl.pause();
433+
i++;
434+
if (i === 1) {
435+
resume();
436+
return;
437+
}
438+
439+
let result;
440+
try {
441+
result = processLine(line);
442+
} catch (err) {
443+
finish(err);
444+
return;
445+
}
446+
447+
if (result && typeof result.then === 'function') {
448+
result.then(() => {
449+
resume();
450+
}).catch(finish);
451+
} else {
452+
resume();
453+
}
454+
});
455+
456+
rl.on('close', () => finish());
457+
rl.on('error', finish);
458+
});
459+
datFile.close();
460+
log.done();
461+
log.info(`${label} processed`);
417462
}
418463

419-
async function processCityData(src, dest) {
464+
async function processCityData(src, dest, label = 'City data') {
420465
let lines = 0;
421466
const dataFile = path.join(dataPath, dest);
422467
const tmpDataFile = path.join(tmpPath, src);
423468

424469
rimraf(dataFile);
425470

426471
process.stdout.write('\n');
427-
log.progress('Processing city data (this may take a moment)');
472+
log.progress(`Processing ${label} (this may take a moment)`);
428473
let tstart = Date.now();
429474
const datFile = fs.createWriteStream(dataFile);
430475

@@ -500,10 +545,11 @@ async function processCityData(src, dest) {
500545
b.writeInt32BE(area, 20);
501546
}
502547

503-
if (Date.now() - tstart > 5000) {
504-
tstart = Date.now();
505-
process.stdout.write('\n[INFO] Processing... (' + lines + ' entries) ');
506-
}
548+
if (Date.now() - tstart > 5000) {
549+
tstart = Date.now();
550+
process.stdout.write('\n');
551+
log.progressCount(`Processing ${label}`, lines);
552+
}
507553

508554
if (datFile._writableState.needDrain) {
509555
return new Promise((resolve) => {
@@ -514,14 +560,54 @@ async function processCityData(src, dest) {
514560
}
515561
}
516562

517-
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
518-
let i = 0;
519-
for await (const line of rl) {
520-
i++;
521-
if (i === 1) continue;
522-
await processLine(line);
523-
}
524-
datFile.close();
563+
await new Promise((resolve, reject) => {
564+
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
565+
let settled = false;
566+
let i = 0;
567+
568+
function finish(err) {
569+
if (settled) return;
570+
settled = true;
571+
if (!rl.closed) rl.close();
572+
if (err) reject(err);
573+
else resolve();
574+
}
575+
576+
function resume() {
577+
if (!settled && !rl.closed) rl.resume();
578+
}
579+
580+
rl.on('line', line => {
581+
rl.pause();
582+
i++;
583+
if (i === 1) {
584+
resume();
585+
return;
586+
}
587+
588+
let result;
589+
try {
590+
result = processLine(line);
591+
} catch (err) {
592+
finish(err);
593+
return;
594+
}
595+
596+
if (result && typeof result.then === 'function') {
597+
result.then(() => {
598+
resume();
599+
}).catch(finish);
600+
} else {
601+
resume();
602+
}
603+
});
604+
605+
rl.on('close', () => finish());
606+
rl.on('error', finish);
607+
});
608+
datFile.close();
609+
log.done();
610+
log.info(`${label} processed`);
525611
}
526612

527613
function processCityDataNames(src, dest, cb) {
@@ -534,6 +620,9 @@ function processCityDataNames(src, dest, cb) {
534620

535621
const datFile = fs.openSync(dataFile, 'w');
536622

623+
process.stdout.write('\n');
624+
log.progress('Processing city names (this may take a moment)');
625+
537626
function processLine(line) {
538627
if (line.match(/^Copyright/) || !line.match(/\d/)) return;
539628

@@ -578,7 +667,11 @@ function processCityDataNames(src, dest, cb) {
578667
lineCount++;
579668
});
580669

581-
rl.on('close', cb);
670+
rl.on('close', () => {
671+
log.done();
672+
log.info('City names processed');
673+
cb();
674+
});
582675
}
583676

584677
// ============================================================================
@@ -595,26 +688,22 @@ function processData(database, cb) {
595688
if (type === 'country') {
596689
if (Array.isArray(src)) {
597690
processLookupCountry(src[0], () => {
598-
processCountryData(src[1], dest[1]).then(() => {
599-
return processCountryData(src[2], dest[2]);
691+
processCountryData(src[1], dest[1], 'Country IPv4 data').then(() => {
692+
return processCountryData(src[2], dest[2], 'Country IPv6 data');
600693
}).then(() => {
601694
cb(null, database);
602695
});
603696
});
604-
}
605-
else {
606-
processCountryData(src, dest, () => {
697+
} else {
698+
processCountryData(src, dest, 'Country data').then(() => {
607699
cb(null, database);
608700
});
609701
}
610702
} else if (type === 'city') {
611703
processCityDataNames(src[0], dest[0], () => {
612-
processCityData(src[1], dest[1]).then(() => {
613-
process.stdout.write('\n');
614-
log.info('City IPv4 data processed');
615-
return processCityData(src[2], dest[2]);
704+
processCityData(src[1], dest[1], 'City IPv4 data').then(() => {
705+
return processCityData(src[2], dest[2], 'City IPv6 data');
616706
}).then(() => {
617-
log.info('City IPv6 data processed');
618707
cb(null, database);
619708
});
620709
});

0 commit comments

Comments
 (0)