Skip to content

Commit 14cc9e9

Browse files
sefineksefinek
committed
Normalize dataset stage logging
1 parent 6c2707e commit 14cc9e9

File tree

1 file changed

+135
-133
lines changed

1 file changed

+135
-133
lines changed

‎tools/updatedb.js‎

Lines changed: 135 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -34,17 +34,38 @@ const { Address6, Address4 } = require('ip-address');
3434
const numberFormatter = new Intl.NumberFormat('en-US');
3535

3636
const log = {
37-
info: (msg, ...logArgs) => console.log('[INFO]', msg, ...logArgs),
38-
success: (msg, ...logArgs) => console.log('[SUCCESS]', msg, ...logArgs),
39-
warn: (msg, ...logArgs) => console.warn('[WARN]', msg, ...logArgs),
40-
error: (msg, ...logArgs) => console.error('[ERROR]', msg, ...logArgs),
41-
progress: (msg) => process.stdout.write(`[INFO] ${msg}... `),
42-
done: () => console.log('Done'),
43-
progressCount: (msg, count, unit = 'entries processed') => {
44-
console.log('[INFO]', `${msg} (${numberFormatter.format(count)} ${unit})`);
45-
},
37+
info: (msg, ...logArgs) => console.log('[INFO]', msg, ...logArgs),
38+
success: (msg, ...logArgs) => console.log('[SUCCESS]', msg, ...logArgs),
39+
warn: (msg, ...logArgs) => console.warn('[WARN]', msg, ...logArgs),
40+
error: (msg, ...logArgs) => console.error('[ERROR]', msg, ...logArgs),
41+
progress: (msg) => process.stdout.write(`[INFO] ${msg}... `),
42+
done: () => console.log('Done'),
43+
progressCount: (msg, count, unit = 'entries processed') => {
44+
console.log('[INFO]', `${msg} (${numberFormatter.format(count)} ${unit})`);
45+
},
4646
};
4747

48+
function startProcessingStage(label, unit = 'entries', intervalMs = 5000) {
49+
log.info(`Processing ${label} (this may take a moment)...`);
50+
let lastLog = Date.now();
51+
return {
52+
progress(count) {
53+
const now = Date.now();
54+
if (now - lastLog >= intervalMs) {
55+
lastLog = now;
56+
log.progressCount(`Processing ${label}`, count, `${unit} processed`);
57+
}
58+
},
59+
complete(total) {
60+
if (typeof total === 'number') {
61+
log.info(`${label} processed (${numberFormatter.format(total)} ${unit})`);
62+
} else {
63+
log.info(`${label} processed`);
64+
}
65+
},
66+
};
67+
}
68+
4869
// ============================================================================
4970
// Configuration
5071
// ============================================================================
@@ -345,17 +366,15 @@ async function processCountryData(src, dest, label = 'Country data') {
345366

346367
rimraf(dataFile);
347368
mkdir(dataFile);
348-
349-
process.stdout.write('\n');
350-
log.progress(`Processing ${label} (this may take a moment)`);
351-
let tstart = Date.now();
369+
const stage = startProcessingStage(label);
352370
const datFile = fs.createWriteStream(dataFile);
353371

354-
function processLine(line) {
355-
const fields = CSVtoArray(line);
356-
if (!fields || fields.length < 6) return log.warn('Malformed line detected:', line);
372+
function processLine(line) {
373+
const fields = CSVtoArray(line);
374+
if (!fields || fields.length < 6) return log.warn('Malformed line detected:', line);
357375

358376
lines++;
377+
stage.progress(lines);
359378

360379
let sip;
361380
let eip;
@@ -394,12 +413,7 @@ async function processCountryData(src, dest, label = 'Country data') {
394413
b.writeUInt32BE(eip, 4);
395414
}
396415

397-
b.write(cc, bsz - 2);
398-
if (Date.now() - tstart > 5000) {
399-
tstart = Date.now();
400-
process.stdout.write('\n');
401-
log.progressCount(`Processing ${label}`, lines);
402-
}
416+
b.write(cc, bsz - 2);
403417

404418
if (datFile._writableState.needDrain) {
405419
return new Promise(resolve => {
@@ -411,54 +425,53 @@ async function processCountryData(src, dest, label = 'Country data') {
411425
}
412426
}
413427

414-
await new Promise((resolve, reject) => {
415-
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
416-
let settled = false;
417-
let i = 0;
418-
419-
function finish(err) {
420-
if (settled) return;
421-
settled = true;
422-
if (!rl.closed) rl.close();
423-
if (err) reject(err);
424-
else resolve();
425-
}
426-
427-
function resume() {
428-
if (!settled && !rl.closed) rl.resume();
429-
}
430-
431-
rl.on('line', line => {
432-
rl.pause();
433-
i++;
434-
if (i === 1) {
435-
resume();
436-
return;
437-
}
438-
439-
let result;
440-
try {
441-
result = processLine(line);
442-
} catch (err) {
443-
finish(err);
444-
return;
445-
}
446-
447-
if (result && typeof result.then === 'function') {
448-
result.then(() => {
449-
resume();
450-
}).catch(finish);
451-
} else {
452-
resume();
453-
}
454-
});
455-
456-
rl.on('close', () => finish());
457-
rl.on('error', finish);
458-
});
459-
datFile.close();
460-
log.done();
461-
log.info(`${label} processed`);
428+
await new Promise((resolve, reject) => {
429+
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
430+
let settled = false;
431+
let i = 0;
432+
433+
function finish(err) {
434+
if (settled) return;
435+
settled = true;
436+
if (!rl.closed) rl.close();
437+
if (err) reject(err);
438+
else resolve();
439+
}
440+
441+
function resume() {
442+
if (!settled && !rl.closed) rl.resume();
443+
}
444+
445+
rl.on('line', line => {
446+
rl.pause();
447+
i++;
448+
if (i === 1) {
449+
resume();
450+
return;
451+
}
452+
453+
let result;
454+
try {
455+
result = processLine(line);
456+
} catch (err) {
457+
finish(err);
458+
return;
459+
}
460+
461+
if (result && typeof result.then === 'function') {
462+
result.then(() => {
463+
resume();
464+
}).catch(finish);
465+
} else {
466+
resume();
467+
}
468+
});
469+
470+
rl.on('close', () => finish());
471+
rl.on('error', finish);
472+
});
473+
datFile.close();
474+
stage.complete(lines);
462475
}
463476

464477
async function processCityData(src, dest, label = 'City data') {
@@ -467,10 +480,7 @@ async function processCityData(src, dest, label = 'City data') {
467480
const tmpDataFile = path.join(tmpPath, src);
468481

469482
rimraf(dataFile);
470-
471-
process.stdout.write('\n');
472-
log.progress(`Processing ${label} (this may take a moment)`);
473-
let tstart = Date.now();
483+
const stage = startProcessingStage(label);
474484
const datFile = fs.createWriteStream(dataFile);
475485

476486
async function processLine(line) {
@@ -491,6 +501,7 @@ async function processCityData(src, dest, label = 'City data') {
491501
let i;
492502

493503
lines++;
504+
stage.progress(lines);
494505

495506
if (fields[0].match(/:/)) {
496507
// IPv6
@@ -545,12 +556,6 @@ async function processCityData(src, dest, label = 'City data') {
545556
b.writeInt32BE(area, 20);
546557
}
547558

548-
if (Date.now() - tstart > 5000) {
549-
tstart = Date.now();
550-
process.stdout.write('\n');
551-
log.progressCount(`Processing ${label}`, lines);
552-
}
553-
554559
if (datFile._writableState.needDrain) {
555560
return new Promise((resolve) => {
556561
datFile.write(b, resolve);
@@ -560,54 +565,53 @@ async function processCityData(src, dest, label = 'City data') {
560565
}
561566
}
562567

563-
await new Promise((resolve, reject) => {
564-
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
565-
let settled = false;
566-
let i = 0;
567-
568-
function finish(err) {
569-
if (settled) return;
570-
settled = true;
571-
if (!rl.closed) rl.close();
572-
if (err) reject(err);
573-
else resolve();
574-
}
575-
576-
function resume() {
577-
if (!settled && !rl.closed) rl.resume();
578-
}
579-
580-
rl.on('line', line => {
581-
rl.pause();
582-
i++;
583-
if (i === 1) {
584-
resume();
585-
return;
586-
}
587-
588-
let result;
589-
try {
590-
result = processLine(line);
591-
} catch (err) {
592-
finish(err);
593-
return;
594-
}
595-
596-
if (result && typeof result.then === 'function') {
597-
result.then(() => {
598-
resume();
599-
}).catch(finish);
600-
} else {
601-
resume();
602-
}
603-
});
604-
605-
rl.on('close', () => finish());
606-
rl.on('error', finish);
607-
});
608-
datFile.close();
609-
log.done();
610-
log.info(`${label} processed`);
568+
await new Promise((resolve, reject) => {
569+
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
570+
let settled = false;
571+
let i = 0;
572+
573+
function finish(err) {
574+
if (settled) return;
575+
settled = true;
576+
if (!rl.closed) rl.close();
577+
if (err) reject(err);
578+
else resolve();
579+
}
580+
581+
function resume() {
582+
if (!settled && !rl.closed) rl.resume();
583+
}
584+
585+
rl.on('line', line => {
586+
rl.pause();
587+
i++;
588+
if (i === 1) {
589+
resume();
590+
return;
591+
}
592+
593+
let result;
594+
try {
595+
result = processLine(line);
596+
} catch (err) {
597+
finish(err);
598+
return;
599+
}
600+
601+
if (result && typeof result.then === 'function') {
602+
result.then(() => {
603+
resume();
604+
}).catch(finish);
605+
} else {
606+
resume();
607+
}
608+
});
609+
610+
rl.on('close', () => finish());
611+
rl.on('error', finish);
612+
});
613+
datFile.close();
614+
stage.complete(lines);
611615
}
612616

613617
function processCityDataNames(src, dest, cb) {
@@ -619,9 +623,7 @@ function processCityDataNames(src, dest, cb) {
619623
rimraf(dataFile);
620624

621625
const datFile = fs.openSync(dataFile, 'w');
622-
623-
process.stdout.write('\n');
624-
log.progress('Processing city names (this may take a moment)');
626+
const stage = startProcessingStage('City names', 'records');
625627

626628
function processLine(line) {
627629
if (line.match(/^Copyright/) || !line.match(/\d/)) return;
@@ -656,6 +658,7 @@ function processCityDataNames(src, dest, cb) {
656658

657659
fs.writeSync(datFile, b, 0, b.length, null);
658660
linesCount++;
661+
stage.progress(linesCount);
659662
}
660663

661664
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile).pipe(decodeStream('utf-8')), output: process.stdout, terminal: false });
@@ -668,8 +671,7 @@ function processCityDataNames(src, dest, cb) {
668671
});
669672

670673
rl.on('close', () => {
671-
log.done();
672-
log.info('City names processed');
674+
stage.complete(linesCount);
673675
cb();
674676
});
675677
}

0 commit comments

Comments
 (0)