Skip to content

Commit 7ab6e08

Browse files
ArttKanArttKan
committed
indeksoinnin onnistumisraporttilokit
1 parent 58d5a8e commit 7ab6e08

File tree

4 files changed

+222
-27
lines changed

4 files changed

+222
-27
lines changed

‎backend/src/app.ts‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ import { buildFinlexUrl, buildJudgmentUrl, listStatutesByYear, setSingleJudgment
1616
import type { JudgmentKey } from './types/judgment.js';
1717
import type { StatuteKey } from './types/statute.js';
1818
import { getRecentLogs, pushLog } from './util/logBuffer.js';
19-
import { deleteCollection, syncStatutes, syncJudgments, upsertJudgmentByUuid, upsertStatuteByUuid } from './search.js';
19+
import { printSyncSummary } from "./util/syncResults.js";
20+
import { deleteCollection, syncStatutes, syncJudgments, upsertJudgmentByUuid, upsertStatuteByUuid, SyncResult } from './search.js';
2021
import { query } from './db/db.js';
2122

2223
const app = express()
@@ -381,6 +382,8 @@ app.post('/api/rebuild-typesense', verifyAdminToken, async (req: express.Request
381382
try {
382383
res.status(200).json({ status: 'started', message: 'Typesense rebuild started in background' });
383384
setImmediate(async () => {
385+
const syncResults: SyncResult[] = [];
386+
384387
try {
385388
console.info('Typesense rebuild started');
386389
await addStatusRow({ action: 'typesense_rebuild_start', startedAt: new Date().toISOString() }, true);
@@ -406,12 +409,20 @@ app.post('/api/rebuild-typesense', verifyAdminToken, async (req: express.Request
406409
const statuteBounds = await getYearBounds('statutes');
407410
const judgmentBounds = await getYearBounds('judgments');
408411

409-
await syncStatutes('fin', statuteBounds ?? undefined);
410-
await syncStatutes('swe', statuteBounds ?? undefined);
411-
await syncJudgments('fin', judgmentBounds ?? undefined);
412-
await syncJudgments('swe', judgmentBounds ?? undefined);
412+
const statuteFinResult = await syncStatutes('fin', statuteBounds ?? undefined);
413+
syncResults.push(statuteFinResult);
414+
const statuteSweResult = await syncStatutes('swe', statuteBounds ?? undefined);
415+
syncResults.push(statuteSweResult);
416+
const judgmentFinResult = await syncJudgments('fin', judgmentBounds ?? undefined);
417+
syncResults.push(judgmentFinResult);
418+
const judgmentSweResult = await syncJudgments('swe', judgmentBounds ?? undefined);
419+
syncResults.push(judgmentSweResult);
420+
413421
console.log('Typesense collections rebuilt');
414422

423+
// Print summary
424+
printSyncSummary(syncResults);
425+
415426
await addStatusRow({ action: 'typesense_rebuild_complete', completedAt: new Date().toISOString() }, false);
416427
console.info('Typesense rebuild completed');
417428
} catch (error) {

‎backend/src/dbSetup.ts‎

Lines changed: 97 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { setPool, dbIsReady, fillDb, createTables, dbIsUpToDate, setupTestDatabase, addStatusRow, clearStatusRows } from "./db/db.js";
2-
import { stopFinlexLimiterLogging, startFinlexLimiterLogging } from "./db/load.js";
2+
import { stopFinlexLimiterLogging } from "./db/load.js";
33
import * as Sentry from '@sentry/node';
44
import './util/config.js';
55
import { exit } from 'process';
6-
import { syncStatutes, deleteCollection, syncJudgments } from "./search.js";
6+
import { syncStatutes, deleteCollection, syncJudgments, SyncResult } from "./search.js";
77
import { yearFrom, yearTo } from "./util/config.js";
88

99
setPool(process.env.PG_URI ?? '');
@@ -41,11 +41,80 @@ async function initDatabase(startYear?: number) {
4141
}
4242
}
4343

44-
export const runSetup = async (startYear?: number) => {
45-
const setupStartTime = Date.now();
44+
function printSyncSummary(results: SyncResult[]) {
45+
console.log('\n' + '='.repeat(80));
46+
console.log('TYPESENSE INDEXING SUMMARY');
47+
console.log('='.repeat(80));
48+
49+
let totalProcessed = 0;
50+
let totalSuccess = 0;
51+
let totalFailures = 0;
52+
let hasAnyFailures = false;
53+
54+
// Group by type
55+
const statuteResults = results.filter(r => r.type === 'statutes');
56+
const judgmentResults = results.filter(r => r.type === 'judgments');
57+
58+
// Print Statutes Summary
59+
console.log('\n📚 STATUTES:');
60+
console.log('-'.repeat(80));
61+
statuteResults.forEach(result => {
62+
const status = result.failureCount === 0 ? '✓' : '⚠️';
63+
const statusText = result.failureCount === 0 ? 'SUCCESS' : 'PARTIAL';
64+
console.log(` ${status} ${result.language.toUpperCase()}: ${statusText}`);
65+
console.log(` Total: ${result.totalProcessed} | Success: ${result.successCount} | Failed: ${result.failureCount}`);
66+
67+
totalProcessed += result.totalProcessed;
68+
totalSuccess += result.successCount;
69+
totalFailures += result.failureCount;
70+
71+
if (result.failureCount > 0) {
72+
hasAnyFailures = true;
73+
}
74+
});
75+
76+
// Print Judgments Summary
77+
console.log('\n⚖️ JUDGMENTS:');
78+
console.log('-'.repeat(80));
79+
judgmentResults.forEach(result => {
80+
const status = result.failureCount === 0 ? '✓' : '⚠️';
81+
const statusText = result.failureCount === 0 ? 'SUCCESS' : 'PARTIAL';
82+
console.log(` ${status} ${result.language.toUpperCase()}: ${statusText}`);
83+
console.log(` Total: ${result.totalProcessed} | Success: ${result.successCount} | Failed: ${result.failureCount}`);
84+
85+
totalProcessed += result.totalProcessed;
86+
totalSuccess += result.successCount;
87+
totalFailures += result.failureCount;
88+
89+
if (result.failureCount > 0) {
90+
hasAnyFailures = true;
91+
}
92+
});
93+
94+
// Print Overall Summary
95+
console.log('\n' + '='.repeat(80));
96+
console.log('OVERALL TOTALS:');
97+
console.log(` Total Documents Processed: ${totalProcessed}`);
98+
console.log(` Successfully Indexed: ${totalSuccess}`);
99+
console.log(` Failed to Index: ${totalFailures}`);
46100

47-
startFinlexLimiterLogging();
101+
const successRate = totalProcessed > 0 ? ((totalSuccess / totalProcessed) * 100).toFixed(2) : '0.00';
102+
console.log(` Success Rate: ${successRate}%`);
48103

104+
console.log('='.repeat(80));
105+
106+
if (hasAnyFailures) {
107+
console.log('\n⚠️ WARNING: Some documents failed to index. Review the detailed logs above.');
108+
console.log('Failed documents have been logged with full details for investigation.\n');
109+
} else {
110+
console.log('\n✓ ALL DOCUMENTS SUCCESSFULLY INDEXED!\n');
111+
}
112+
}
113+
114+
export const runSetup = async (startYear?: number) => {
115+
const setupStartTime = Date.now();
116+
const syncResults: SyncResult[] = [];
117+
49118
try {
50119
if (process.env.NODE_ENV === 'test') {
51120
console.log('[SETUP] Running in test mode...');
@@ -68,33 +137,41 @@ export const runSetup = async (startYear?: number) => {
68137

69138
const step3Start = Date.now();
70139
console.log('[SETUP] Step 3: Sync statutes to Typesense...');
71-
await syncStatutes('fin');
72-
await syncStatutes('swe');
140+
const statuteFinResult = await syncStatutes('fin');
141+
syncResults.push(statuteFinResult);
142+
const statuteSweResult = await syncStatutes('swe');
143+
syncResults.push(statuteSweResult);
73144
console.log(`[SETUP] Step 3 completed in ${Date.now() - step3Start}ms`);
74145

75146
const step4Start = Date.now();
76147
console.log('[SETUP] Step 4: Sync judgments to Typesense...');
77-
await syncJudgments('fin');
78-
await syncJudgments('swe');
148+
const judgmentFinResult = await syncJudgments('fin');
149+
syncResults.push(judgmentFinResult);
150+
const judgmentSweResult = await syncJudgments('swe');
151+
syncResults.push(judgmentSweResult);
79152
console.log(`[SETUP] Step 4 completed in ${Date.now() - step4Start}ms`);
80153

81154
const step5Start = Date.now();
82155
console.log('[SETUP] Step 5: Write completion status...');
83156
const from = startYear ?? yearFrom();
84157
const to = yearTo();
85-
await addStatusRow({ message: 'updated', from, to, timestamp: new Date().toISOString() }, false);
158+
await addStatusRow({ message: 'updated', from, to }, false);
86159
console.log(`[SETUP] Step 5 completed in ${Date.now() - step5Start}ms`);
160+
161+
// Print comprehensive summary
162+
printSyncSummary(syncResults);
163+
}
164+
165+
const totalTime = Date.now() - setupStartTime;
166+
console.log(`[SETUP] Total setup time: ${totalTime}ms (${(totalTime / 1000 / 60).toFixed(2)} minutes)`);
167+
168+
if (process.env.NODE_ENV !== 'test') {
169+
stopFinlexLimiterLogging();
170+
exit(0);
87171
}
88-
const totalDuration = Date.now() - setupStartTime;
89-
const minutes = Math.floor(totalDuration / 60000);
90-
const seconds = Math.floor((totalDuration % 60000) / 1000);
91-
console.log(`[SETUP] Database setup completed in ${minutes}m ${seconds}s (${totalDuration}ms)`);
92172
} catch (error) {
93-
const errorDuration = Date.now() - setupStartTime;
173+
console.error('[SETUP] Setup failed:', error);
94174
Sentry.captureException(error);
95-
console.error(`[SETUP] Setup failed after ${errorDuration}ms:`, error);
96-
throw error;
97-
} finally {
98-
stopFinlexLimiterLogging();
175+
exit(1);
99176
}
100-
}
177+
};

‎backend/src/search.ts‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,22 @@ const tsClient = new Typesense.Client({
3333
connectionTimeoutSeconds: 30
3434
});
3535

36+
export interface SyncResult {
37+
type: 'statutes' | 'judgments';
38+
language: string;
39+
totalProcessed: number;
40+
successCount: number;
41+
failureCount: number;
42+
failures: Array<{
43+
id: string;
44+
year: number;
45+
number: string;
46+
title?: string;
47+
level?: string;
48+
error: string;
49+
}>;
50+
}
51+
3652
function flattenHeadings(headings: Heading[]) {
3753
const out: string[] = [];
3854
function recurse(arr: Heading[]) {
@@ -203,7 +219,7 @@ async function upsertWithRetry(collectionName: string, document: Record<string,
203219
}
204220

205221

206-
export async function syncStatutes(lang: string, range?: { startYear?: number; endYear?: number }) {
222+
export async function syncStatutes(lang: string, range?: { startYear?: number; endYear?: number }): Promise<SyncResult> {
207223
const collectionName = await ensureStatuteCollection(lang);
208224
const langKey: "fin" | "swe" = lang === "fin" ? "fin" : "swe";
209225
console.log(`Indexing: ${lang} -> ${collectionName}`);
@@ -219,6 +235,8 @@ export async function syncStatutes(lang: string, range?: { startYear?: number; e
219235
error: string;
220236
}> = [];
221237

238+
let totalProcessed = 0;
239+
222240
for (let year = startYear; year <= endYear; year++) {
223241
console.log('syncStatutes ' + lang + ' ' + year)
224242

@@ -244,6 +262,7 @@ export async function syncStatutes(lang: string, range?: { startYear?: number; e
244262

245263
while (rows.length > 0) {
246264
const row = rows.pop()
265+
totalProcessed++;
247266

248267
try {
249268
const parsed_xml = await parseStringPromise(row.content, { explicitArray: false })
@@ -292,9 +311,18 @@ export async function syncStatutes(lang: string, range?: { startYear?: number; e
292311
} else {
293312
console.log(`✓ Successfully indexed all statutes for language ${lang}`);
294313
}
314+
315+
return {
316+
type: 'statutes',
317+
language: lang,
318+
totalProcessed,
319+
successCount: totalProcessed - failedStatutes.length,
320+
failureCount: failedStatutes.length,
321+
failures: failedStatutes
322+
};
295323
}
296324

297-
export async function syncJudgments(lang: string, range?: { startYear?: number; endYear?: number }) {
325+
export async function syncJudgments(lang: string, range?: { startYear?: number; endYear?: number }): Promise<SyncResult> {
298326
const collectionName = await ensureJudgmentCollection(lang);
299327
const langKey: "fin" | "swe" = lang === "fin" ? "fin" : "swe";
300328
console.log(`\n=== Indexing: ${lang} -> ${collectionName}`);
@@ -310,6 +338,8 @@ export async function syncJudgments(lang: string, range?: { startYear?: number;
310338
error: string;
311339
}> = [];
312340

341+
let totalProcessed = 0;
342+
313343
for (let year = startYear; year <= endYear; year++) {
314344
const { rows } = await query(
315345
`
@@ -332,6 +362,7 @@ export async function syncJudgments(lang: string, range?: { startYear?: number;
332362

333363
while (rows.length > 0) {
334364
const row = rows.pop()
365+
totalProcessed++;
335366

336367
try {
337368
const headingTree: Heading[] = parseHtmlHeadings(row.content) ?? [];
@@ -375,6 +406,15 @@ export async function syncJudgments(lang: string, range?: { startYear?: number;
375406
} else {
376407
console.log(`✓ Successfully indexed all judgments for language ${lang}`);
377408
}
409+
410+
return {
411+
type: 'judgments',
412+
language: lang,
413+
totalProcessed,
414+
successCount: totalProcessed - failedJudgments.length,
415+
failureCount: failedJudgments.length,
416+
failures: failedJudgments
417+
};
378418
}
379419

380420
export async function upsertStatuteByUuid(lang: string, statuteUuid: string): Promise<void> {

‎backend/src/util/syncResults.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { SyncResult } from "../search.js";
2+
3+
export function printSyncSummary(results: SyncResult[]) {
4+
console.log('\n' + '='.repeat(80));
5+
console.log('TYPESENSE INDEXING SUMMARY');
6+
console.log('='.repeat(80));
7+
8+
let totalProcessed = 0;
9+
let totalSuccess = 0;
10+
let totalFailures = 0;
11+
let hasAnyFailures = false;
12+
13+
const statuteResults = results.filter(r => r.type === 'statutes');
14+
const judgmentResults = results.filter(r => r.type === 'judgments');
15+
16+
console.log('\n📚 STATUTES:');
17+
console.log('-'.repeat(80));
18+
statuteResults.forEach(result => {
19+
const status = result.failureCount === 0 ? '✓' : '⚠️';
20+
const statusText = result.failureCount === 0 ? 'SUCCESS' : 'PARTIAL';
21+
console.log(` ${status} ${result.language.toUpperCase()}: ${statusText}`);
22+
console.log(` Total: ${result.totalProcessed} | Success: ${result.successCount} | Failed: ${result.failureCount}`);
23+
24+
totalProcessed += result.totalProcessed;
25+
totalSuccess += result.successCount;
26+
totalFailures += result.failureCount;
27+
28+
if (result.failureCount > 0) {
29+
hasAnyFailures = true;
30+
}
31+
});
32+
33+
console.log('\n⚖️ JUDGMENTS:');
34+
console.log('-'.repeat(80));
35+
judgmentResults.forEach(result => {
36+
const status = result.failureCount === 0 ? '✓' : '⚠️';
37+
const statusText = result.failureCount === 0 ? 'SUCCESS' : 'PARTIAL';
38+
console.log(` ${status} ${result.language.toUpperCase()}: ${statusText}`);
39+
console.log(` Total: ${result.totalProcessed} | Success: ${result.successCount} | Failed: ${result.failureCount}`);
40+
41+
totalProcessed += result.totalProcessed;
42+
totalSuccess += result.successCount;
43+
totalFailures += result.failureCount;
44+
45+
if (result.failureCount > 0) {
46+
hasAnyFailures = true;
47+
}
48+
});
49+
50+
console.log('\n' + '='.repeat(80));
51+
console.log('OVERALL TOTALS:');
52+
console.log(` Total Documents Processed: ${totalProcessed}`);
53+
console.log(` Successfully Indexed: ${totalSuccess}`);
54+
console.log(` Failed to Index: ${totalFailures}`);
55+
56+
const successRate = totalProcessed > 0 ? ((totalSuccess / totalProcessed) * 100).toFixed(2) : '0.00';
57+
console.log(` Success Rate: ${successRate}%`);
58+
59+
console.log('='.repeat(80));
60+
61+
if (hasAnyFailures) {
62+
console.log('WARNING: Some documents failed to index. Review the detailed logs above.');
63+
console.log('Failed documents have been logged with full details for investigation.\n');
64+
} else {
65+
console.log('\n✓ ALL DOCUMENTS SUCCESSFULLY INDEXED!\n');
66+
}
67+
}

0 commit comments

Comments
 (0)