kumottujenpoisto-endpoint

Author: ArttKan

backend/src/app.ts

@@ -7,12 +7,13 @@ import statuteRouter from './controllers/statute.js';
 import judgmentRouter from './controllers/judgment.js';
 import keywordRouter from './controllers/keyword.js';
 import judgmentKeywordRouter from './controllers/judgmentKeyword.js';
+import axios from 'axios';
 import { fileURLToPath } from 'url';
 import { runSetup } from './dbSetup.js';
 import { getLatestStatusEntry, getAllStatusEntries, clearAllStatusEntries } from './db/models/status.js';
 import { addStatusRow, createTables, dropTables, dropJudgmentsTables, createJudgmentsTables, deleteStatutesByYear, normalizeStatuteTitlesByYear } from './db/db.js';
 import { VALID_LANGUAGES, yearFrom, yearTo } from './util/config.js';
-import { buildFinlexUrl, buildJudgmentUrl, listStatutesByYear, setSingleJudgment, setSingleStatute } from './db/load.js';
+import { buildFinlexUrl, buildJudgmentUrl, listStatutesByYear, setSingleJudgment, setSingleStatute, parseIsInForceFromXml, finlexLimiter, fetchWithBackoff, startFinlexLimiterLogging, stopFinlexLimiterLogging } from './db/load.js';
 import type { JudgmentKey } from './types/judgment.js';
 import type { StatuteKey } from './types/statute.js';
 import { getRecentLogs, pushLog } from './util/logBuffer.js';
@@ -494,6 +495,147 @@ app.get('/api/admin/check-title-issues', verifyAdminToken, async (req: express.R
   }
 });
 
+app.post('/api/admin/purge-not-in-force-statutes', verifyAdminToken, async (req: express.Request, res: express.Response): Promise<void> => {
+  const dryRun = String(req.query?.dryRun ?? 'false') === 'true';
+  const startYear = req.query.startYear ? parseInt(String(req.query.startYear), 10) : null;
+  const endYear = req.query.endYear ? parseInt(String(req.query.endYear), 10) : null;
+
+  console.log(`[maintenance] purge-not-in-force-statutes started dryRun=${dryRun} startYear=${startYear ?? '-'} endYear=${endYear ?? '-'}`);
+  startFinlexLimiterLogging();
+
+  try {
+    const { rows } = await query(
+      `SELECT uuid, year, number, language, version
+       FROM statutes
+       ORDER BY year ASC, number ASC, language ASC`
+    );
+
+    let checkedTotal = 0;
+    let deletedTotal = 0;
+    let fetchFailed = 0;
+
+    let currentYear: number | null = null;
+    let checkedThisYear = 0;
+    let deletedThisYear = 0;
+
+    const headers = { Accept: 'application/xml', 'Accept-Encoding': 'gzip' };
+
+    for (const row of rows) {
+      const year = Number(row.year);
+
+      if (startYear !== null && year < startYear) continue;
+      if (endYear !== null && year > endYear) continue;
+
+      if (currentYear === null) {
+        currentYear = year;
+        checkedThisYear = 0;
+        deletedThisYear = 0;
+        console.log(`[maintenance] Year ${currentYear}: starting`);
+      } else if (year !== currentYear) {
+        console.log(`[maintenance] Year ${currentYear}: done checked=${checkedThisYear} deleted=${deletedThisYear}`);
+        currentYear = year;
+        checkedThisYear = 0;
+        deletedThisYear = 0;
+        console.log(`[maintenance] Year ${currentYear}: starting`);
+      }
+
+      checkedTotal += 1;
+      checkedThisYear += 1;
+
+const baseUrl = 'https://opendata.finlex.fi/finlex/avoindata/v1/akn/fi/act';
+
+// Build candidate URLs (try in this exact order)
+const candidates: string[] = [
+  // 1) canonical / work-level endpoint (often best)
+  `${baseUrl}/statute-consolidated/${row.year}/${row.number}/`,
+
+  // 2) expression-level, language but no version
+  `${baseUrl}/statute-consolidated/${row.year}/${row.number}/${row.language}@`,
+];
+
+if (row.version) {
+  // 3) expression-level, language + version
+  candidates.push(
+    `${baseUrl}/statute-consolidated/${row.year}/${row.number}/${row.language}@${row.version}`
+  );
+}
+
+let xml: string | null = null;
+let attemptedUris: string[] = [];
+
+for (const u of candidates) {
+  attemptedUris.push(u);
+  try {
+    xml = (await fetchWithBackoff<string>(u, { headers })).data;
+    break; // success
+  } catch {
+    // try next candidate
+  }
+}
+
+if (!xml) {
+  fetchFailed += 1;
+  if (fetchFailed <= 20 || fetchFailed % 200 === 0) {
+    console.warn(
+      `[maintenance] Finlex fetch failed for ${row.year}/${row.number}/${row.language}@${row.version ?? ''} ` +
+      `tried=${attemptedUris.join(' | ')}`
+    );
+  }
+  continue;
+}
+
+      const isInForce = await parseIsInForceFromXml(xml);
+
+      if (isInForce === false) {
+        if (!dryRun) {
+          await query('DELETE FROM statutes WHERE uuid = $1', [row.uuid]);
+        }
+
+        deletedTotal += 1;
+        deletedThisYear += 1;
+
+        console.log(
+          `[maintenance] deleted (isInForce=false): ` +
+          `${row.year}/${row.number}/${row.language}@${row.version ?? ''} ` +
+          `uuid=${row.uuid} deletedThisYear=${deletedThisYear} deletedTotal=${deletedTotal}`
+        );
+      }
+
+      if (checkedTotal % 250 === 0) {
+        console.log(
+          `[maintenance] progress year=${currentYear} checkedThisYear=${checkedThisYear} deletedThisYear=${deletedThisYear} ` +
+          `checkedTotal=${checkedTotal} deletedTotal=${deletedTotal} fetchFailed=${fetchFailed}`
+        );
+      }
+    }
+
+    if (currentYear !== null) {
+      console.log(`[maintenance] Year ${currentYear}: done checked=${checkedThisYear} deleted=${deletedThisYear}`);
+    }
+
+    console.log(
+      `[maintenance] purge-not-in-force-statutes finished dryRun=${dryRun} ` +
+      `checkedTotal=${checkedTotal} deletedTotal=${deletedTotal} fetchFailed=${fetchFailed}`
+    );
+
+    res.status(200).json({
+      ok: true,
+      dryRun,
+      checkedTotal,
+      deletedTotal,
+      fetchFailed,
+      startYear,
+      endYear,
+    });
+  } catch (error) {
+    console.error('[maintenance] purge-not-in-force-statutes error:', error);
+    Sentry.captureException(error);
+    res.status(500).json({ ok: false, error: 'Failed to purge statutes' });
+  } finally {
+    stopFinlexLimiterLogging();
+  }
+});
+
 app.get('/favicon.ico', (request: express.Request, response: express.Response): void => {
   response.status(204).end();
 })

backend/src/db/load.ts

@@ -913,4 +913,61 @@ async function listJudgmentsByYear(year: number, language: string, level: string
   return Array.from(judgmentURLsSet);
 }
 
-export { listStatutesByYear, setSingleStatute, listJudgmentNumbersByYear, listJudgmentsByYear, parseURLfromJudgmentID, setSingleJudgment, parseAkomafromURL, parseFinlexUrl, parseJudgmentUrl, buildFinlexUrl, buildJudgmentUrl }
+export async function parseIsInForceFromXml(xmlString: string): Promise<boolean | null> {
+  try {
+    const parser = new XMLParser({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      removeNSPrefix: true, // finlex:isInForce => isInForce
+    });
+
+    const parsed = parser.parse(xmlString);
+
+    const coerce = (value: unknown): boolean | null => {
+      if (value === 'true' || value === true) return true;
+      if (value === 'false' || value === false) return false;
+      return null;
+    };
+
+    const extractFromAkoma = (akoma: any): boolean | null => {
+      const node = akoma?.act?.meta?.proprietary?.isInForce ?? null;
+      const v = node?.['@_value'];
+      return coerce(v);
+    };
+
+    // Helper: given "akomaNtoso" which may be object or array, find first parseable
+    const extractFromAkomaNtoso = (akomaNtoso: any): boolean | null => {
+      if (!akomaNtoso) return null;
+
+      if (Array.isArray(akomaNtoso)) {
+        for (const a of akomaNtoso) {
+          const r = extractFromAkoma(a);
+          if (r !== null) return r;
+        }
+        return null;
+      }
+
+      return extractFromAkoma(akomaNtoso);
+    };
+
+    // Finlex responses can come in a couple of wrappers depending on endpoint.
+    const candidates = [
+      parsed?.AknXmlList?.Results?.akomaNtoso,
+      parsed?.AknXmlList?.Results?.Results?.akomaNtoso, // defensive
+      parsed?.Results?.akomaNtoso,
+      parsed?.akomaNtoso,
+    ];
+
+    for (const c of candidates) {
+      const r = extractFromAkomaNtoso(c);
+      if (r !== null) return r;
+    }
+
+    return null;
+  } catch (e) {
+    console.warn('Failed to parse isInForce from XML:', e);
+    return null;
+  }
+}
+
+export { listStatutesByYear, setSingleStatute, listJudgmentNumbersByYear, listJudgmentsByYear, parseURLfromJudgmentID, setSingleJudgment, parseAkomafromURL, parseFinlexUrl, parseJudgmentUrl, buildFinlexUrl, buildJudgmentUrl, finlexLimiter, fetchWithBackoff }