fix: implement backwards compatibility with old sqlite formats (#658)

* feat: implement database versioning and migration support

* test: commit v0 report

* fix: use debug logging for snapshot saving instead of console

* test: implement sqlite formats backwards compatibility e2e test

* test: add unit tests for database migrations

* chore: fix review issues

* chore: update package-lock

* fix: adjust player dimensions and hide player when streaming is available, but snapshots found

* test: fix e2e tests

Author: shadowusr

lib/adapters/event-handling/testplane/snapshots.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import path from 'path';
 
 import type {eventWithTime as RrwebEvent} from '@rrweb/types';
+import makeDebug from 'debug';
 import fsExtra from 'fs-extra';
 import _ from 'lodash';
 import type Testplane from 'testplane';
@@ -14,6 +15,8 @@ import {AttachmentType, SnapshotAttachment, TestStepKey, SnapshotsSaver} from '.
 import {EventSource} from '../../../gui/event-source';
 import {ClientEvents} from '../../../gui/constants';
 
+const debug = makeDebug('html-reporter:event-handling:snapshots');
+
 export interface TestContext {
     testPath: string[];
     browserId: string;
@@ -86,12 +89,10 @@ export const finalizeSnapshotsForTest = async ({testResult, attempt, reportPath,
 
         // Here we only check LastFailedRun, because in case of Off, we wouldn't even be here. LastFailedRun is the only case when we may want to not save snapshots.
         const shouldSave = RecordMode && recordConfig && (recordConfig.mode !== RecordMode.LastFailedRun || (eventName === events.TEST_FAIL));
-        if (!shouldSave) {
-            return [];
-        }
-
-        if (!snapshots || snapshots.length === 0) {
-            console.warn(`No snapshots found for test hash: ${hash}`);
+        if (!shouldSave || !snapshots || snapshots.length === 0) {
+            debug('Not saving snapshots for test "%s"', hash);
+            debug('shouldSave evaluated to: %s', shouldSave, ', recordConfig: ', recordConfig, ', eventName: ', eventName);
+            debug('snapshots: ', snapshots);
             return [];
         }
 

lib/constants/database.ts

@@ -38,6 +38,17 @@ export const SUITES_TABLE_COLUMNS = [
     {name: DB_COLUMNS.ATTACHMENTS, type: DB_TYPES.text}
 ] as const;
 
+export const DB_VERSION_TABLE_NAME = 'version';
+export const VERSION_TABLE_COLUMNS = [
+    {name: 'version_number', type: DB_TYPES.int}
+] as const;
+
+export type LabeledVersionRow = {
+    [K in (typeof VERSION_TABLE_COLUMNS)[number]['name']]: string;
+};
+
+export const DB_CURRENT_VERSION = 1;
+
 export const DB_MAX_AVAILABLE_PAGE_SIZE = 65536; // helps to speed up queries
 export const DB_SUITES_TABLE_NAME = 'suites';
 export const LOCAL_DATABASE_NAME = 'sqlite.db';

lib/db-utils/common.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import {logger} from '../common-utils';
-import {DB_MAX_AVAILABLE_PAGE_SIZE, DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS, DB_COLUMN_INDEXES} from '../constants';
+import {DB_MAX_AVAILABLE_PAGE_SIZE, DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS, DB_COLUMN_INDEXES, DB_VERSION_TABLE_NAME, VERSION_TABLE_COLUMNS} from '../constants';
 import {DbUrlsJsonData, RawSuitesRow, ReporterConfig} from '../types';
 import type {Database as BetterSqlite3Database, Statement} from 'better-sqlite3';
 import {ReadonlyDeep} from 'type-fest';
@@ -9,7 +9,8 @@ export const selectAllQuery = (tableName: string): string => `SELECT * FROM ${ta
 export const selectAllSuitesQuery = (): string => selectAllQuery(DB_SUITES_TABLE_NAME);
 
 export const createTablesQuery = (): string[] => [
-    createTableQuery(DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS)
+    createTableQuery(DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS),
+    createTableQuery(DB_VERSION_TABLE_NAME, VERSION_TABLE_COLUMNS)
 ];
 
 export const compareDatabaseRowsByTimestamp = (row1: RawSuitesRow, row2: RawSuitesRow): number => {
@@ -86,7 +87,7 @@ export const mergeTables = ({db, dbPaths, getExistingTables = (): string[] => []
     }
 };
 
-function createTableQuery(tableName: string, columns: ReadonlyDeep<{name: string, type: string }[]>): string {
+export function createTableQuery(tableName: string, columns: ReadonlyDeep<{name: string, type: string }[]>): string {
     const formattedColumns = columns
         .map(({name, type}) => `${name} ${type}`)
         .join(', ');

lib/db-utils/migrations.ts

@@ -0,0 +1,114 @@
+import fs from 'fs-extra';
+import makeDebug from 'debug';
+import Database from 'better-sqlite3';
+import {
+    DB_VERSION_TABLE_NAME,
+    DB_CURRENT_VERSION,
+    DB_SUITES_TABLE_NAME,
+    DB_COLUMNS,
+    VERSION_TABLE_COLUMNS,
+    LabeledVersionRow
+} from '../constants/database';
+import {isEqual} from 'lodash';
+import {createTableQuery} from './common';
+const debug = makeDebug('html-reporter:db-migrations');
+
+export const getDatabaseVersion = (db: Database.Database): number | null => {
+    debug('getDatabaseVersion');
+    try {
+        const versionTableExists = db.prepare(
+            `SELECT name FROM sqlite_master WHERE type='table' AND name=?`
+        ).get(DB_VERSION_TABLE_NAME);
+        debug('versionTableExists:', versionTableExists);
+
+        if (versionTableExists) {
+            const versionRow = db.prepare(`SELECT ${VERSION_TABLE_COLUMNS[0].name} FROM ${DB_VERSION_TABLE_NAME} LIMIT 1`).get() as LabeledVersionRow | null;
+            debug('versionRow: ', versionRow);
+
+            const version = Number(versionRow?.[VERSION_TABLE_COLUMNS[0].name]);
+            if (!isNaN(version) && version >= 0) {
+                return version;
+            }
+        }
+
+        const tableInfo = db.prepare(`PRAGMA table_info(${DB_SUITES_TABLE_NAME})`).all() as {name: string}[];
+        const columnNames = tableInfo.map(col => col.name);
+        debug('column names in db:', columnNames);
+
+        const version0Columns = [
+            DB_COLUMNS.SUITE_PATH,
+            DB_COLUMNS.SUITE_NAME,
+            DB_COLUMNS.NAME,
+            DB_COLUMNS.SUITE_URL,
+            DB_COLUMNS.META_INFO,
+            DB_COLUMNS.HISTORY,
+            DB_COLUMNS.DESCRIPTION,
+            DB_COLUMNS.ERROR,
+            DB_COLUMNS.SKIP_REASON,
+            DB_COLUMNS.IMAGES_INFO,
+            DB_COLUMNS.SCREENSHOT,
+            DB_COLUMNS.MULTIPLE_TABS,
+            DB_COLUMNS.STATUS,
+            DB_COLUMNS.TIMESTAMP,
+            DB_COLUMNS.DURATION
+        ];
+
+        if (isEqual(columnNames, version0Columns)) {
+            return 0;
+        }
+
+        if (isEqual(columnNames, [...version0Columns, DB_COLUMNS.ATTACHMENTS])) {
+            return 1;
+        }
+    } catch (error) {
+        debug(`Error getting database version: ${error}`);
+    }
+    return null;
+};
+
+export const setDatabaseVersion = (db: Database.Database, version: number): void => {
+    db.prepare(`DELETE FROM ${DB_VERSION_TABLE_NAME}`).run();
+    db.prepare(`INSERT INTO ${DB_VERSION_TABLE_NAME} (${VERSION_TABLE_COLUMNS[0].name}) VALUES (?)`).run(version);
+};
+
+/**
+ * Migration from version 0 to version 1
+ * - Adds attachments column to suites table
+ */
+const migrateV0ToV1 = (db: Database.Database): void => {
+    debug('migrating from v0 to v1');
+    db.prepare(`ALTER TABLE ${DB_SUITES_TABLE_NAME} ADD COLUMN ${DB_COLUMNS.ATTACHMENTS} TEXT DEFAULT '[]'`).run();
+
+    db.prepare(createTableQuery(DB_VERSION_TABLE_NAME, VERSION_TABLE_COLUMNS)).run();
+};
+
+export const migrateDatabase = async (db: Database.Database, version: number): Promise<void> => {
+    const migrations = [
+        migrateV0ToV1
+    ];
+
+    try {
+        if (version < 0 || version >= migrations.length) {
+            throw new Error(`Unsupported database version encountered. Try deleting your report directory and restarting html-reporter.`);
+        }
+
+        for (const migration of migrations.slice(version)) {
+            migration(db);
+        }
+
+        setDatabaseVersion(db, DB_CURRENT_VERSION);
+    } catch (error) {
+        debug(`Error during database migration: ${error}`);
+        throw error;
+    }
+};
+
+export const backupAndReset = async (reportPath: string): Promise<string> => {
+    const timestamp = Date.now();
+    const backupDir = `${reportPath}_backup_${timestamp}`;
+    debug(`Creating backup of corrupted database at: ${backupDir}`);
+
+    await fs.move(reportPath, backupDir);
+
+    return backupDir;
+};

lib/gui/tool-runner/index.ts

@@ -17,7 +17,7 @@ import {ImagesInfoSaver} from '../../images-info-saver';
 import {SqliteImageStore} from '../../image-store';
 import * as reporterHelper from '../../reporter-helpers';
 import {logger, getShortMD5, isUpdatedStatus} from '../../common-utils';
-import {formatId, mkFullTitle, mergeDatabasesForReuse, filterByEqualDiffSizes} from './utils';
+import {formatId, mkFullTitle, mergeDatabasesForReuse, filterByEqualDiffSizes, prepareLocalDatabase} from './utils';
 import {getExpectedCacheKey} from '../../server-utils';
 import {getTestsTreeFromDatabase} from '../../db-utils/server';
 import {
@@ -106,6 +106,7 @@ export class ToolRunner {
 
     async initialize(): Promise<void> {
         await mergeDatabasesForReuse(this._reportPath);
+        await prepareLocalDatabase(this._reportPath);
 
         const dbClient = await SqliteClient.create({htmlReporter: this._toolAdapter.htmlReporter, reportPath: this._reportPath, reuse: true});
         const imageStore = new SqliteImageStore(dbClient);

lib/gui/tool-runner/utils.ts

@@ -6,10 +6,15 @@ import Database from 'better-sqlite3';
 import type {CoordBounds} from 'looks-same';
 
 import {logger} from '../../common-utils';
-import {DATABASE_URLS_JSON_NAME, LOCAL_DATABASE_NAME} from '../../constants';
+import {DATABASE_URLS_JSON_NAME, DB_CURRENT_VERSION, LOCAL_DATABASE_NAME} from '../../constants';
 import {mergeTables} from '../../db-utils/server';
 import {TestEqualDiffsData, TestRefUpdateData} from '../../tests-tree-builder/gui';
 import {ImageInfoDiff, ImageSize} from '../../types';
+import {backupAndReset, getDatabaseVersion, migrateDatabase} from '../../db-utils/migrations';
+
+import makeDebug from 'debug';
+
+const debug = makeDebug('html-reporter:gui:tool-runner:utils');
 
 export const formatId = (hash: string, browserId: string): string => `${hash}/${browserId}`;
 
@@ -50,6 +55,36 @@ export const mergeDatabasesForReuse = async (reportPath: string): Promise<void>
     await Promise.all(dbPaths.map(p => fs.remove(p)));
 };
 
+export const prepareLocalDatabase = async (reportPath: string): Promise<void> => {
+    debug('prepareLocalDatabase', reportPath);
+    const dbPath = path.resolve(reportPath, LOCAL_DATABASE_NAME);
+
+    if (!fs.existsSync(dbPath)) {
+        return;
+    }
+
+    const db = new Database(dbPath);
+    try {
+        const version = getDatabaseVersion(db);
+        debug('determined db version', version);
+
+        if (version !== null && version < DB_CURRENT_VERSION) {
+            await migrateDatabase(db, version);
+        } else if (version === null) {
+            const backupPath = await backupAndReset(reportPath);
+            console.warn(`SQLite db at ${dbPath} is of unknown unsupported version.\nBacked up to ${backupPath} and starting from scratch.`);
+        } else if (version > DB_CURRENT_VERSION) {
+            const backupPath = await backupAndReset(reportPath);
+            console.warn(`SQLite db at ${dbPath} is of unsupported version. ` +
+                'This probably happened because the report was generated with a newer version of html-reporter than you are trying to use now. ' +
+                'Please update html-reporter to the latest version to open this report.\n' +
+                `Backed up to ${backupPath} and starting from scratch.`);
+        }
+    } finally {
+        db.close();
+    }
+};
+
 export const filterByEqualDiffSizes = (imagesInfo: TestEqualDiffsData[], refDiffClusters?: CoordBounds[]): TestEqualDiffsData[] => {
     if (!refDiffClusters || _.isEmpty(refDiffClusters)) {
         return [];

lib/sqlite-client.ts

@@ -5,8 +5,9 @@ import fs from 'fs-extra';
 import NestedError from 'nested-error-stacks';
 
 import {getShortMD5} from './common-utils';
-import {TestStatus, DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS, LOCAL_DATABASE_NAME, DATABASE_URLS_JSON_NAME} from './constants';
+import {TestStatus, DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS, LOCAL_DATABASE_NAME, DATABASE_URLS_JSON_NAME, DB_CURRENT_VERSION} from './constants';
 import {createTablesQuery} from './db-utils/common';
+import {setDatabaseVersion} from './db-utils/migrations';
 import type {Attachment, ImageInfoFull, TestError, TestStepCompressed} from './types';
 import {HtmlReporter} from './plugin-api';
 import {ReporterTestResult} from './adapters/test-result';
@@ -90,6 +91,10 @@ export class SqliteClient {
             debug('db connection opened');
 
             createTablesQuery().forEach((query) => db?.prepare(query).run());
+
+            if (!options.reuse) {
+                setDatabaseVersion(db, DB_CURRENT_VERSION);
+            }
         } catch (err: any) { // eslint-disable-line @typescript-eslint/no-explicit-any
             throw new NestedError(`Error creating database at "${dbPath}"`, err);
         }

lib/static/new-ui/experiments/time-travel/components/SnapshotsPlayer/index.tsx

@@ -72,7 +72,7 @@ export function SnapshotsPlayer(): ReactNode {
     const [playerWidth, setPlayerWidth] = useState<number>(1200);
     const [playerHeight, setPlayerHeight] = useState<number>(800);
     const isLiveMaxSizeInitialized = useRef(false);
-    const [maxPlayerSize, setMaxPlayerSize] = useState<ImageSize>({height: 0, width: 0});
+    const [maxPlayerSize, setMaxPlayerSize] = useState<ImageSize>({height: 400, width: 700});
 
     const [totalTime, setTotalTime] = useState(0);
     const [isPlaying, setIsPlaying] = useState(false);

lib/static/new-ui/experiments/time-travel/components/TestInfo/index.tsx

@@ -17,13 +17,13 @@ export function TestInfo(): ReactNode {
     const currentResult = useSelector(getCurrentResult);
     const browserFeatures = useSelector(state => state.browserFeatures);
 
-    const isSnapshotAvailable = currentResult?.attachments?.some(attachment => attachment.type === AttachmentType.Snapshot);
-    const isLiveStreamingAvailable = browserFeatures[currentResult?.name ?? '']?.some(feature => feature === BrowserFeature.LiveSnapshotsStreaming);
-    const shouldShowPlayer = isSnapshotAvailable || isLiveStreamingAvailable;
-
     const steps = useSelector(getTestSteps);
     const isRunning = useSelector(state => state.running);
 
+    const isSnapshotAvailable = currentResult?.attachments?.some(attachment => attachment.type === AttachmentType.Snapshot);
+    const isLiveStreamingAvailable = browserFeatures[currentResult?.name ?? '']?.some(feature => feature === BrowserFeature.LiveSnapshotsStreaming);
+    const shouldShowPlayer = isSnapshotAvailable || (isRunning && isLiveStreamingAvailable);
+
     return <>
         <CollapsibleSection id={'actions'} title={'Actions'}>
             <div className={styles.stepsContainer}>