feat: merge all existing tables inside sqlite database (#404)

Author: DudaGod

lib/common-utils.js

@@ -83,23 +83,22 @@ function createTableQuery(tableName, columns) {
     return `CREATE TABLE IF NOT EXISTS ${tableName} (${formattedColumns})`;
 }
 
-exports.mergeTablesQueries = (dbPaths) => {
-    const tablesToMerge = [
-        DB_SUITES_TABLE_NAME
-    ];
-    const mkMergeQuery = (table) => `INSERT OR IGNORE INTO ${table} SELECT * FROM attached.${table}`;
-    const queries = [].concat(
-        `PRAGMA page_size = ${DB_MAX_AVAILABLE_PAGE_SIZE}`,
-        exports.createTablesQuery(),
-    );
+exports.mergeTables = ({db, dbPaths, getExistingTables = () => {}}) => {
+    db.prepare(`PRAGMA page_size = ${DB_MAX_AVAILABLE_PAGE_SIZE}`).run();
 
     for (const dbPath of dbPaths) {
-        queries.push(`ATTACH '${dbPath}' AS attached`);
-        queries.push.apply(queries, tablesToMerge.map(mkMergeQuery));
-        queries.push('DETACH attached');
-    }
+        db.prepare(`ATTACH '${dbPath}' AS attached`).run();
+
+        const getTablesStatement = db.prepare(`SELECT name FROM attached.sqlite_master WHERE type='table'`);
+        const tables = getExistingTables(getTablesStatement);
 
-    return queries;
+        for (const tableName of tables) {
+            db.prepare(`CREATE TABLE IF NOT EXISTS ${tableName} AS SELECT * FROM attached.${tableName} LIMIT 0`).run();
+            db.prepare(`INSERT OR IGNORE INTO ${tableName} SELECT * FROM attached.${tableName}`).run();
+        }
+
+        db.prepare(`DETACH attached`).run();
+    }
 };
 
 exports.compareDatabaseRowsByTimestamp = (row1, row2) => {

lib/gui/tool-runner/utils.js

@@ -11,8 +11,7 @@ const NestedError = require('nested-error-stacks');
 const StaticTestsTreeBuilder = require('../../tests-tree-builder/static');
 const {logger} = require('../../server-utils');
 const constantFileNames = require('../../constants/file-names');
-const {mergeTablesQueries, selectAllSuitesQuery, compareDatabaseRowsByTimestamp} = require('../../common-utils');
-const {writeDatabaseUrlsFile} = require('../../server-utils');
+const {mergeTables, selectAllSuitesQuery, compareDatabaseRowsByTimestamp} = require('../../common-utils');
 
 exports.formatId = (hash, browserId) => `${hash}/${browserId}`;
 
@@ -48,26 +47,11 @@ exports.mergeDatabasesForReuse = async (reportPath) => {
     logger.warn(chalk.yellow(`Merge databases to ${constantFileNames.LOCAL_DATABASE_NAME}`));
 
     const mergedDatabase = new Database(mergedDbPath);
-
-    for (const query of mergeTablesQueries(dbPaths)) {
-        try {
-            mergedDatabase.prepare(query).run();
-        } catch (err) {
-            // TODO: To be able to work with files that have been created before
-            // TODO: Remove in next versions
-            if (/no such table/.test(err)) {
-                console.warn(err);
-
-                continue;
-            }
-
-            throw err;
-        }
-    }
-
+    mergeTables({db: mergedDatabase, dbPaths, getExistingTables: (statement) => {
+        return statement.all().map((table) => table.name);
+    }});
     mergedDatabase.close();
 
-    await writeDatabaseUrlsFile(reportPath, [constantFileNames.LOCAL_DATABASE_NAME]);
     await Promise.all(dbPaths.map(p => fs.remove(p)));
 };
 

lib/static/modules/sqlite.js

@@ -2,7 +2,7 @@
 
 import axios from 'axios';
 import {flattenDeep} from 'lodash';
-import {mergeTablesQueries} from '../../common-utils';
+import {mergeTables} from '../../common-utils';
 
 function isRelativeUrl(url) {
     try {
@@ -114,21 +114,15 @@ async function mergeDatabases(dataForDbs) {
     const mergedDbConnection = new SQL.Database(undefined, sumOfChunkSizes);
     const dbPaths = connections.map(db => db.filename);
 
-    for (const query of mergeTablesQueries(dbPaths)) {
-        try {
-            mergedDbConnection.run(query);
-        } catch (err) {
-            // TODO: To be able to work with files that have been created before
-            // TODO: Remove in next versions
-            if (/no such table/.test(err)) {
-                console.warn(err);
-
-                continue;
-            }
+    mergeTables({db: mergedDbConnection, dbPaths, getExistingTables: (statement) => {
+        const tables = [];
 
-            throw err;
+        while (statement.step()) {
+            tables.push(...statement.get());
         }
-    }
+
+        return tables;
+    }});
 
     connections.forEach(db => db.close());
 

test/unit/lib/static/modules/sqlite.js

@@ -1,7 +1,7 @@
 'use strict';
 
 import axios from 'axios';
-import * as _ from 'lodash';
+import proxyquire from 'proxyquire';
 
 import {
     fetchDatabases,
@@ -162,18 +162,27 @@ describe('lib/static/modules/sqlite', () => {
     });
 
     describe('mergeDatabases', () => {
-        let SQL;
+        let SQL, statement;
         let DatabaseConstructorSpy;
 
         beforeEach(() => {
+            statement = {
+                step: () => {},
+                get: () => {},
+                run: () => {}
+            };
+
             SQL = {
                 Database: class {
                     run() {}
                     close() {}
+                    prepare() {
+                        return statement;
+                    }
                 }
             };
             DatabaseConstructorSpy = sinon.spy(SQL, 'Database');
-            sandbox.spy(SQL.Database.prototype, 'run');
+            sandbox.spy(SQL.Database.prototype, 'prepare');
             sandbox.spy(SQL.Database.prototype, 'close');
 
             global.window = {
@@ -198,7 +207,7 @@ describe('lib/static/modules/sqlite', () => {
 
             assert.instanceOf(mergedDbConnection, SQL.Database);
             assert.calledOnceWith(DatabaseConstructorSpy, new Uint8Array(1));
-            assert.notCalled(SQL.Database.prototype.run);
+            assert.notCalled(SQL.Database.prototype.prepare);
             assert.notCalled(SQL.Database.prototype.close);
         });
 
@@ -219,17 +228,30 @@ describe('lib/static/modules/sqlite', () => {
             assert.calledTwice(SQL.Database.prototype.close);
         });
 
-        it('should merge both "suites" tables', async () => {
-            const data1 = new ArrayBuffer(1);
-            const data2 = new ArrayBuffer(1);
+        describe('merge tables', () => {
+            let mergeDatabases, mergeTables;
+
+            beforeEach(() => {
+                mergeTables = sandbox.stub();
 
-            await mergeDatabases([data1, data2]);
+                mergeDatabases = proxyquire('lib/static/modules/sqlite', {
+                    '../../common-utils': {mergeTables}
+                }).mergeDatabases;
+            });
+
+            it('should get existing tables', async () => {
+                const statement = {
+                    step: sandbox.stub().returns(true).onThirdCall().returns(false),
+                    get: sandbox.stub()
+                        .onFirstCall().returns(['table1'])
+                        .onSecondCall().returns(['table2'])
+                };
 
-            const rawQueries = _
-                .flatten(SQL.Database.prototype.run.args)
-                .join(' ');
+                await mergeDatabases([new ArrayBuffer(1), new ArrayBuffer(2)]);
+                const tables = mergeTables.getCall(0).args[0].getExistingTables(statement);
 
-            assert.include(rawQueries, 'suites');
+                assert.deepEqual(tables, ['table1', 'table2']);
+            });
         });
     });