Skip to content

Commit 553196e

Browse files
baghbidibaghbidi
authored
Handle SQLITE_BUSY error (#615)
* Retry on SQLITE_BUSY error
1 parent 91ce615 commit 553196e

File tree

10 files changed

+118
-98
lines changed

10 files changed

+118
-98
lines changed

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,7 @@ node_modules
33
*.sqlite
44
.nyc_output
55
/.idea
6-
/src/config/tracking-uuid
6+
/src/config/tracking-uuid
7+
*.sqlite-shm
8+
*.sqlite-wal
9+
*.pid

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
"child_process": "1.0.2",
6868
"command-line-args": "5.0.2",
6969
"command-line-usage": "5.0.5",
70-
"continuation-local-storage": "3.2.1",
70+
"concurrent-queue": "7.0.2",
7171
"cookie-parser": "1.4.3",
7272
"daemonize2": "0.4.2",
7373
"ejs": "2.6.1",
@@ -78,8 +78,8 @@
7878
"jsonschema": "1.2.4",
7979
"moment": "2.24.0",
8080
"morgan": "1.9.1",
81-
"newman": "4.5.0",
8281
"nconf": "0.10.0",
82+
"newman": "4.5.0",
8383
"nodemailer": "5.1.1",
8484
"nodemailer-smtp-transport": "2.7.4",
8585
"os": "0.1.1",

‎src/cli/start.js‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ class Start extends BaseCLIHandler {
3838

3939
async initDB() {
4040
try {
41-
await db.migrate()
42-
await db.seed()
41+
await db.initDB()
4342
} catch (err) {
4443
logger.error('Unable to initialize the database.', err)
4544
process.exit(1)

‎src/decorators/transaction-decorator.js‎

Lines changed: 50 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -10,82 +10,80 @@
1010
* *******************************************************************************
1111
*
1212
*/
13-
14-
const db = require('./../sequelize/models')
15-
const retry = require('retry-as-promised')
16-
const sequelize = db.sequelize
13+
const cq = require('concurrent-queue')
1714
const Transaction = require('sequelize/lib/transaction')
15+
1816
const { isTest } = require('../helpers/app-helper')
1917

18+
const transactionsQueue = cq()
19+
.limit({ concurrency: 1 })
20+
.process((task, cb) => {
21+
task.transaction
22+
.apply(task.that, task.args)
23+
.then((res) => cb(null, res))
24+
.catch((err) => cb(err, null))
25+
})
26+
2027
function transaction(f) {
21-
return async function(...fArgs) {
28+
const fakeTransactionObject = { fakeTransaction: true }
29+
return function(...fArgs) {
2230
if (isTest()) {
23-
return await f.apply(this, fArgs)
31+
return f.apply(this, fArgs)
2432
}
2533

26-
// TODO [when transactions concurrency issue fixed]: Remove 'fArgs[fArgs.length - 1].fakeTransaction'
27-
if (fArgs.length > 0 && fArgs[fArgs.length - 1]
28-
&& (fArgs[fArgs.length - 1] instanceof Transaction || fArgs[fArgs.length - 1].fakeTransaction)) {
29-
return await f.apply(this, fArgs)
34+
if (fArgs.length > 0 && fArgs[fArgs.length - 1] instanceof Transaction) {
35+
fArgs[fArgs.length - 1] = fakeTransactionObject
36+
return f.apply(this, fArgs)
3037
} else {
31-
// return f.apply(this, fArgs)
32-
return sequelize.transaction(async (t) => {
33-
fArgs.push(t)
34-
return await f.apply(this, fArgs)
35-
})
38+
fArgs.push(fakeTransactionObject)
39+
return f.apply(this, fArgs)
3640
}
3741
}
3842
}
3943

40-
function generateTransaction(f) {
41-
return function(...args) {
42-
return retry(() => {
43-
const t = transaction(f)
44-
return t.apply(this, args)
45-
}, {
46-
max: 5,
47-
match: [
48-
sequelize.ConnectionError,
49-
'SQLITE_BUSY',
50-
],
51-
})
44+
function queueTransaction(resolve, reject, transaction, that, retries, ...args) {
45+
const task = {
46+
transaction,
47+
that,
48+
retries,
49+
args,
5250
}
53-
}
5451

55-
function fakeTransaction(f) {
56-
const fakeTransactionObject = { fakeTransaction: true }
57-
return async function(...fArgs) {
58-
if (isTest()) {
59-
return await f.apply(this, fArgs)
52+
transactionsQueue(task, (error, success) => {
53+
if (error === null) {
54+
return resolve(success)
6055
}
6156

62-
if (fArgs.length > 0 && fArgs[fArgs.length - 1] instanceof Transaction) {
63-
fArgs[fArgs.length - 1] = fakeTransactionObject
64-
return await f.apply(this, fArgs)
65-
} else {
66-
fArgs.push(fakeTransactionObject)
67-
return await f.apply(this, fArgs)
57+
if (retries < 1 || (error.message || '').indexOf('SQLITE_BUSY') === -1) {
58+
return reject(error)
6859
}
69-
}
60+
61+
queueTransaction(resolve, reject, transaction, that, retries - 1, ...args)
62+
})
7063
}
7164

72-
// TODO [when transactions concurrency issue fixed]: Remove
73-
function generateFakeTransaction(f) {
65+
function applyTransaction(resolve, reject, transaction, that, ...args) {
66+
transaction.apply(that, args)
67+
.then(resolve)
68+
.catch((error) => {
69+
if ((error.message || '').indexOf('SQLITE_BUSY') === -1) {
70+
return reject(error)
71+
}
72+
73+
queueTransaction(resolve, reject, transaction, this, 5, ...args)
74+
})
75+
}
76+
77+
function generateTransaction(f) {
7478
return function(...args) {
75-
return retry(() => {
76-
const t = fakeTransaction(f)
77-
return t.apply(this, args)
78-
}, {
79-
max: 5,
80-
match: [
81-
sequelize.ConnectionError,
82-
'SQLITE_BUSY',
83-
],
79+
const t = transaction(f)
80+
81+
return new Promise((resolve, reject) => {
82+
applyTransaction(resolve, reject, t, this, ...args)
8483
})
8584
}
8685
}
8786

8887
module.exports = {
8988
generateTransaction: generateTransaction,
90-
generateFakeTransaction: generateFakeTransaction,
9189
}

‎src/jobs/fog-status-job.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class FogStatusJob extends BaseJobHandler {
2929
}
3030

3131
run() {
32-
setInterval(TransactionDecorator.generateFakeTransaction(updateFogsConnectionStatus), this.scheduleTime)
32+
setInterval(TransactionDecorator.generateTransaction(updateFogsConnectionStatus), this.scheduleTime)
3333
}
3434
}
3535

‎src/jobs/time-tracking-job.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class TimeTrackingJob extends BaseJobHandler {
3939
async trackTime() {
4040
let agentsCount = 0
4141
try {
42-
const agents = await TransactionDecorator.generateFakeTransaction(FogAccessTokenService.all)()
42+
const agents = await TransactionDecorator.generateTransaction(FogAccessTokenService.all)()
4343
agentsCount = (agents || []).length
4444
} catch (e) {
4545
logger.warn('Unable to count ioFog agents')

‎src/sequelize/config/config.json‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,16 @@
44
"storage": "dev_database.sqlite",
55
"logging": false,
66
"operatorsAliases": false,
7-
"transactionType": "IMMEDIATE",
7+
"transactionType": "DEFERRED",
8+
"retry": {
9+
"match": [
10+
"/SQLITE_BUSY/"
11+
],
12+
"max": 0
13+
},
814
"pool": {
915
"maxactive": 1,
10-
"max": 1,
16+
"max": 5,
1117
"min": 0,
1218
"idle": 20000
1319
}
@@ -17,10 +23,16 @@
1723
"storage": "test_database.sqlite",
1824
"logging": false,
1925
"operatorsAliases": false,
20-
"transactionType": "IMMEDIATE",
26+
"transactionType": "DEFERRED",
27+
"retry": {
28+
"match": [
29+
"/SQLITE_BUSY/"
30+
],
31+
"max": 0
32+
},
2133
"pool": {
2234
"maxactive": 1,
23-
"max": 1,
35+
"max": 5,
2436
"min": 0,
2537
"idle": 20000
2638
}
@@ -30,10 +42,16 @@
3042
"storage": "prod_database.sqlite",
3143
"logging": false,
3244
"operatorsAliases": false,
33-
"transactionType": "IMMEDIATE",
45+
"transactionType": "DEFERRED",
46+
"retry": {
47+
"match": [
48+
"/SQLITE_BUSY/"
49+
],
50+
"max": 0
51+
},
3452
"pool": {
3553
"maxactive": 1,
36-
"max": 1,
54+
"max": 5,
3755
"min": 0,
3856
"idle": 20000
3957
}

‎src/sequelize/models/index.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ Object.keys(db).forEach((modelName) => {
5757
db.sequelize = sequelize
5858
db.Sequelize = Sequelize
5959

60-
db.migrate = () => createUmzug(path.resolve(__dirname, '../migrations')).up()
61-
db.seed = () => createUmzug(path.resolve(__dirname, '../seeders')).up()
60+
db.initDB = async () => {
61+
await createUmzug(path.resolve(__dirname, '../migrations')).up()
62+
await createUmzug(path.resolve(__dirname, '../seeders')).up()
63+
}
6264

6365
module.exports = db

‎src/services/agent-service.js‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -520,23 +520,23 @@ const agentProvisionWithTracking = TrackingDecorator.trackEvent(agentProvision,
520520

521521

522522
module.exports = {
523-
agentProvision: TransactionDecorator.generateFakeTransaction(agentProvisionWithTracking),
524-
agentDeprovision: TransactionDecorator.generateFakeTransaction(agentDeprovision),
523+
agentProvision: TransactionDecorator.generateTransaction(agentProvisionWithTracking),
524+
agentDeprovision: TransactionDecorator.generateTransaction(agentDeprovision),
525525
getAgentConfig: getAgentConfig,
526-
updateAgentConfig: TransactionDecorator.generateFakeTransaction(updateAgentConfig),
527-
getAgentConfigChanges: TransactionDecorator.generateFakeTransaction(getAgentConfigChanges),
528-
updateAgentStatus: TransactionDecorator.generateFakeTransaction(updateAgentStatus),
529-
getAgentMicroservices: TransactionDecorator.generateFakeTransaction(getAgentMicroservices),
530-
getAgentMicroservice: TransactionDecorator.generateFakeTransaction(getAgentMicroservice),
531-
getAgentRegistries: TransactionDecorator.generateFakeTransaction(getAgentRegistries),
532-
getAgentTunnel: TransactionDecorator.generateFakeTransaction(getAgentTunnel),
533-
getAgentStrace: TransactionDecorator.generateFakeTransaction(getAgentStrace),
534-
updateAgentStrace: TransactionDecorator.generateFakeTransaction(updateAgentStrace),
535-
getAgentChangeVersionCommand: TransactionDecorator.generateFakeTransaction(getAgentChangeVersionCommand),
536-
updateHalHardwareInfo: TransactionDecorator.generateFakeTransaction(updateHalHardwareInfo),
537-
updateHalUsbInfo: TransactionDecorator.generateFakeTransaction(updateHalUsbInfo),
538-
deleteNode: TransactionDecorator.generateFakeTransaction(deleteNode),
539-
getImageSnapshot: TransactionDecorator.generateFakeTransaction(getImageSnapshot),
540-
putImageSnapshot: TransactionDecorator.generateFakeTransaction(putImageSnapshot),
541-
postTracking: TransactionDecorator.generateFakeTransaction(postTracking),
526+
updateAgentConfig: TransactionDecorator.generateTransaction(updateAgentConfig),
527+
getAgentConfigChanges: TransactionDecorator.generateTransaction(getAgentConfigChanges),
528+
updateAgentStatus: TransactionDecorator.generateTransaction(updateAgentStatus),
529+
getAgentMicroservices: TransactionDecorator.generateTransaction(getAgentMicroservices),
530+
getAgentMicroservice: TransactionDecorator.generateTransaction(getAgentMicroservice),
531+
getAgentRegistries: TransactionDecorator.generateTransaction(getAgentRegistries),
532+
getAgentTunnel: TransactionDecorator.generateTransaction(getAgentTunnel),
533+
getAgentStrace: TransactionDecorator.generateTransaction(getAgentStrace),
534+
updateAgentStrace: TransactionDecorator.generateTransaction(updateAgentStrace),
535+
getAgentChangeVersionCommand: TransactionDecorator.generateTransaction(getAgentChangeVersionCommand),
536+
updateHalHardwareInfo: TransactionDecorator.generateTransaction(updateHalHardwareInfo),
537+
updateHalUsbInfo: TransactionDecorator.generateTransaction(updateHalUsbInfo),
538+
deleteNode: TransactionDecorator.generateTransaction(deleteNode),
539+
getImageSnapshot: TransactionDecorator.generateTransaction(getImageSnapshot),
540+
putImageSnapshot: TransactionDecorator.generateTransaction(putImageSnapshot),
541+
postTracking: TransactionDecorator.generateTransaction(postTracking),
542542
}

‎src/services/kubelet-service.js‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -405,17 +405,17 @@ const microservicesTopologicalOrder = function (msMetadata) {
405405
}
406406

407407
module.exports = {
408-
kubeletCreatePod: TransactionDecorator.generateFakeTransaction(kubeletCreatePod),
409-
kubeletUpdatePod: TransactionDecorator.generateFakeTransaction(kubeletUpdatePod),
410-
kubeletDeletePod: TransactionDecorator.generateFakeTransaction(kubeletDeletePod),
411-
kubeletGetPod: TransactionDecorator.generateFakeTransaction(kubeletGetPod),
412-
kubeletGetContainerLogs: TransactionDecorator.generateFakeTransaction(kubeletGetContainerLogs),
413-
kubeletGetPodStatus: TransactionDecorator.generateFakeTransaction(kubeletGetPodStatus),
414-
kubeletGetPods: TransactionDecorator.generateFakeTransaction(kubeletGetPods),
415-
kubeletGetCapacity: TransactionDecorator.generateFakeTransaction(kubeletGetCapacity),
416-
kubeletGetAllocatable: TransactionDecorator.generateFakeTransaction(kubeletGetAllocatable),
417-
kubeletGetNodeConditions: TransactionDecorator.generateFakeTransaction(kubeletGetNodeConditions),
418-
kubeletGetNodeAddresses: TransactionDecorator.generateFakeTransaction(kubeletGetNodeAddresses),
419-
kubeletGetVkToken: TransactionDecorator.generateFakeTransaction(kubeletGetVkToken),
420-
kubeletGetSchedulerToken: TransactionDecorator.generateFakeTransaction(kubeletGetSchedulerToken),
408+
kubeletCreatePod: TransactionDecorator.generateTransaction(kubeletCreatePod),
409+
kubeletUpdatePod: TransactionDecorator.generateTransaction(kubeletUpdatePod),
410+
kubeletDeletePod: TransactionDecorator.generateTransaction(kubeletDeletePod),
411+
kubeletGetPod: TransactionDecorator.generateTransaction(kubeletGetPod),
412+
kubeletGetContainerLogs: TransactionDecorator.generateTransaction(kubeletGetContainerLogs),
413+
kubeletGetPodStatus: TransactionDecorator.generateTransaction(kubeletGetPodStatus),
414+
kubeletGetPods: TransactionDecorator.generateTransaction(kubeletGetPods),
415+
kubeletGetCapacity: TransactionDecorator.generateTransaction(kubeletGetCapacity),
416+
kubeletGetAllocatable: TransactionDecorator.generateTransaction(kubeletGetAllocatable),
417+
kubeletGetNodeConditions: TransactionDecorator.generateTransaction(kubeletGetNodeConditions),
418+
kubeletGetNodeAddresses: TransactionDecorator.generateTransaction(kubeletGetNodeAddresses),
419+
kubeletGetVkToken: TransactionDecorator.generateTransaction(kubeletGetVkToken),
420+
kubeletGetSchedulerToken: TransactionDecorator.generateTransaction(kubeletGetSchedulerToken),
421421
}

0 commit comments

Comments
 (0)