Skip to content

Commit a49d39e

Browse files
zsculaczsculac
authored
Merge pull request #4084 from OriginTrail/v6/prerelease/mainnet
8.2.5 Release Mainnet
2 parents f99ce4d + 3c31e32 commit a49d39e

File tree

15 files changed

+484
-52
lines changed

15 files changed

+484
-52
lines changed

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "origintrail_node",
3-
"version": "8.2.2",
3+
"version": "8.2.5",
44
"description": "OTNode V8",
55
"main": "index.js",
66
"type": "module",

‎src/commands/cleaners/operation-id-cleaner-command.js‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
BYTES_IN_KILOBYTE,
44
OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER,
55
OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS,
6+
OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS,
67
OPERATION_ID_STATUS,
78
COMMAND_PRIORITY,
89
} from '../../constants/constants.js';
@@ -23,14 +24,33 @@ class OperationIdCleanerCommand extends Command {
2324
* @param command
2425
*/
2526
async execute() {
27+
let memoryBytes = 0;
28+
let fileBytes = 0;
29+
try {
30+
memoryBytes = this.operationIdService.getOperationIdMemoryCacheSizeBytes();
31+
} catch (error) {
32+
this.logger.warn(`Unable to read memory cache footprint: ${error.message}`);
33+
}
34+
try {
35+
fileBytes = await this.operationIdService.getOperationIdFileCacheSizeBytes();
36+
} catch (error) {
37+
this.logger.warn(`Unable to read file cache footprint: ${error.message}`);
38+
}
39+
const bytesInMegabyte = 1024 * 1024;
40+
this.logger.debug(
41+
`Operation cache footprint before cleanup: memory=${(
42+
memoryBytes / bytesInMegabyte
43+
).toFixed(2)}MB, files=${(fileBytes / bytesInMegabyte).toFixed(2)}MB`,
44+
);
45+
2646
this.logger.debug('Starting command for removal of expired cache files');
2747
const timeToBeDeleted = Date.now() - OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS;
2848
await this.repositoryModuleManager.removeOperationIdRecord(timeToBeDeleted, [
2949
OPERATION_ID_STATUS.COMPLETED,
3050
OPERATION_ID_STATUS.FAILED,
3151
]);
3252
let removed = await this.operationIdService.removeExpiredOperationIdMemoryCache(
33-
OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS,
53+
OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS,
3454
);
3555
if (removed) {
3656
this.logger.debug(
@@ -68,7 +88,7 @@ class OperationIdCleanerCommand extends Command {
6888
default(map) {
6989
const command = {
7090
name: 'operationIdCleanerCommand',
71-
period: OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS,
91+
period: OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS,
7292
data: {},
7393
transactional: false,
7494
priority: COMMAND_PRIORITY.LOWEST,

‎src/commands/protocols/publish/publish-finalization-command.js‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ class PublishFinalizationCommand extends Command {
3333
const { id, publishOperationId, merkleRoot, byteSize } = eventData;
3434
const { blockchain, contractAddress } = event;
3535
const operationId = this.operationIdService.generateId();
36+
const ual = this.ualService.deriveUAL(blockchain, contractAddress, id);
37+
3638
this.operationIdService.emitChangeEvent(
3739
OPERATION_ID_STATUS.PUBLISH_FINALIZATION.PUBLISH_FINALIZATION_START,
3840
operationId,
@@ -70,8 +72,10 @@ class PublishFinalizationCommand extends Command {
7072
cachedMerkleRoot = result.merkleRoot;
7173
assertion = result.assertion;
7274
publisherPeerId = result.remotePeerId;
73-
} catch (error) {
74-
this.logger.error(`Failed to read cached publish data: ${error.message}`); // TODO: Make this log more descriptive
75+
} catch (_error) {
76+
this.logger.warn(
77+
`[Cache] Failed to read cached publish data for UAL ${ual} (publishOperationId: ${publishOperationId}, txHash: ${txHash}, operationId: ${operationId})`,
78+
);
7579
this.operationIdService.emitChangeEvent(
7680
OPERATION_ID_STATUS.FAILED,
7781
operationId,
@@ -81,8 +85,6 @@ class PublishFinalizationCommand extends Command {
8185
return Command.empty();
8286
}
8387

84-
const ual = this.ualService.deriveUAL(blockchain, contractAddress, id);
85-
8688
try {
8789
await this.validatePublishData(merkleRoot, cachedMerkleRoot, byteSize, assertion, ual);
8890
} catch (e) {
@@ -185,23 +187,24 @@ class PublishFinalizationCommand extends Command {
185187

186188
async readWithRetries(publishOperationId) {
187189
let attempt = 0;
190+
const datasetPath = this.fileService.getPendingStorageDocumentPath(publishOperationId);
188191

189192
while (attempt < MAX_RETRIES_READ_CACHED_PUBLISH_DATA) {
190193
try {
191-
const datasetPath =
192-
this.fileService.getPendingStorageDocumentPath(publishOperationId);
193194
// eslint-disable-next-line no-await-in-loop
194195
const cachedData = await this.fileService.readFile(datasetPath, true);
195196
return cachedData;
196197
} catch (error) {
197198
attempt += 1;
198-
199199
// eslint-disable-next-line no-await-in-loop
200200
await new Promise((resolve) => {
201201
setTimeout(resolve, RETRY_DELAY_READ_CACHED_PUBLISH_DATA);
202202
});
203203
}
204204
}
205+
this.logger.warn(
206+
`[Cache] Exhausted retries reading cached publish data (publishOperationId: ${publishOperationId}, path: ${datasetPath}).`,
207+
);
205208
// TODO: Mark this operation as failed
206209
throw new Error('Failed to read cached publish data');
207210
}

‎src/constants/constants.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -722,6 +722,11 @@ export const EXPECTED_TRANSACTION_ERRORS = {
722722
* operation id command cleanup interval time 24h
723723
*/
724724
export const OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000;
725+
/**
726+
* @constant {number} OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS -
727+
* operation id memory cleanup interval time 1h
728+
*/
729+
export const OPERATION_ID_MEMORY_CLEANUP_TIME_MILLS = 60 * 60 * 1000;
725730
/**
726731
* @constant {number} FINALIZED_COMMAND_CLEANUP_TIME_MILLS - Command cleanup interval time
727732
* finalized commands command cleanup interval time 24h

‎src/controllers/http-api/v1/publish-http-api-controller-v1.js‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
OPERATION_STATUS,
66
LOCAL_STORE_TYPES,
77
COMMAND_PRIORITY,
8+
PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS,
89
} from '../../../constants/constants.js';
910

1011
class PublishController extends BaseController {
@@ -16,6 +17,7 @@ class PublishController extends BaseController {
1617
this.repositoryModuleManager = ctx.repositoryModuleManager;
1718
this.pendingStorageService = ctx.pendingStorageService;
1819
this.networkModuleManager = ctx.networkModuleManager;
20+
this.blockchainModuleManager = ctx.blockchainModuleManager;
1921
}
2022

2123
async handleRequest(req, res) {
@@ -62,6 +64,37 @@ class PublishController extends BaseController {
6264
datasetRoot,
6365
});
6466

67+
let effectiveMinReplications = minimumNumberOfNodeReplications;
68+
let chainMinNumber = null;
69+
try {
70+
const chainMin = await this.blockchainModuleManager.getMinimumRequiredSignatures(
71+
blockchain,
72+
);
73+
chainMinNumber = Number(chainMin);
74+
} catch (err) {
75+
this.logger.warn(
76+
`Failed to fetch on-chain minimumRequiredSignatures for ${blockchain}: ${err.message}`,
77+
);
78+
}
79+
80+
const userMinNumber = Number(effectiveMinReplications);
81+
const resolvedUserMin =
82+
!Number.isNaN(userMinNumber) && userMinNumber > 0
83+
? userMinNumber
84+
: PUBLISH_MIN_NUM_OF_NODE_REPLICATIONS;
85+
86+
if (!Number.isNaN(chainMinNumber) && chainMinNumber > 0) {
87+
effectiveMinReplications = Math.max(chainMinNumber, resolvedUserMin);
88+
} else {
89+
effectiveMinReplications = resolvedUserMin;
90+
}
91+
92+
if (effectiveMinReplications === 0) {
93+
this.logger.error(
94+
`Effective minimum replications resolved to 0 for operationId: ${operationId}, blockchain: ${blockchain}. This should never happen.`,
95+
);
96+
}
97+
6598
const publisherNodePeerId = this.networkModuleManager.getPeerId().toB58String();
6699
await this.pendingStorageService.cacheDataset(
67100
operationId,
@@ -80,7 +113,7 @@ class PublishController extends BaseController {
80113
blockchain,
81114
operationId,
82115
storeType: LOCAL_STORE_TYPES.TRIPLE,
83-
minimumNumberOfNodeReplications,
116+
minimumNumberOfNodeReplications: effectiveMinReplications,
84117
},
85118
transactional: false,
86119
priority: COMMAND_PRIORITY.HIGHEST,

‎src/modules/blockchain/blockchain-module-manager.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,10 @@ class BlockchainModuleManager extends BaseModuleManager {
211211
return this.callImplementationFunction(blockchain, 'getMaximumStake');
212212
}
213213

214+
async getMinimumRequiredSignatures(blockchain) {
215+
return this.callImplementationFunction(blockchain, 'getMinimumRequiredSignatures');
216+
}
217+
214218
async getLatestBlock(blockchain) {
215219
return this.callImplementationFunction(blockchain, 'getLatestBlock');
216220
}

‎src/modules/blockchain/implementation/web3-service.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1024,6 +1024,15 @@ class Web3Service {
10241024
return Number(ethers.utils.formatEther(maximumStake));
10251025
}
10261026

1027+
async getMinimumRequiredSignatures() {
1028+
return this.callContractFunction(
1029+
this.contracts.ParametersStorage,
1030+
'minimumRequiredSignatures',
1031+
[],
1032+
CONTRACTS.PARAMETERS_STORAGE,
1033+
);
1034+
}
1035+
10271036
async getShardingTableHead() {
10281037
return this.callContractFunction(this.contracts.ShardingTableStorage, 'head', []);
10291038
}

‎src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -177,12 +177,24 @@ class OtBlazegraph extends OtTripleStore {
177177
}
178178

179179
async queryVoid(repository, query, timeout) {
180-
return axios.post(this.repositories[repository].sparqlEndpoint, query, {
181-
headers: {
182-
'Content-Type': 'application/sparql-update; charset=UTF-8',
183-
'X-BIGDATA-MAX-QUERY-MILLIS': timeout,
184-
},
185-
});
180+
try {
181+
return await axios.post(this.repositories[repository].sparqlEndpoint, query, {
182+
headers: {
183+
'Content-Type': 'application/sparql-update; charset=UTF-8',
184+
'X-BIGDATA-MAX-QUERY-MILLIS': timeout,
185+
},
186+
});
187+
} catch (error) {
188+
const status = error?.response?.status;
189+
const dataSnippet =
190+
typeof error?.response?.data === 'string' ? error.response.data.slice(0, 200) : '';
191+
this.logger.error(
192+
`[OtBlazegraph.queryVoid] Update failed for ${repository} (status: ${status}): ${
193+
error.message
194+
}${dataSnippet ? ` | data: ${dataSnippet}` : ''}`,
195+
);
196+
throw error;
197+
}
186198
}
187199

188200
async deleteRepository(repository) {

‎src/service/operation-id-service.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,33 @@ class OperationIdService {
150150
delete this.memoryCachedHandlersData[operationId];
151151
}
152152

153+
getOperationIdMemoryCacheSizeBytes() {
154+
let total = 0;
155+
for (const operationId in this.memoryCachedHandlersData) {
156+
const { data } = this.memoryCachedHandlersData[operationId];
157+
total += Buffer.from(JSON.stringify(data)).byteLength;
158+
}
159+
return total;
160+
}
161+
162+
async getOperationIdFileCacheSizeBytes() {
163+
const cacheFolderPath = this.fileService.getOperationIdCachePath();
164+
const cacheFolderExists = await this.fileService.pathExists(cacheFolderPath);
165+
if (!cacheFolderExists) return 0;
166+
167+
const fileList = await this.fileService.readDirectory(cacheFolderPath);
168+
const sizeResults = await Promise.allSettled(
169+
fileList.map((fileName) =>
170+
this.fileService
171+
.stat(path.join(cacheFolderPath, fileName))
172+
.then((stats) => stats.size),
173+
),
174+
);
175+
return sizeResults
176+
.filter((res) => res.status === 'fulfilled')
177+
.reduce((acc, res) => acc + res.value, 0);
178+
}
179+
153180
async removeExpiredOperationIdMemoryCache(expiredTimeout) {
154181
const now = Date.now();
155182
let deleted = 0;

0 commit comments

Comments
 (0)