From 6e192b29f1ac27404c37bd5541ad8fec35d34ac8 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 23 Jul 2025 13:36:14 +0200 Subject: [PATCH 01/13] Bump gas price on nonce issue --- src/constants/constants.js | 3 + .../blockchain/implementation/web3-service.js | 85 ++++++++++++++----- 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 5900aec02..6e532678f 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -712,6 +712,9 @@ export const EXPECTED_TRANSACTION_ERRORS = { NODE_NOT_AWARDED: 'NodeNotAwarded', WRONG_MERKLE_PROOF: 'WrongMerkleProof', NO_MINTED_ASSETS: 'NoMintedAssets', + NONCE_TOO_LOW: 'nonce too low', + REPLACEMENT_UNDERPRICED: 'replacement transaction underpriced', + ALREADY_KNOWN: 'already known', }; /** diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 879983e8d..4b1149cce 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -23,6 +23,7 @@ import { TRANSACTION_PRIORITY, CONTRACT_FUNCTION_GAS_LIMIT_INCREASE_FACTORS, ABIs, + EXPECTED_TRANSACTION_ERRORS, } from '../../../constants/constants.js'; import Web3ServiceValidator from './web3-service-validator.js'; @@ -561,8 +562,10 @@ class Web3Service { operationalWallet, ) { let result; - const gasPrice = predefinedGasPrice ?? (await this.getGasPrice()); + let gasPrice = predefinedGasPrice ?? (await this.getGasPrice()); let gasLimit; + let retryCount = 0; + const maxRetries = 3; try { /* eslint-disable no-await-in-loop */ @@ -577,33 +580,69 @@ class Web3Service { gasLimit = gasLimit.mul(gasLimitMultiplier * 100).div(100); - this.logger.debug( - `Sending signed transaction ${functionName} to the blockchain ${this.getBlockchainId()}` + - ` with gas limit: ${gasLimit.toString()} and gasPrice ${gasPrice.toString()}. ` + - `Transaction queue length: ${this.getTotalTransactionQueueLength()}. Wallet used: ${ - operationalWallet.address - }`, - ); + while (retryCount < maxRetries) { + try { + this.logger.debug( + `Sending signed transaction ${functionName} to the blockchain ${this.getBlockchainId()}` + + ` with gas limit: ${gasLimit.toString()} and gasPrice ${gasPrice.toString()}. ` + + `Transaction queue length: ${this.getTotalTransactionQueueLength()}. Wallet used: ${ + operationalWallet.address + }${retryCount > 0 ? ` (retry ${retryCount})` : ''}`, + ); - const tx = await contractInstance.connect(operationalWallet)[functionName](...args, { - gasPrice, - gasLimit, - }); + const tx = await contractInstance + .connect(operationalWallet) + [functionName](...args, { + gasPrice, + gasLimit, + }); - try { - result = await this.provider.waitForTransaction( - tx.hash, - TRANSACTION_CONFIRMATIONS, - TRANSACTION_POLLING_TIMEOUT_MILLIS, - ); + try { + result = await this.provider.waitForTransaction( + tx.hash, + TRANSACTION_CONFIRMATIONS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, + ); - if (result.status === 0) { - await this.provider.call(tx, tx.blockNumber); + if (result.status === 0) { + await this.provider.call(tx, tx.blockNumber); + } + } catch (error) { + this._decodeWaitForTxError(contractInstance, functionName, error, args); + } + return result; + } catch (error) { + const errorMessage = error.message.toLowerCase(); + + // Check for nonce-related errors + if ( + errorMessage.includes( + EXPECTED_TRANSACTION_ERRORS.NONCE_TOO_LOW.toLowerCase(), + ) || + errorMessage.includes( + EXPECTED_TRANSACTION_ERRORS.REPLACEMENT_UNDERPRICED.toLowerCase(), + ) || + errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ALREADY_KNOWN.toLowerCase()) + ) { + retryCount += 1; + if (retryCount < maxRetries) { + // Increase gas price by 20% for nonce errors + gasPrice = Math.ceil(gasPrice * 1.2); + this.logger.warn( + `Nonce error detected for ${functionName}. Retrying with increased gas price: ${gasPrice} (retry ${retryCount}/${maxRetries})`, + ); + continue; + } else { + this.logger.error( + `Max retries (${maxRetries}) reached for nonce error in ${functionName}. Final gas price: ${gasPrice}`, + ); + } + } + + // If it's not a nonce error or we've exhausted retries, re-throw the error + throw error; } - } catch (error) { - this._decodeWaitForTxError(contractInstance, functionName, error, args); } - return result; } _decodeEstimateGasError(contractInstance, functionName, error, args) { From bbbd9c154f8b1948d724a9c223e874d5c6a5f5fc Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 23 Jul 2025 13:46:31 +0200 Subject: [PATCH 02/13] Fix setCompletedAndFinalizedRandomSamplingChallengeRecord parameters --- src/service/proofing-service.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/service/proofing-service.js b/src/service/proofing-service.js index b8cb6906a..5e01bd96f 100644 --- a/src/service/proofing-service.js +++ b/src/service/proofing-service.js @@ -172,7 +172,9 @@ class ProofingService { latestChallenge.sentSuccessfully = false; latestChallenge.finalized = false; await this.repositoryModuleManager.setCompletedAndFinalizedRandomSamplingChallengeRecord( - latestChallenge, + latestChallenge.id, + latestChallenge.sentSuccessfully, + latestChallenge.finalized, ); await this.prepareAndSendProof(blockchainId, identityId); } From 132ba712af91d809adf59c6d09d0a3d769bb45ba Mon Sep 17 00:00:00 2001 From: Lexpeartha Date: Wed, 23 Jul 2025 16:03:57 +0200 Subject: [PATCH 03/13] chore: add time logs to batch get protocol --- ...v1-0-0-handle-batch-get-request-command.js | 11 ++++++++ .../protocols/get/sender/batch-get-command.js | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js index 46d510de5..0d1ab00a0 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js @@ -30,6 +30,7 @@ class HandleBatchGetRequestCommand extends HandleProtocolMessageCommand { const { operationId, blockchain, includeMetadata } = commandData; let { uals, tokenIds } = commandData; + console.time(`HandleBatchGetRequestCommand [PREPARE]: ${operationId} ${uals.length}`); await this.operationIdService.updateOperationIdStatus( operationId, blockchain, @@ -57,6 +58,10 @@ class HandleBatchGetRequestCommand extends HandleProtocolMessageCommand { } } + console.timeEnd(`HandleBatchGetRequestCommand [PREPARE]: ${operationId} ${uals.length}`); + + console.time(`HandleBatchGetRequestCommand [PROCESSING]: ${operationId} ${uals.length}`); + const assertionPromise = this.tripleStoreService.getAssertionsInBatch( TRIPLE_STORE_REPOSITORY.DKG, uals, @@ -81,6 +86,10 @@ class HandleBatchGetRequestCommand extends HandleProtocolMessageCommand { ...(includeMetadata && metadata && { metadata }), }; + console.timeEnd(`HandleBatchGetRequestCommand [PROCESSING]: ${operationId} ${uals.length}`); + + console.time(`HandleBatchGetRequestCommand [RESPONSE]: ${operationId} ${uals.length}`); + if (assertions?.length) { await this.operationIdService.updateOperationIdStatus( operationId, @@ -89,6 +98,8 @@ class HandleBatchGetRequestCommand extends HandleProtocolMessageCommand { ); } + console.timeEnd(`HandleBatchGetRequestCommand [RESPONSE]: ${operationId} ${uals.length}`); + return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: responseData }; } diff --git a/src/commands/protocols/get/sender/batch-get-command.js b/src/commands/protocols/get/sender/batch-get-command.js index 9dee3cd65..f2cf41298 100644 --- a/src/commands/protocols/get/sender/batch-get-command.js +++ b/src/commands/protocols/get/sender/batch-get-command.js @@ -68,6 +68,8 @@ class BatchGetCommand extends Command { paranetNodesAccessPolicy, } = command.data; + console.time(`BatchGetCommand [PREPARE]: ${operationId} ${uals.length}`); + await this.operationIdService.updateOperationIdStatus( operationId, blockchain, @@ -88,6 +90,8 @@ class BatchGetCommand extends Command { const { isValid, errorMessage } = await this.validateUALs(operationId, blockchain, uals); + console.timeEnd(`BatchGetCommand [PREPARE]: ${operationId} ${uals.length}`); + if (!isValid) { await this.handleError( operationId, @@ -98,6 +102,8 @@ class BatchGetCommand extends Command { return Command.empty(); } + console.time(`BatchGetCommand [NETWORK]: ${operationId} ${uals.length}`); + const currentPeerId = this.networkModuleManager.getPeerId().toB58String(); // let paranetId; const repository = TRIPLE_STORE_REPOSITORIES.DKG; @@ -127,6 +133,10 @@ class BatchGetCommand extends Command { OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_LOCAL_START, ); + console.timeEnd(`BatchGetCommand [NETWORK]: ${operationId} ${uals.length}`); + + console.time(`BatchGetCommand [TOKEN_IDS]: ${operationId} ${uals.length}`); + const tokenIds = {}; const tokenIdPromises = uals.map(async (ual) => { @@ -149,6 +159,10 @@ class BatchGetCommand extends Command { await Promise.all(tokenIdPromises); + console.timeEnd(`BatchGetCommand [TOKEN_IDS]: ${operationId} ${uals.length}`); + + console.time(`BatchGetCommand [UALS]: ${operationId} ${uals.length}`); + const promises = []; const assertionPromise = this.tripleStoreService.getAssertionsInBatch( TRIPLE_STORE_REPOSITORY.DKG, @@ -178,6 +192,10 @@ class BatchGetCommand extends Command { (ual) => !localGetResultValid[ual], ); + console.timeEnd(`BatchGetCommand [UALS]: ${operationId} ${uals.length}`); + + console.time(`BatchGetCommand [LOCAL]: ${operationId} ${uals.length}`); + ualPresentLocally.forEach((ual) => { finalResult.local.push(ual); delete tokenIds[ual]; @@ -198,6 +216,8 @@ class BatchGetCommand extends Command { return Command.empty(); } + console.timeEnd(`BatchGetCommand [LOCAL]: ${operationId} ${uals.length}`); + await this.operationIdService.updateOperationIdStatus( operationId, blockchain, @@ -210,6 +230,8 @@ class BatchGetCommand extends Command { OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_FIND_SHARD_START, ); + console.time(`BatchGetCommand [FIND_SHARD]: ${operationId} ${uals.length}`); + let nodesInfo = []; // if (paranetNodesAccessPolicy === PARANET_ACCESS_POLICY.PERMISSIONED) { // const onChainNodes = await this.blockchainModuleManager.getPermissionedNodes( @@ -247,12 +269,16 @@ class BatchGetCommand extends Command { return Command.empty(); } + console.timeEnd(`BatchGetCommand [FIND_SHARD]: ${operationId} ${uals.length}`); + await this.operationIdService.updateOperationIdStatus( operationId, blockchain, OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_FIND_SHARD_END, ); + console.time(`BatchGetCommand [MAIN]: ${operationId} ${uals.length}`); + let index = 0; let commandCompleted = false; @@ -357,6 +383,8 @@ class BatchGetCommand extends Command { ); } + console.timeEnd(`BatchGetCommand [MAIN]: ${operationId} ${uals.length}`); + return Command.empty(); } From ed3c9aa3a9c54de8cae1f5196acfddfabede1020 Mon Sep 17 00:00:00 2001 From: Lexpeartha Date: Wed, 23 Jul 2025 16:48:56 +0200 Subject: [PATCH 04/13] chore: integrate review feedback --- .../protocols/get/sender/batch-get-command.js | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/commands/protocols/get/sender/batch-get-command.js b/src/commands/protocols/get/sender/batch-get-command.js index f2cf41298..1926e7051 100644 --- a/src/commands/protocols/get/sender/batch-get-command.js +++ b/src/commands/protocols/get/sender/batch-get-command.js @@ -161,7 +161,7 @@ class BatchGetCommand extends Command { console.timeEnd(`BatchGetCommand [TOKEN_IDS]: ${operationId} ${uals.length}`); - console.time(`BatchGetCommand [UALS]: ${operationId} ${uals.length}`); + console.time(`BatchGetCommand [LOCAL_BATCH_GET]: ${operationId} ${uals.length}`); const promises = []; const assertionPromise = this.tripleStoreService.getAssertionsInBatch( @@ -192,7 +192,7 @@ class BatchGetCommand extends Command { (ual) => !localGetResultValid[ual], ); - console.timeEnd(`BatchGetCommand [UALS]: ${operationId} ${uals.length}`); + console.timeEnd(`BatchGetCommand [LOCAL_BATCH_GET]: ${operationId} ${uals.length}`); console.time(`BatchGetCommand [LOCAL]: ${operationId} ${uals.length}`); @@ -277,7 +277,7 @@ class BatchGetCommand extends Command { OPERATION_ID_STATUS.BATCH_GET.BATCH_GET_FIND_SHARD_END, ); - console.time(`BatchGetCommand [MAIN]: ${operationId} ${uals.length}`); + console.time(`BatchGetCommand [NETWORK]: ${operationId} ${uals.length}`); let index = 0; let commandCompleted = false; @@ -306,12 +306,21 @@ class BatchGetCommand extends Command { // eslint-disable-next-line no-loop-func const messagePromises = batch.map(async (node) => { try { + console.time( + `BatchGetCommand [NETWORK_SEND_MESSAGE]: ${operationId} ${uals.length} ${node.id}`, + ); const result = await this.sendMessage(node, operationId, message); + console.timeEnd( + `BatchGetCommand [NETWORK_SEND_MESSAGE]: ${operationId} ${uals.length} ${node.id}`, + ); if (commandCompleted || !result.success) { return; } + console.time( + `BatchGetCommand [NETWORK_VALIDATE_RESPONSE]: ${operationId} ${uals.length} ${node.id}`, + ); const validationResult = await this.validateBatchResponse( result.responseData.assertions, blockchain, @@ -320,6 +329,9 @@ class BatchGetCommand extends Command { finalResult, [OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED], ); + console.timeEnd( + `BatchGetCommand [NETWORK_VALIDATE_RESPONSE]: ${operationId} ${uals.length} ${node.id}`, + ); if (commandCompleted) { return; @@ -338,12 +350,18 @@ class BatchGetCommand extends Command { if (hasReachedThreshold() && !commandCompleted) { commandCompleted = true; + console.time( + `BatchGetCommand [NETWORK_MARK_AS_COMPLETED]: ${operationId} ${uals.length} ${node.id}`, + ); await this.operationService.markOperationAsCompleted( operationId, blockchain, finalResult, [OPERATION_ID_STATUS.GET.GET_END, OPERATION_ID_STATUS.COMPLETED], ); + console.timeEnd( + `BatchGetCommand [NETWORK_MARK_AS_COMPLETED]: ${operationId} ${uals.length} ${node.id}`, + ); } } catch (err) { this.logger.warn(`Node ${node.id} failed: ${err.message}`); @@ -383,7 +401,7 @@ class BatchGetCommand extends Command { ); } - console.timeEnd(`BatchGetCommand [MAIN]: ${operationId} ${uals.length}`); + console.timeEnd(`BatchGetCommand [NETWORK]: ${operationId} ${uals.length}`); return Command.empty(); } From 482854e74b968f2464baace468ba0bbbeb4a5c2f Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 23 Jul 2025 17:45:38 +0200 Subject: [PATCH 05/13] remove unneeded toNumber --- .../random-sampling-challenge-repository.js | 2 +- src/service/proofing-service.js | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/modules/repository/implementation/sequelize/repositories/random-sampling-challenge-repository.js b/src/modules/repository/implementation/sequelize/repositories/random-sampling-challenge-repository.js index bfe2b025f..305210a08 100644 --- a/src/modules/repository/implementation/sequelize/repositories/random-sampling-challenge-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/random-sampling-challenge-repository.js @@ -41,7 +41,7 @@ class RandomSamplingChallengeRepository { where: { blockchainId, }, - order: [['createdAt', 'DESC']], // Should this be camel case ? + order: [['createdAt', 'DESC']], }); } diff --git a/src/service/proofing-service.js b/src/service/proofing-service.js index b8cb6906a..aa46aed34 100644 --- a/src/service/proofing-service.js +++ b/src/service/proofing-service.js @@ -153,12 +153,12 @@ class ProofingService { 'PROOF_CHALANGE_FINALIZED', this.generateOperationId( blockchainId, - latestChallenge.epoch.toNumber(), - latestChallenge.activeProofPeriodStartBlock.toNumber(), + latestChallenge.epoch, + latestChallenge.activeProofPeriodStartBlock, ), blockchainId, - latestChallenge.epoch.toNumber(), - latestChallenge.activeProofPeriodStartBlock.toNumber(), + latestChallenge.epoch, + latestChallenge.activeProofPeriodStartBlock, ); } else { this.logger.info( @@ -190,12 +190,12 @@ class ProofingService { 'PROOF_ASSERTION_FETCHED', this.generateOperationId( blockchainId, - latestChallenge.epoch.toNumber(), - latestChallenge.activeProofPeriodStartBlock.toNumber(), + latestChallenge.epoch, + latestChallenge.activeProofPeriodStartBlock, ), blockchainId, - latestChallenge.epoch.toNumber(), - latestChallenge.activeProofPeriodStartBlock.toNumber(), + latestChallenge.epoch, + latestChallenge.activeProofPeriodStartBlock, ); if (data.public.length === 0) { From e7786c3b4f18c3f586b1b53b1c651fafe4bc8ee3 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 23 Jul 2025 18:30:07 +0200 Subject: [PATCH 06/13] Fix in another event --- src/service/proofing-service.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/service/proofing-service.js b/src/service/proofing-service.js index aa46aed34..88d2831dc 100644 --- a/src/service/proofing-service.js +++ b/src/service/proofing-service.js @@ -246,12 +246,12 @@ class ProofingService { 'PROOF_ASSERTION_FETCHED', this.generateOperationId( blockchainId, - newChallenge.epoch.toNumber(), - newChallenge.activeProofPeriodStartBlock.toNumber(), + newChallenge.epoch, + newChallenge.activeProofPeriodStartBlock, ), blockchainId, - newChallenge.epoch.toNumber(), - newChallenge.activeProofPeriodStartBlock.toNumber(), + newChallenge.epoch, + newChallenge.activeProofPeriodStartBlock, ); if (data.public.length === 0) { From 8ee55cd96611e6b417b8ce5938c1d8b4d947ea35 Mon Sep 17 00:00:00 2001 From: Lexpeartha Date: Thu, 24 Jul 2025 13:15:56 +0200 Subject: [PATCH 07/13] chore: add even more logs to validate triple store --- .../protocols/get/sender/batch-get-command.js | 9 ++- src/service/triple-store-service.js | 58 ++++++++++++++++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/commands/protocols/get/sender/batch-get-command.js b/src/commands/protocols/get/sender/batch-get-command.js index 1926e7051..4b0869eee 100644 --- a/src/commands/protocols/get/sender/batch-get-command.js +++ b/src/commands/protocols/get/sender/batch-get-command.js @@ -169,6 +169,7 @@ class BatchGetCommand extends Command { uals, tokenIds, TRIPLES_VISIBILITY.PUBLIC, + operationId, ); promises.push(assertionPromise); @@ -176,6 +177,10 @@ class BatchGetCommand extends Command { const finalResult = { local: [], remote: {}, metadata: {} }; + console.timeEnd(`BatchGetCommand [LOCAL_BATCH_GET]: ${operationId} ${uals.length}`); + + console.time(`BatchGetCommand [LOCAL_BATCH_GET_VALIDATE]: ${operationId} ${uals.length}`); + const localGetResultValid = await this.validateBatchResponse( batchAssertions, blockchain, @@ -192,7 +197,9 @@ class BatchGetCommand extends Command { (ual) => !localGetResultValid[ual], ); - console.timeEnd(`BatchGetCommand [LOCAL_BATCH_GET]: ${operationId} ${uals.length}`); + console.timeEnd( + `BatchGetCommand [LOCAL_BATCH_GET_VALIDATE]: ${operationId} ${uals.length}`, + ); console.time(`BatchGetCommand [LOCAL]: ${operationId} ${uals.length}`); diff --git a/src/service/triple-store-service.js b/src/service/triple-store-service.js index 29ba03758..5a589bdbb 100644 --- a/src/service/triple-store-service.js +++ b/src/service/triple-store-service.js @@ -414,13 +414,29 @@ class TripleStoreService { migrationFlag, visibility = TRIPLES_VISIBILITY.PUBLIC, repository = TRIPLE_STORE_REPOSITORY.DKG, + operationId = undefined, ) { // TODO: Use stateId let ual = `did:dkg:${blockchain}/${contract}/${knowledgeCollectionId}`; + // Performance instrumentation (enable only if operationId is supplied) + const logTime = operationId !== undefined; + const startTimer = (label) => { + if (logTime) console.time(label); + }; + const endTimer = (label) => { + if (logTime) console.timeEnd(label); + }; + + const totalLabel = `[TripleStoreService.getAssertion TOTAL] ${operationId} ${ual}`; + startTimer(totalLabel); + let nquads = {}; if (typeof knowledgeAssetId === 'number') { ual = `${ual}/${knowledgeAssetId}`; + const singleLabel = `[TripleStoreService.getAssertion SINGLE] ${operationId} ${ual}`; + startTimer(singleLabel); + this.logger.debug(`Getting Assertion with the UAL: ${ual}.`); nquads = await this.tripleStoreModuleManager.getKnowledgeAssetNamedGraph( this.repositoryImplementations[repository], @@ -430,9 +446,14 @@ class TripleStoreService { visibility, this.config.modules.tripleStore.timeout.get, ); + + endTimer(singleLabel); } else { this.logger.debug(`Getting Assertion with the UAL: ${ual}.`); + const existsLabel = `[TripleStoreService.getAssertion EXISTS_CHECK] ${operationId} ${ual}`; + startTimer(existsLabel); + // first check if the knowledge collection exists in triple store using ASK const firstKAInCollection = `${ual}/${tokenIds.startTokenId}/${TRIPLES_VISIBILITY.PUBLIC}`; const lastKAInCollection = `${ual}/${tokenIds.endTokenId}/${TRIPLES_VISIBILITY.PUBLIC}`; @@ -449,14 +470,21 @@ class TripleStoreService { const [firstKAResult, lastKAResult] = await Promise.all([firstKAExists, lastKAExists]); + endTimer(existsLabel); + if (!(firstKAResult && lastKAResult)) { this.logger.warn( `Knowledge Collection with the UAL: ${ual} does not exist in the Triple Store's ${repository} repository.`, ); + endTimer(totalLabel); return { public: [], private: [] }; } // tokenIds are used to construct named graphs // do pagination through tokenIds + + const collectionLabel = `[TripleStoreService.getAssertion COLLECTION] ${operationId} ${ual}`; + startTimer(collectionLabel); + if (visibility === TRIPLES_VISIBILITY.PUBLIC || visibility === TRIPLES_VISIBILITY.ALL) { nquads.public = []; } @@ -492,6 +520,8 @@ class TripleStoreService { ); } } + + endTimer(collectionLabel); } const numberOfnquads = (nquads?.public?.length ?? 0) + (nquads?.private?.length ?? 0); @@ -508,15 +538,34 @@ class TripleStoreService { ); } + endTimer(totalLabel); return nquads; } - async getAssertionsInBatch(repository, uals, ualTokenIds, visibility = 'public') { + async getAssertionsInBatch( + repository, + uals, + ualTokenIds, + visibility = 'public', + operationId = undefined, + ) { + // Conditional performance logging + const logTime = operationId !== undefined; + const startTimer = (label) => { + if (logTime) console.time(label); + }; + const endTimer = (label) => { + if (logTime) console.timeEnd(label); + }; + + const totalLabel = `[TripleStoreService.getAssertionsInBatch TOTAL] ${operationId} ${uals.length}`; + startTimer(totalLabel); + const results = await Promise.all( uals.map(async (ual) => { const { blockchain, contract, knowledgeCollectionId } = this.ualService.resolveUAL(ual); - const nquads = await this.getAssertion( + return this.getAssertion( blockchain, contract, knowledgeCollectionId, @@ -524,15 +573,18 @@ class TripleStoreService { ualTokenIds[ual], false, visibility, + repository, + operationId, ); - return nquads; }), ); + const result = {}; for (const [index, ual] of uals.entries()) { result[ual] = results[index]; } + endTimer(totalLabel); return result; } From c72f59d30b534e445aabedc6e96772f5b50252bd Mon Sep 17 00:00:00 2001 From: Lexpeartha Date: Thu, 24 Jul 2025 14:21:52 +0200 Subject: [PATCH 08/13] chore: add missing parameter --- .../receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js index 0d1ab00a0..d7ab663f7 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-batch-get-request-command.js @@ -67,6 +67,7 @@ class HandleBatchGetRequestCommand extends HandleProtocolMessageCommand { uals, tokenIds, TRIPLES_VISIBILITY.PUBLIC, + operationId, ); promises.push(assertionPromise); From f801ddda71f09460ec849fc4ee0a56ccf1cb014f Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 24 Jul 2025 17:49:54 +0200 Subject: [PATCH 09/13] Add ASK timeout to blazgraph query --- .../implementation/ot-blazegraph/ot-blazegraph.js | 5 +++++ src/modules/triple-store/implementation/ot-triple-store.js | 4 ++-- src/service/triple-store-service.js | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js b/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js index 648e543e8..7650f85be 100644 --- a/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js +++ b/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js @@ -83,6 +83,11 @@ class OtBlazegraph extends OtTripleStore { return result ? JSON.parse(result) : []; } + async ask(repository, query, timeout = 10000) { + const result = await this._executeQuery(repository, query, MEDIA_TYPES.JSON, timeout); + return result ? JSON.parse(result).boolean : false; + } + async _executeQuery(repository, query, mediaType, timeout) { const result = await axios.post(this.repositories[repository].sparqlEndpoint, query, { headers: { diff --git a/src/modules/triple-store/implementation/ot-triple-store.js b/src/modules/triple-store/implementation/ot-triple-store.js index dcbd41288..64d8ed952 100644 --- a/src/modules/triple-store/implementation/ot-triple-store.js +++ b/src/modules/triple-store/implementation/ot-triple-store.js @@ -546,7 +546,7 @@ class OtTripleStore { return result.data; } - async checkIfKnowledgeAssetExists(repository, kaUAL) { + async checkIfKnowledgeAssetExists(repository, kaUAL, timeout = 10000) { const query = ` ASK { GRAPH <${kaUAL}> { @@ -554,7 +554,7 @@ class OtTripleStore { } }`; - return this.ask(repository, query); + return this.ask(repository, query, timeout); } async getKnowledgeCollectionNamedGraphs( diff --git a/src/service/triple-store-service.js b/src/service/triple-store-service.js index 5a589bdbb..cc51d9c12 100644 --- a/src/service/triple-store-service.js +++ b/src/service/triple-store-service.js @@ -461,11 +461,13 @@ class TripleStoreService { this.repositoryImplementations[repository], repository, firstKAInCollection, + this.config.modules.tripleStore.timeout.ask, ); const lastKAExists = this.tripleStoreModuleManager.checkIfKnowledgeAssetExists( this.repositoryImplementations[repository], repository, lastKAInCollection, + this.config.modules.tripleStore.timeout.ask, ); const [firstKAResult, lastKAResult] = await Promise.all([firstKAExists, lastKAExists]); From cc5c8544263a18de30705823ec203902e1aa8380 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 24 Jul 2025 17:53:06 +0200 Subject: [PATCH 10/13] Add default config --- config/config.json | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/config/config.json b/config/config.json index 4c061076d..f3c9dec3f 100644 --- a/config/config.json +++ b/config/config.json @@ -95,7 +95,8 @@ "query": 60000, "get": 10000, "batchGet": 10000, - "insert": 300000 + "insert": 300000, + "ask": 10000 }, "implementation": { "ot-blazegraph": { @@ -349,7 +350,8 @@ "query": 60000, "get": 10000, "batchGet": 10000, - "insert": 300000 + "insert": 300000, + "ask": 10000 }, "implementation": { "ot-blazegraph": { @@ -567,7 +569,8 @@ "query": 60000, "get": 10000, "batchGet": 10000, - "insert": 300000 + "insert": 300000, + "ask": 10000 }, "implementation": { "ot-blazegraph": { @@ -776,7 +779,8 @@ "query": 60000, "get": 10000, "batchGet": 10000, - "insert": 300000 + "insert": 300000, + "ask": 10000 }, "implementation": { "ot-blazegraph": { @@ -991,7 +995,8 @@ "query": 60000, "get": 10000, "batchGet": 10000, - "insert": 300000 + "insert": 300000, + "ask": 10000 }, "implementation": { "ot-blazegraph": { From d3becd3661ae2305d58488f0d023e8763c48301a Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 24 Jul 2025 18:10:13 +0200 Subject: [PATCH 11/13] Add ASK json handling --- .../ot-blazegraph/ot-blazegraph.js | 87 ++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js b/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js index 7650f85be..42764bb6d 100644 --- a/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js +++ b/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js @@ -98,48 +98,55 @@ class OtBlazegraph extends OtTripleStore { }); let response; if (mediaType === MEDIA_TYPES.JSON) { - const { bindings } = result.data.results; - - let output = '[\n'; - - bindings.forEach((binding, bindingIndex) => { - let string = ' {\n'; - - const keys = Object.keys(binding); - - keys.forEach((key, index) => { - let value = ''; - const entry = binding[key]; - - if (entry.datatype) { - // e.g., "\"6900000\"^^http://www.w3.org/2001/XMLSchema#integer" - const literal = `"${entry.value}"^^${entry.datatype}`; - value = JSON.stringify(literal); - } else if (entry['xml:lang']) { - // e.g., "\"text here\"@en" - const literal = `"${entry.value}"@${entry['xml:lang']}`; - value = JSON.stringify(literal); - } else if (entry.type === 'uri') { - // URIs should be escaped and quoted directly - value = JSON.stringify(entry.value); - } else { - // For plain literals, wrap in quotes and stringify - const literal = `"${entry.value}"`; - value = JSON.stringify(literal); - } - - const isLast = index === keys.length - 1; - string += ` "${key}": ${value}${isLast ? '' : ','}\n`; + // Check if this is an ASK query by looking for the boolean property + if (result.data.boolean !== undefined) { + // This is an ASK query response + response = JSON.stringify(result.data); + } else { + // This is a SELECT query response + const { bindings } = result.data.results; + + let output = '[\n'; + + bindings.forEach((binding, bindingIndex) => { + let string = ' {\n'; + + const keys = Object.keys(binding); + + keys.forEach((key, index) => { + let value = ''; + const entry = binding[key]; + + if (entry.datatype) { + // e.g., "\"6900000\"^^http://www.w3.org/2001/XMLSchema#integer" + const literal = `"${entry.value}"^^${entry.datatype}`; + value = JSON.stringify(literal); + } else if (entry['xml:lang']) { + // e.g., "\"text here\"@en" + const literal = `"${entry.value}"@${entry['xml:lang']}`; + value = JSON.stringify(literal); + } else if (entry.type === 'uri') { + // URIs should be escaped and quoted directly + value = JSON.stringify(entry.value); + } else { + // For plain literals, wrap in quotes and stringify + const literal = `"${entry.value}"`; + value = JSON.stringify(literal); + } + + const isLast = index === keys.length - 1; + string += ` "${key}": ${value}${isLast ? '' : ','}\n`; + }); + + const isLastBinding = bindingIndex === bindings.length - 1; + string += ` }${isLastBinding ? '\n' : ',\n'}`; + + output += string; }); - const isLastBinding = bindingIndex === bindings.length - 1; - string += ` }${isLastBinding ? '\n' : ',\n'}`; - - output += string; - }); - - output += ']'; - response = output; + output += ']'; + response = output; + } } else { response = result.data; } From fc87600c3cc17667ef9cd8ffe2940b8624a0e14b Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 25 Jul 2025 12:26:22 +0200 Subject: [PATCH 12/13] Add try catch in checkIfKnowledgeAssetExists --- .../triple-store/implementation/ot-triple-store.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/modules/triple-store/implementation/ot-triple-store.js b/src/modules/triple-store/implementation/ot-triple-store.js index 64d8ed952..81c27f735 100644 --- a/src/modules/triple-store/implementation/ot-triple-store.js +++ b/src/modules/triple-store/implementation/ot-triple-store.js @@ -553,8 +553,12 @@ class OtTripleStore { ?s ?p ?o } }`; - - return this.ask(repository, query, timeout); + try { + return this.ask(repository, query, timeout); + } catch (error) { + this.logger.error(`Error checking if knowledge asset exists: ${error}`); + return false; + } } async getKnowledgeCollectionNamedGraphs( From 55605b6d3a79684f7ff889e58b489e5f4df65994 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 25 Jul 2025 15:01:48 +0200 Subject: [PATCH 13/13] version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f7759e6a3..dee54ae10 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "8.1.1-rc.7", + "version": "8.1.1-rc.8", "description": "OTNode V8", "main": "index.js", "type": "module",