Skip to content

Commit e8743fd

Browse files
dwernerdwerner
committed
cli,common,agent: auto-graft sync feature
1 parent 2679d30 commit e8743fd

File tree

11 files changed

+145
-41
lines changed

11 files changed

+145
-41
lines changed

‎docs/action-queue.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ enum ActionStatus {
191191
queued
192192
approved
193193
pending
194+
deploying
194195
success
195196
failed
196197
canceled

‎packages/indexer-agent/src/db/migrations/18-actions-expand-action-status-add-deploying.ts

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import { Logger } from '@graphprotocol/common-ts'
2+
import { DataTypes, QueryInterface } from 'sequelize'
3+
4+
interface MigrationContext {
5+
queryInterface: QueryInterface
6+
logger: Logger
7+
}
8+
9+
interface Context {
10+
context: MigrationContext
11+
}
12+
13+
export async function up({ context }: Context): Promise<void> {
14+
const { queryInterface, logger } = context
15+
16+
logger.debug(`Checking if 'Actions' table exists`)
17+
const tables = await queryInterface.showAllTables()
18+
if (!tables.includes('Actions')) {
19+
logger.info(`Actions table does not exist, migration not necessary`)
20+
return
21+
}
22+
23+
logger.debug(`Checking if 'Actions' table needs to be migrated`)
24+
const table = await queryInterface.describeTable('Actions')
25+
const statusColumn = table.status
26+
if (statusColumn) {
27+
logger.debug(`'status' column exists with type = ${statusColumn.type}`)
28+
logger.info(`Update 'status' column to support variant 'deploying' status`)
29+
await queryInterface.changeColumn('Actions', 'status', {
30+
type: DataTypes.ENUM(
31+
'queued',
32+
'approved',
33+
'deploying',
34+
'pending',
35+
'success',
36+
'failed',
37+
'canceled',
38+
),
39+
allowNull: false,
40+
})
41+
return
42+
}
43+
}
44+
45+
export async function down({ context }: Context): Promise<void> {
46+
const { logger } = context
47+
logger.info(
48+
`No 'down' migration needed since the 'up' migration simply added a new status 'deploying'`,
49+
)
50+
return
51+
}

‎packages/indexer-cli/src/commands/indexer/actions/delete.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ ${chalk.bold('graph indexer actions delete')} [options] [<actionID1> ...]
1313
${chalk.dim('Options:')}
1414
1515
-h, --help Show usage information
16-
--status queued|approved|pending|success|failed|canceled Filter by status
16+
--status queued|approved|deploying|pending|success|failed|canceled Filter by status
1717
-o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML
1818
`
1919

‎packages/indexer-cli/src/commands/indexer/actions/get.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,18 @@ module.exports = {
114114

115115
if (
116116
status &&
117-
!['queued', 'approved', 'pending', 'success', 'failed', 'canceled'].includes(
118-
status,
119-
)
117+
![
118+
'queued',
119+
'approved',
120+
'deploying',
121+
'pending',
122+
'success',
123+
'failed',
124+
'canceled',
125+
].includes(status)
120126
) {
121127
throw Error(
122-
`Invalid '--status' provided, must be one of ['queued', 'approved', 'pending', 'success', 'failed', 'canceled]`,
128+
`Invalid '--status' provided, must be one of ['queued', 'approved', 'deploying', 'pending', 'success', 'failed', 'canceled]`,
123129
)
124130
}
125131

‎packages/indexer-cli/src/commands/indexer/actions/update.ts

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ ${chalk.bold('graph indexer actions update')} [options] [<key1> <value1> ...]
2222
2323
${chalk.dim('Options:')}
2424
25-
-h, --help Show usage information
26-
--id <actionID> Filter by actionID
27-
--type allocate|unallocate|reallocate Filter by type
28-
--status queued|approved|pending|success|failed|canceled Filter by status
29-
--source <source> Filter by source
30-
--reason <reason> Filter by reason string
31-
-o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML
25+
-h, --help Show usage information
26+
--id <actionID> Filter by actionID
27+
--type allocate|unallocate|reallocate Filter by type
28+
--status queued|approved|deploying|pending|success|failed|canceled Filter by status
29+
--source <source> Filter by source
30+
--reason <reason> Filter by reason string
31+
-o, --output table|json|yaml Choose the output format: table (default), JSON, or YAML
3232
`
3333

3434
module.exports = {

‎packages/indexer-common/src/actions.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ export const validateActionInputs = async (
108108
)
109109
}
110110

111-
// Must have status QUEUED or APPROVED
111+
// Must have status QUEUED or APPROVED, or DEPLOYING
112112
if (
113113
[
114114
ActionStatus.FAILED,
@@ -156,7 +156,7 @@ export const validateActionInputs = async (
156156
export interface ActionFilter {
157157
id?: number | undefined
158158
type?: ActionType
159-
status?: ActionStatus
159+
status?: ActionStatus | ActionStatus[]
160160
source?: string
161161
reason?: string
162162
updatedAt?: WhereOperators
@@ -204,6 +204,7 @@ export enum ActionStatus {
204204
QUEUED = 'queued',
205205
APPROVED = 'approved',
206206
PENDING = 'pending',
207+
DEPLOYING = 'deploying',
207208
SUCCESS = 'success',
208209
FAILED = 'failed',
209210
CANCELED = 'canceled',

‎packages/indexer-common/src/graph-node.ts

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -832,6 +832,8 @@ export class GraphNode {
832832
// first ensure it's been deployed and is active, or already paused
833833
let deployed: SubgraphDeploymentAssignment[] = []
834834
let attempt = 0
835+
836+
// wait and poll for the assignment to be created.
835837
while (attempt < 5) {
836838
await waitForMs(3000)
837839
deployed = await this.subgraphDeploymentAssignmentsByDeploymentID(
@@ -852,6 +854,7 @@ export class GraphNode {
852854
})
853855
attempt += 1
854856
}
857+
855858
if (attempt >= 5) {
856859
this.logger.error(`Subgraph not deployed and active`, {
857860
subgraph: subgraphDeployment.ipfsHash,
@@ -900,17 +903,20 @@ export class GraphNode {
900903
}
901904

902905
// Is the graftBaseBlock within the range of the earliest and head of the chain?
903-
if (
904-
!deployed[0].paused &&
905-
chain.latestBlock &&
906-
chain.latestBlock.number >= blockHeight
907-
) {
908-
this.logger.debug(`Subgraph synced to block! Pausing as requirement is met.`, {
909-
subgraph: subgraphDeployment.ipfsHash,
910-
indexingStatus,
911-
})
912-
// pause the subgraph to prevent further indexing
913-
await this.pause(subgraphDeployment)
906+
if (chain.latestBlock && chain.latestBlock.number >= blockHeight) {
907+
if (!deployed[0].paused) {
908+
this.logger.debug(`Subgraph synced to block! Pausing as requirement is met.`, {
909+
subgraph: subgraphDeployment.ipfsHash,
910+
indexingStatus,
911+
})
912+
// pause the subgraph to prevent further indexing
913+
await this.pause(subgraphDeployment)
914+
} else {
915+
this.logger.debug(`Subgraph already paused and synced to block.`, {
916+
subgraph: subgraphDeployment.ipfsHash,
917+
indexingStatus,
918+
})
919+
}
914920
break
915921
}
916922

‎packages/indexer-common/src/indexer-management/actions.ts

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ export class ActionManager {
129129
let actions: Action[] = []
130130
try {
131131
actions = await ActionManager.fetchActions(this.models, null, {
132-
status: ActionStatus.APPROVED,
132+
status: [ActionStatus.APPROVED, ActionStatus.DEPLOYING],
133133
})
134134
logger.trace(`Fetched ${actions.length} approved actions`)
135135
} catch (err) {
@@ -299,17 +299,20 @@ export class ActionManager {
299299
{ isolationLevel: Transaction.ISOLATION_LEVELS.SERIALIZABLE },
300300
async (transaction) => {
301301
batchStartTime = Date.now()
302-
let approvedActions
302+
let approvedAndDeployingActions
303303
try {
304304
// Execute already approved actions in the order of type and priority.
305305
// Unallocate actions are prioritized to free up stake that can be used
306306
// in subsequent reallocate and allocate actions.
307307
// Reallocate actions are prioritized before allocate as they are for
308308
// existing syncing deployments with relatively smaller changes made.
309309
const actionTypePriority = ['unallocate', 'reallocate', 'allocate']
310-
approvedActions = (
310+
approvedAndDeployingActions = (
311311
await this.models.Action.findAll({
312-
where: { status: ActionStatus.APPROVED, protocolNetwork },
312+
where: {
313+
status: [ActionStatus.APPROVED, ActionStatus.DEPLOYING],
314+
protocolNetwork,
315+
},
313316
order: [['priority', 'ASC']],
314317
transaction,
315318
lock: transaction.LOCK.UPDATE,
@@ -324,25 +327,31 @@ export class ActionManager {
324327
transaction,
325328
})
326329
if (pendingActions.length > 0) {
327-
logger.warn(`${pendingActions} Actions found in PENDING state when execution began. Was there a crash? \
328-
These indicate that execution was interrupted and will need to be cleared manually.`)
330+
logger.warn(
331+
`${pendingActions} Actions found in PENDING state when execution began. Was there a crash?` +
332+
`These indicate that execution was interrupted while calling contracts, and will need to be cleared manually.`,
333+
)
329334
}
330335

331-
if (approvedActions.length === 0) {
336+
if (approvedAndDeployingActions.length === 0) {
332337
logger.debug('No approved actions were found for this network')
333338
return []
334339
}
335340
logger.debug(
336-
`Found ${approvedActions.length} approved actions for this network `,
337-
{ approvedActions },
341+
`Found ${approvedAndDeployingActions.length} approved actions for this network `,
342+
{ approvedActions: approvedAndDeployingActions },
338343
)
339344
} catch (error) {
340345
logger.error('Failed to query approved actions for network', { error })
341346
return []
342347
}
343-
// mark all approved actions as PENDING, this serves as a lock on other processing of them
344-
await this.markActions(approvedActions, transaction, ActionStatus.PENDING)
345-
return approvedActions
348+
// mark all approved actions as DEPLOYING, this serves as a lock on other processing of them
349+
await this.markActions(
350+
approvedAndDeployingActions,
351+
transaction,
352+
ActionStatus.DEPLOYING,
353+
)
354+
return approvedAndDeployingActions
346355
},
347356
)
348357

@@ -357,8 +366,28 @@ export class ActionManager {
357366

358367
let results
359368
try {
369+
// TODO: we should lift the batch execution (graph-node, then contracts) up to here so we can
370+
// mark the actions appropriately
371+
const onFinishedDeploying = async (validatedActions) => {
372+
// After we ensure that we have finished deploying new subgraphs (and possibly their dependencies) to graph-node,
373+
// we can mark the actions as PENDING.
374+
logger.debug('Finished deploying actions, marking as PENDING')
375+
this.models.Action.sequelize!.transaction(
376+
{ isolationLevel: Transaction.ISOLATION_LEVELS.SERIALIZABLE },
377+
async (transaction) => {
378+
return await this.markActions(
379+
validatedActions,
380+
transaction,
381+
ActionStatus.PENDING,
382+
)
383+
},
384+
)
385+
}
360386
// This will return all results if successful, if failed it will return the failed actions
361-
results = await allocationManager.executeBatch(prioritizedActions)
387+
results = await this.allocationManager.executeBatch(
388+
prioritizedActions,
389+
onFinishedDeploying,
390+
)
362391
logger.debug('Completed batch action execution', {
363392
results,
364393
endTimeMs: Date.now() - batchStartTime,

‎packages/indexer-common/src/indexer-management/allocations.ts

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,18 +105,24 @@ export class AllocationManager {
105105
private network: Network,
106106
) {}
107107

108-
async executeBatch(actions: Action[]): Promise<AllocationResult[]> {
108+
async executeBatch(
109+
actions: Action[],
110+
onFinishedDeploying: (actions: Action[]) => Promise<void>,
111+
): Promise<AllocationResult[]> {
109112
const logger = this.logger.child({ function: 'executeBatch' })
110113
logger.trace('Executing action batch', { actions })
111-
const result = await this.executeTransactions(actions)
114+
const result = await this.executeTransactions(actions, onFinishedDeploying)
112115
if (Array.isArray(result)) {
113116
logger.trace('Execute batch transaction failed', { actionBatchResult: result })
114117
return result as ActionFailure[]
115118
}
116119
return await this.confirmTransactions(result, actions)
117120
}
118121

119-
private async executeTransactions(actions: Action[]): Promise<TransactionResult> {
122+
private async executeTransactions(
123+
actions: Action[],
124+
onFinishedDeploying: (actions: Action[]) => Promise<void>,
125+
): Promise<TransactionResult> {
120126
const logger = this.logger.child({ function: 'executeTransactions' })
121127
logger.trace('Begin executing transactions', { actions })
122128
if (actions.length < 1) {
@@ -127,6 +133,7 @@ export class AllocationManager {
127133
logger.trace('Validated actions', { validatedActions })
128134

129135
await this.deployBeforeAllocating(logger, validatedActions)
136+
await onFinishedDeploying(validatedActions)
130137

131138
const populateTransactionsResults = await this.prepareTransactions(validatedActions)
132139

@@ -462,6 +469,7 @@ export class AllocationManager {
462469
if (receipt === 'paused' || receipt === 'unauthorized') {
463470
throw indexerError(
464471
IndexerErrorCode.IE062,
472+
465473
`Allocation not created. ${
466474
receipt === 'paused' ? 'Network paused' : 'Operator not authorized'
467475
}`,

‎packages/indexer-common/src/indexer-management/client.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ const SCHEMA_SDL = gql`
104104
enum ActionStatus {
105105
queued
106106
approved
107+
deploying
107108
pending
108109
success
109110
failed

0 commit comments

Comments
 (0)