all: address bugs in auto-graft from v0.24.2 release

dwerner · dwerner · commit d655f60c34f6 · 2025-08-19T11:20:23.000-07:00
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -131,7 +131,7 @@ Tests require:
    ```
 
 ## Current State
-- Version: v0.24.2
+- Version: v0.24.3
 - Active branch: release-v0.24
 - Main branch: main
 - Uncommitted changes in docs/ and config/
diff --git a/docs/auto-graft.md b/docs/auto-graft.md
@@ -2,20 +2,30 @@
 
 ## Quick Start
 
-Auto-graft automatically handles subgraph dependencies when deploying grafted subgraphs. It requires an IPFS endpoint to fetch dependency manifests.
+Auto-graft automatically handles subgraph dependencies when deploying grafted subgraphs. It requires an IPFS endpoint to fetch dependency manifests and must be explicitly enabled.
 
 ### Prerequisites
 
-**Required Configuration - IPFS Endpoint:**
+**Required Configuration:**
 
+1. **Enable Auto-Graft (New in v0.24.3):**
+```bash
+# Option 1: Command-line argument
+graph-indexer-agent start --enable-auto-graft
+
+# Option 2: Environment variable
+export INDEXER_AGENT_ENABLE_AUTO_GRAFT=true
+```
+Default: `false` (disabled - must be explicitly enabled)
+
+2. **IPFS Endpoint:**
 ```bash
 # Option 1: Environment variable
 export INDEXER_AGENT_IPFS_ENDPOINT=https://ipfs.thegraph.com
 
 # Option 2: Command-line argument
 graph-indexer-agent start --ipfs-endpoint https://ipfs.thegraph.com
 ```
-
 Default: `https://ipfs.network.thegraph.com` (if not specified)
 
 ## Introduction
@@ -62,11 +72,12 @@ Before using auto-graft, ensure you have:
 
 - **Graph Node**: Running with grafting support enabled (grafting is supported on all networks)
 - **IPFS Access**: The indexer must be able to fetch subgraph manifests from IPFS (configured via `--ipfs-endpoint` or `INDEXER_AGENT_IPFS_ENDPOINT` environment variable)
+- **Auto-Graft Enabled**: Starting from v0.24.3, auto-graft must be explicitly enabled via `--enable-auto-graft` flag or `INDEXER_AGENT_ENABLE_AUTO_GRAFT=true`
 - **Indexer Infrastructure**: Standard indexer setup with indexer-agent and indexer-service
 
 ## Using Auto-Graft
 
-The beauty of auto-graft is that it requires no configuration or special commands. Simply deploy your grafted subgraph as you would any other subgraph, and auto-graft handles the rest.
+Once enabled with the `--enable-auto-graft` flag, auto-graft requires no additional configuration or special commands. Simply deploy your grafted subgraph as you would any other subgraph, and auto-graft handles the rest.
 
 ### Example Subgraph Manifest with Graft
 
@@ -170,7 +181,7 @@ When a dependency reaches its target block, auto-graft will:
 - Continue indexing past the graft point to ensure the dependency remains available
 - Allow the main subgraph to successfully graft from this dependency
 
-*Note: Prior to v0.24.2, dependencies were automatically paused after reaching their target block. This behavior was removed to ensure better availability and prevent issues with allocations.*
+*Note: Prior to v0.24.3, auto-graft was enabled by default when an IPFS endpoint was configured. Starting from v0.24.3, it must be explicitly enabled with the `--enable-auto-graft` flag.*
 
 ## Monitoring Auto-Graft
 
diff --git a/docs/networks/arbitrum-sepolia.md b/docs/networks/arbitrum-sepolia.md
@@ -7,8 +7,8 @@ The Graph Network's testnet is on Arbitrum Sepolia (eip155:421614). Sepolia netw
 | Component          | Release                                                                              |
 | ------------------ | ------------------------------------------------------------------------------------ |
 | contracts          | [5.3.3](https://github.com/graphprotocol/contracts/releases/tag/v5.3.3)              |
-| indexer-agent      | [0.24.2](https://github.com/graphprotocol/indexer/releases/tag/v0.24.2)              |
-| indexer-cli        | [0.24.2](https://github.com/graphprotocol/indexer/releases/tag/v0.24.2)              |
+| indexer-agent      | [0.24.3](https://github.com/graphprotocol/indexer/releases/tag/v0.24.3)              |
+| indexer-cli        | [0.24.3](https://github.com/graphprotocol/indexer/releases/tag/v0.24.3)              |
 | indexer-service-rs | [1.0.0](https://github.com/graphprotocol/indexer-rs/releases/tag/v1.0.0)             |
 | tap-agent          | [1.0.0](https://github.com/graphprotocol/indexer-rs/releases/tag/v1.0.0)             |
 | graph-node         | [0.35.1](https://github.com/graphprotocol/graph-node/releases/tag/v0.35.1)           |
diff --git a/lerna.json b/lerna.json
@@ -4,5 +4,5 @@
   ],
   "npmClient": "yarn",
   "useWorkspaces": true,
-  "version": "0.24.2"
+  "version": "0.24.3"
 }
diff --git a/packages/indexer-agent/package.json b/packages/indexer-agent/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@graphprotocol/indexer-agent",
-  "version": "0.24.2",
+  "version": "0.24.3",
   "description": "Indexer agent",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -30,7 +30,7 @@
   },
   "dependencies": {
     "@graphprotocol/common-ts": "2.0.11",
-    "@graphprotocol/indexer-common": "^0.24.2",
+    "@graphprotocol/indexer-common": "^0.24.3",
     "@thi.ng/heaps": "^1.3.1",
     "axios": "0.26.1",
     "bs58": "5.0.0",
diff --git a/packages/indexer-agent/src/__tests__/indexer.ts b/packages/indexer-agent/src/__tests__/indexer.ts
@@ -140,6 +140,7 @@ const setup = async () => {
     'https://test-query-endpoint.xyz',
     'https://test-status-endpoint.xyz',
     'https://test-ipfs-endpoint.xyz',
+    false, // disable auto-graft for tests
   )
 
   const yamlObj = loadTestYamlConfig()
diff --git a/packages/indexer-agent/src/commands/common-options.ts b/packages/indexer-agent/src/commands/common-options.ts
@@ -108,6 +108,14 @@ export function injectCommonStartupOptions(argv: Argv): Argv {
       group: 'Indexer Infrastructure',
       default: 'https://ipfs.network.thegraph.com',
     })
+    .option('enable-auto-graft', {
+      description:
+        'Automatically deploy and sync graft dependencies for subgraphs',
+      type: 'boolean',
+      required: false,
+      group: 'Indexer Infrastructure',
+      default: false,
+    })
     .option('graph-node-query-endpoint', {
       description: 'Graph Node endpoint for querying subgraphs',
       type: 'string',
diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts
@@ -503,6 +503,7 @@ export async function run(
     argv.graphNodeQueryEndpoint,
     argv.graphNodeStatusEndpoint,
     argv.ipfsEndpoint,
+    argv.enableAutoGraft,
   )
 
   // --------------------------------------------------------------------------------
diff --git a/packages/indexer-cli/package.json b/packages/indexer-cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@graphprotocol/indexer-cli",
-  "version": "0.24.2",
+  "version": "0.24.3",
   "description": "Indexer CLI for The Graph Network",
   "main": "./dist/cli.js",
   "files": [
@@ -27,7 +27,7 @@
   },
   "dependencies": {
     "@graphprotocol/common-ts": "2.0.11",
-    "@graphprotocol/indexer-common": "^0.24.2",
+    "@graphprotocol/indexer-common": "^0.24.3",
     "@iarna/toml": "2.2.5",
     "@thi.ng/iterators": "5.1.74",
     "@urql/core": "3.1.0",
diff --git a/packages/indexer-common/package.json b/packages/indexer-common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@graphprotocol/indexer-common",
-  "version": "0.24.2",
+  "version": "0.24.3",
   "description": "Common library for Graph Protocol indexer components",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
diff --git a/packages/indexer-common/src/allocations/__tests__/tap.test.ts b/packages/indexer-common/src/allocations/__tests__/tap.test.ts
@@ -52,6 +52,7 @@ const setup = async () => {
     'https://test-query-endpoint.xyz',
     'https://test-status-endpoint.xyz',
     'https://test-ipfs-endpoint.xyz',
+    false, // disable auto-graft for tests
   )
 
   const network = await Network.create(
diff --git a/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts
@@ -45,6 +45,7 @@ const setup = async () => {
     'https://test-query-endpoint.xyz',
     'https://test-status-endpoint.xyz',
     'https://test-ipfs-endpoint.xyz',
+    false, // disable auto-graft for tests
   )
 
   const network = await Network.create(
diff --git a/packages/indexer-common/src/graph-node.ts b/packages/indexer-common/src/graph-node.ts
@@ -142,7 +142,8 @@ export class SubgraphManifestResolver {
 export class GraphNode {
   admin: RpcClient
   private queryBaseURL: URL
-  private manifestResolver: SubgraphManifestResolver
+  private manifestResolver: SubgraphManifestResolver | null
+  private enableAutoGraft: boolean
 
   status: Client
   logger: Logger
@@ -153,6 +154,7 @@ export class GraphNode {
     queryEndpoint: string,
     statusEndpoint: string,
     ipfsEndpoint: string,
+    enableAutoGraft: boolean = false,
   ) {
     this.logger = logger.child({ component: 'GraphNode' })
     this.status = createClient({
@@ -170,10 +172,19 @@ export class GraphNode {
     }
 
     this.queryBaseURL = new URL(`/subgraphs/id/`, queryEndpoint)
-    this.manifestResolver = new SubgraphManifestResolver(
-      ipfsEndpoint,
-      this.logger.child({ component: 'SubgraphManifestResolver' }),
-    )
+    this.enableAutoGraft = enableAutoGraft
+
+    // Only initialize manifest resolver if auto-graft is enabled
+    if (enableAutoGraft && ipfsEndpoint) {
+      this.manifestResolver = new SubgraphManifestResolver(
+        ipfsEndpoint,
+        this.logger.child({ component: 'SubgraphManifestResolver' }),
+      )
+      this.logger.info('Auto-graft feature enabled', { ipfsEndpoint })
+    } else {
+      this.manifestResolver = null
+      this.logger.info('Auto-graft feature disabled')
+    }
   }
 
   async connect(): Promise<void> {
@@ -693,7 +704,11 @@ export class GraphNode {
         await this.resume(deployment)
       } else {
         // Subgraph deployment not found
-        await this.autoGraftDeployDependencies(deployment, deploymentAssignments, name)
+
+        // Only attempt auto-graft if enabled
+        if (this.enableAutoGraft) {
+          await this.autoGraftDeployDependencies(deployment, deploymentAssignments, name)
+        }
 
         // Create and deploy the subgraph
         this.logger.debug(
@@ -734,6 +749,16 @@ export class GraphNode {
     name: string,
   ) {
     this.logger.debug('Auto graft deploy subgraph dependencies')
+
+    // Safety check - should not happen if called correctly from ensure()
+    if (!this.manifestResolver) {
+      this.logger.error('Auto-graft called but manifest resolver not initialized', {
+        name,
+        deployment: deployment.display,
+      })
+      return
+    }
+
     const { network: subgraphChainName } = await this.subgraphFeatures(deployment)
     const dependencies = await this.manifestResolver.resolveWithDependencies(deployment)
     if (dependencies.dependencies.length == 0) {
@@ -742,10 +767,9 @@ export class GraphNode {
         deployment: deployment.display,
       })
     } else {
-      this.logger.debug(
-        'graft dep chain found',
-        dependencies.dependencies.map((d) => d.base.ipfsHash),
-      )
+      this.logger.debug('graft dependency chain found', {
+        dependencies: dependencies.dependencies.map((d) => d.base.ipfsHash),
+      })
 
       for (const dependency of dependencies.dependencies) {
         const queriedAssignments = await this.subgraphDeploymentAssignmentsByDeploymentID(
@@ -774,6 +798,7 @@ export class GraphNode {
             (status) => status.subgraphDeployment.ipfsHash === dependency.base.ipfsHash,
           )
           if (!deploymentStatus) {
+            // we found a dependency assignment, but it's not present in indexing status
             this.logger.error(`Subgraph not found in indexing status`, {
               subgraph: dependency.base.ipfsHash,
               indexingStatus,
@@ -788,15 +813,18 @@ export class GraphNode {
             )
           }
         } else if (!dependencyAssignment) {
+          // important:
+          const idempotentName = `autograft-${dependency.base.ipfsHash.slice(0, 8)}`
           this.logger.debug('Dependency subgraph not found, creating, deploying...', {
-            name,
+            parentName: name,
+            idempotentName,
             deployment: dependency.base.display,
             block_required: dependency.block,
           })
           // are we paused at the block we wanted?
 
-          await this.create(name)
-          await this.deploy(name, dependency.base)
+          await this.create(idempotentName)
+          await this.deploy(idempotentName, dependency.base)
         }
         await this.syncToBlock(dependency.block, dependency.base, subgraphChainName)
       }
@@ -823,6 +851,10 @@ export class GraphNode {
       blockHeight,
     })
 
+    let lastProgressBlock = 0
+    let stuckCounter = 0
+    const maxStuckIterations = 20 // Increased since we're checking every 3s
+
     // loop-wait for the block to be synced
     for (;;) {
       // first ensure it's been deployed and is active, or already paused
@@ -875,6 +907,28 @@ export class GraphNode {
         throw new Error(`Subgraph not found in indexing status`)
       }
 
+      // Check for fatal errors - no point continuing if subgraph has failed
+      if (deploymentStatus.fatalError) {
+        this.logger.error(`Subgraph has fatal error, cannot sync to block`, {
+          subgraph: subgraphDeployment.ipfsHash,
+          targetBlock: blockHeight,
+          fatalError: deploymentStatus.fatalError,
+        })
+        throw new Error(
+          `Subgraph has fatal error: ${deploymentStatus.fatalError.message}`,
+        )
+      }
+
+      // Check health status
+      if (deploymentStatus.health === 'failed') {
+        this.logger.error(`Subgraph is in failed state, cannot sync to block`, {
+          subgraph: subgraphDeployment.ipfsHash,
+          targetBlock: blockHeight,
+          health: deploymentStatus.health,
+        })
+        throw new Error(`Subgraph is in failed state, cannot sync to target block`)
+      }
+
       const chain = deploymentStatus.chains.find((chain) => chain.network === chainName)
 
       if (!chain) {
@@ -902,6 +956,28 @@ export class GraphNode {
         await this.resume(subgraphDeployment)
       }
 
+      // Check for progress to detect stuck syncs
+      const currentBlock = chain.latestBlock?.number || 0
+      if (currentBlock > 0) {
+        if (currentBlock === lastProgressBlock) {
+          stuckCounter++
+          if (stuckCounter >= maxStuckIterations) {
+            this.logger.error(`Subgraph sync appears stuck at block ${currentBlock}`, {
+              subgraph: subgraphDeployment.ipfsHash,
+              targetBlock: blockHeight,
+              stuckAtBlock: currentBlock,
+              iterations: stuckCounter,
+            })
+            throw new Error(
+              `Sync appears stuck at block ${currentBlock} after ${stuckCounter} checks`,
+            )
+          }
+        } else {
+          stuckCounter = 0
+          lastProgressBlock = currentBlock
+        }
+      }
+
       // Is the graftBaseBlock within the range of the earliest and head of the chain?
       if (chain.latestBlock && chain.latestBlock.number >= blockHeight) {
         this.logger.warn(
@@ -919,7 +995,12 @@ export class GraphNode {
         `Subgraph not yet synced to block ${blockHeight}, waiting for 3s`,
         {
           subgraph: subgraphDeployment.ipfsHash,
-          indexingStatus,
+          currentBlock,
+          targetBlock: blockHeight,
+          progress:
+            currentBlock > 0
+              ? `${((currentBlock / blockHeight) * 100).toFixed(2)}%`
+              : 'starting',
         },
       )
       await waitForMs(waitTimeMs)
diff --git a/packages/indexer-common/src/indexer-management/__tests__/allocations.test.ts b/packages/indexer-common/src/indexer-management/__tests__/allocations.test.ts
@@ -57,6 +57,7 @@ const setup = async () => {
     'https://test-query-endpoint.xyz',
     'https://test-status-endpoint.xyz',
     'https://test-ipfs-endpoint.xyz',
+    false, // disable auto-graft for tests
   )
 
   const network = await Network.create(
diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts
@@ -105,6 +105,7 @@ const setupMonitor = async () => {
     'https://test-query-endpoint.xyz',
     statusEndpoint,
     'https://test-ipfs-endpoint.xyz',
+    false, // disable auto-graft for tests
   )
 
   const indexerOptions = spec.IndexerOptions.parse({
diff --git a/packages/indexer-common/src/indexer-management/__tests__/util.ts b/packages/indexer-common/src/indexer-management/__tests__/util.ts
@@ -41,6 +41,7 @@ export const createTestManagementClient = async (
     'https://test-query-endpoint.xyz',
     statusEndpoint,
     'https://test-ipfs-endpoint.xyz',
+    false, // disable auto-graft for tests
   )
 
   const networkSpecification = { ...testNetworkSpecification }

Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,5 @@`
`4`	`4`	`],`
`5`	`5`	`"npmClient": "yarn",`
`6`	`6`	`"useWorkspaces": true,`
`7`		`- "version": "0.24.2"`
	`7`	`+ "version": "0.24.3"`
`8`	`8`	`}`
Original file line number	Diff line number	Diff line change
`@@ -140,6 +140,7 @@ const setup = async () => {`
`140`	`140`	`'https://test-query-endpoint.xyz',`
`141`	`141`	`'https://test-status-endpoint.xyz',`
`142`	`142`	`'https://test-ipfs-endpoint.xyz',`
	`143`	`+ false, // disable auto-graft for tests`
`143`	`144`	`)`
`144`	`145`
`145`	`146`	`const yamlObj = loadTestYamlConfig()`
Original file line number	Diff line number	Diff line change
`@@ -503,6 +503,7 @@ export async function run(`
`503`	`503`	`argv.graphNodeQueryEndpoint,`
`504`	`504`	`argv.graphNodeStatusEndpoint,`
`505`	`505`	`argv.ipfsEndpoint,`
	`506`	`+ argv.enableAutoGraft,`
`506`	`507`	`)`
`507`	`508`
`508`	`509`	`// --------------------------------------------------------------------------------`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@graphprotocol/indexer-common",`
`3`		`- "version": "0.24.2",`
	`3`	`+ "version": "0.24.3",`
`4`	`4`	`"description": "Common library for Graph Protocol indexer components",`
`5`	`5`	`"main": "./dist/index.js",`
`6`	`6`	`"types": "./dist/index.d.ts",`