agent,cli,common: feature - auto graft sync (#1087)

Author: dwerner

docs/action-queue.md

@@ -191,6 +191,7 @@ enum ActionStatus {
     queued
     approved
     pending
+    deploying
     success
     failed
     canceled

docs/networks/arbitrum-sepolia.md

@@ -56,6 +56,7 @@ testnet (for now) are Mainnet subgraphs. This means:
 | `INDEXER_AGENT_EPOCH_SUBGRAPH_ENDPOINT`     | `--epoch-subgraph-endpoint`     | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/BhnsdeZihU4SuokxZMLF4FQBVJ3jgtZf6v51gHvz3bSS` |
 | `INDEXER_AGENT_TAP_SUBGRAPH_DEPLOYMENT`     | `--tap-subgraph-deployment`     | `QmUiLdbsk6c51UMdcNBxsP3KadJpkmp6a3k2NCprR4ZFeM`                                                                        |
 | `INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT`       | `--tap-subgraph-endpoint`       | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/7ubx365MiqBH5iUz6XWXWT8PTof5BVAyEzdb8m17RvbD` |
+| `INDEXER_AGENT_IPFS_ENDPOINT` | `--ipfs-endpoint` | `https://ipfs.network.thegraph.com` |
 
 In order to avoid collecting or claiming query fees below a certain threshold
 (e.g. below the cost of the two transactions), the following configuration

packages/indexer-agent/src/__tests__/indexer.ts

@@ -139,6 +139,7 @@ const setup = async () => {
     'http://test-admin-endpoint.xyz',
     'https://test-query-endpoint.xyz',
     'https://test-status-endpoint.xyz',
+    'https://test-ipfs-endpoint.xyz',
   )
 
   const yamlObj = loadTestYamlConfig()

packages/indexer-agent/src/commands/common-options.ts

@@ -101,6 +101,13 @@ export function injectCommonStartupOptions(argv: Argv): Argv {
       default: 50,
       group: 'Postgres',
     })
+    .option('ipfs-endpoint', {
+      description: 'IPFS endpoint for querying manifests.',
+      type: 'string',
+      required: true,
+      group: 'Indexer Infrastructure',
+      default: 'https://ipfs.network.thegraph.com',
+    })
     .option('graph-node-query-endpoint', {
       description: 'Graph Node endpoint for querying subgraphs',
       type: 'string',

packages/indexer-agent/src/commands/start.ts

@@ -502,6 +502,7 @@ export async function run(
     argv.graphNodeAdminEndpoint,
     argv.graphNodeQueryEndpoint,
     argv.graphNodeStatusEndpoint,
+    argv.ipfsEndpoint,
   )
 
   // --------------------------------------------------------------------------------

packages/indexer-agent/src/db/migrations/18-actions-expand-action-status-add-deploying.ts

@@ -0,0 +1,51 @@
+import { Logger } from '@graphprotocol/common-ts'
+import { DataTypes, QueryInterface } from 'sequelize'
+
+interface MigrationContext {
+  queryInterface: QueryInterface
+  logger: Logger
+}
+
+interface Context {
+  context: MigrationContext
+}
+
+export async function up({ context }: Context): Promise<void> {
+  const { queryInterface, logger } = context
+
+  logger.debug(`Checking if 'Actions' table exists`)
+  const tables = await queryInterface.showAllTables()
+  if (!tables.includes('Actions')) {
+    logger.info(`Actions table does not exist, migration not necessary`)
+    return
+  }
+
+  logger.debug(`Checking if 'Actions' table needs to be migrated`)
+  const table = await queryInterface.describeTable('Actions')
+  const statusColumn = table.status
+  if (statusColumn) {
+    logger.debug(`'status' column exists with type = ${statusColumn.type}`)
+    logger.info(`Update 'status' column to support variant 'deploying' status`)
+    await queryInterface.changeColumn('Actions', 'status', {
+      type: DataTypes.ENUM(
+        'queued',
+        'approved',
+        'deploying',
+        'pending',
+        'success',
+        'failed',
+        'canceled',
+      ),
+      allowNull: false,
+    })
+    return
+  }
+}
+
+export async function down({ context }: Context): Promise<void> {
+  const { logger } = context
+  logger.info(
+    `No 'down' migration needed since the 'up' migration simply added a new status 'deploying'`,
+  )
+  return
+}

packages/indexer-cli/src/__tests__/indexer/cost.test.ts

@@ -5,7 +5,6 @@ import {
   deleteFromAllTables,
   seedCostModels,
   setupSingleNetwork,
-  setupMultiNetworks,
 } from '../util'
 import path from 'path'
 
@@ -199,30 +198,3 @@ describe('Indexer cost tests', () => {
     )
   })
 })
-
-describe('Indexer cost tests multiNetworks', () => {
-  beforeAll(setupMultiNetworks)
-  afterAll(teardown)
-  beforeEach(seedCostModels)
-  afterEach(deleteFromAllTables)
-
-  describe('Cost set...', () => {
-    cliTest(
-      'Indexer cost set model deployment id - reject multinetwork mode',
-      [
-        'indexer',
-        'cost',
-        'set',
-        'model',
-        'QmXRpJW3qBuYaiBYHdhv8DF4bHDZhXBmh91MtrnhJfQ5Lk',
-        'references/basic.agora',
-      ],
-      'references/indexer-cost-model-deployment-multinetworks',
-      {
-        expectedExitCode: 1,
-        cwd: baseDir,
-        timeout: 10000,
-      },
-    )
-  })
-})

packages/indexer-cli/src/__tests__/util.ts

@@ -87,6 +87,7 @@ export const setup = async (multiNetworksEnabled: boolean) => {
     'http://test-admin-endpoint.xyz',
     'https://test-query-endpoint.xyz',
     statusEndpoint,
+    'https://test-ipfs-endpoint.xyz',
   )
 
   const network = await Network.create(

packages/indexer-cli/src/commands/indexer/actions/delete.ts

@@ -13,7 +13,7 @@ ${chalk.bold('graph indexer actions delete')} [options] [<actionID1> ...]
 ${chalk.dim('Options:')}
 
   -h, --help                                                        Show usage information
-      --status  queued|approved|pending|success|failed|canceled     Filter by status
+      --status  queued|approved|deploying|pending|success|failed|canceled     Filter by status
   -o, --output table|json|yaml                                      Choose the output format: table (default), JSON, or YAML 
 `
 

packages/indexer-cli/src/commands/indexer/actions/get.ts

@@ -114,12 +114,18 @@ module.exports = {
 
       if (
         status &&
-        !['queued', 'approved', 'pending', 'success', 'failed', 'canceled'].includes(
-          status,
-        )
+        ![
+          'queued',
+          'approved',
+          'deploying',
+          'pending',
+          'success',
+          'failed',
+          'canceled',
+        ].includes(status)
       ) {
         throw Error(
-          `Invalid '--status' provided, must be one of ['queued', 'approved', 'pending', 'success', 'failed', 'canceled]`,
+          `Invalid '--status' provided, must be one of ['queued', 'approved', 'deploying', 'pending', 'success', 'failed', 'canceled]`,
         )
       }