Generate types for source subgraph schema

Author: incrypto32

packages/cli/src/commands/codegen.ts

@@ -1,6 +1,7 @@
 import path from 'path';
 import { Args, Command, Flags } from '@oclif/core';
 import * as DataSourcesExtractor from '../command-helpers/data-sources';
+import { DEFAULT_IPFS_URL } from '../command-helpers/ipfs';
 import { assertGraphTsVersion, assertManifestApiVersion } from '../command-helpers/version';
 import debug from '../debug';
 import Protocol from '../protocols';
@@ -38,6 +39,11 @@ export default class CodegenCommand extends Command {
       summary: 'Generate Float Subgraph Uncrashable helper file.',
       char: 'u',
     }),
+    ipfs: Flags.string({
+      summary: 'IPFS node to use for fetching subgraph data.',
+      char: 'i',
+      default: DEFAULT_IPFS_URL,
+    }),
     'uncrashable-config': Flags.file({
       summary: 'Directory for uncrashable config.',
       aliases: ['uc'],
@@ -54,6 +60,7 @@ export default class CodegenCommand extends Command {
         'output-dir': outputDir,
         'skip-migrations': skipMigrations,
         watch,
+        ipfs,
         uncrashable,
         'uncrashable-config': uncrashableConfig,
       },
@@ -62,6 +69,7 @@ export default class CodegenCommand extends Command {
     codegenDebug('Initialized codegen manifest: %o', manifest);
 
     let protocol;
+    let subgraphSources;
     try {
       // Checks to make sure codegen doesn't run against
       // older subgraphs (both apiVersion and graph-ts version).
@@ -73,8 +81,10 @@ export default class CodegenCommand extends Command {
       await assertGraphTsVersion(path.dirname(manifest), '0.25.0');
 
       const dataSourcesAndTemplates = await DataSourcesExtractor.fromFilePath(manifest);
-
       protocol = Protocol.fromDataSources(dataSourcesAndTemplates);
+      subgraphSources = dataSourcesAndTemplates
+        .filter((ds: any) => ds.kind == 'subgraph')
+        .map((ds: any) => ds.source.address);
     } catch (e) {
       this.error(e, { exit: 1 });
     }
@@ -85,7 +95,9 @@ export default class CodegenCommand extends Command {
       skipMigrations,
       protocol,
       uncrashable,
+      subgraphSources,
       uncrashableConfig: uncrashableConfig || 'uncrashable-config.yaml',
+      ipfsUrl: ipfs,
     });
 
     // Watch working directory for file updates or additions, trigger

packages/cli/src/schema.ts

@@ -23,4 +23,13 @@ export default class Schema {
     const ast = graphql.parse(document);
     return new Schema(filename, document, ast);
   }
+
+  static async loadFromString(filename: string, document: string) {
+    try {
+      const ast = graphql.parse(document);
+      return new Schema(filename, document, ast);
+    } catch (e) {
+      throw new Error(`Failed to load schema from string: ${e.message}`);
+    }
+  }
 }

packages/cli/src/type-generator.ts

@@ -3,13 +3,17 @@ import fs from 'fs-extra';
 import * as toolbox from 'gluegun';
 import * as graphql from 'graphql/language';
 import immutable from 'immutable';
+import { create } from 'ipfs-http-client';
+import yaml from 'js-yaml';
 import prettier from 'prettier';
 // @ts-expect-error TODO: type out if necessary
 import uncrashable from '@float-capital/float-subgraph-uncrashable/src/Index.bs.js';
 import DataSourceTemplateCodeGenerator from './codegen/template';
 import { GENERATED_FILE_NOTE, ModuleImports } from './codegen/typescript';
+import { appendApiVersionForGraph } from './command-helpers/compiler';
 import { displayPath } from './command-helpers/fs';
 import { Spinner, step, withSpinner } from './command-helpers/spinner';
+import { GRAPH_CLI_SHARED_HEADERS } from './constants';
 import debug from './debug';
 import { applyMigrations } from './migrations';
 import Protocol from './protocols';
@@ -28,6 +32,8 @@ export interface TypeGeneratorOptions {
   skipMigrations?: boolean;
   uncrashable: boolean;
   uncrashableConfig: string;
+  subgraphSources: string[];
+  ipfsUrl: string;
 }
 
 export default class TypeGenerator {
@@ -65,6 +71,11 @@ export default class TypeGenerator {
       return;
     }
 
+    if (this.options.subgraphSources.length > 0) {
+      typeGenDebug.extend('generateTypes')('Subgraph uses subgraph datasources.');
+      toolbox.print.success('Subgraph uses subgraph datasources.');
+    }
+
     try {
       if (!this.options.skipMigrations && this.options.subgraphManifest) {
         await applyMigrations({
@@ -93,6 +104,30 @@ export default class TypeGenerator {
       typeGenDebug.extend('generateTypes')('Generating types for schema');
       await this.generateTypesForSchema(schema);
 
+      if (this.options.subgraphSources.length > 0) {
+        const ipfsClient = create({
+          url: appendApiVersionForGraph(this.options.ipfsUrl.toString()),
+          headers: {
+            ...GRAPH_CLI_SHARED_HEADERS,
+          },
+        });
+
+        await Promise.all(
+          this.options.subgraphSources.map(async manifest => {
+            const subgraphSchemaFile = await this.loadSubgraphSchemaFromIPFS(ipfsClient, manifest);
+
+            const subgraphSchema = await Schema.loadFromString(
+              `js${manifest}.graphql`,
+              subgraphSchemaFile,
+            );
+            typeGenDebug.extend('generateTypes')(
+              `Generating types for subgraph datasource ${manifest}`,
+            );
+            await this.generateTypesForSchema(subgraphSchema, `subgraph-${manifest}.ts`);
+          }),
+        );
+      }
+
       toolbox.print.success('\nTypes generated successfully\n');
 
       if (this.options.uncrashable && this.options.uncrashableConfig) {
@@ -105,6 +140,37 @@ export default class TypeGenerator {
     }
   }
 
+  async loadSubgraphSchemaFromIPFS(ipfsClient: any, manifest: string) {
+    typeGenDebug.extend('loadSubgraphSchemaFromIPFS')(`Loading schema from IPFS ${manifest}`);
+    try {
+      const manifestBuffer = ipfsClient.cat(manifest);
+      let manifestFile = '';
+      for await (const chunk of manifestBuffer) {
+        manifestFile += Buffer.from(chunk).toString('utf8'); // Explicitly convert each chunk to UTF-8
+      }
+
+      const manifestYaml: any = yaml.safeLoad(manifestFile);
+      let schema = manifestYaml.schema.file['/'];
+
+      if (schema.startsWith('/ipfs/')) {
+        schema = schema.slice(6);
+      }
+
+      const schemaBuffer = ipfsClient.cat(schema);
+      let schemaFile = '';
+      for await (const chunk of schemaBuffer) {
+        schemaFile += Buffer.from(chunk).toString('utf8'); // Explicitly convert each chunk to UTF-8
+      }
+      return schemaFile;
+    } catch (e) {
+      typeGenDebug.extend('loadSubgraphSchemaFromIPFS')(
+        `Failed to load schema from IPFS ${manifest}`,
+      );
+      typeGenDebug.extend('loadSubgraphSchemaFromIPFS')(e);
+      throw Error(`Failed to load schema from IPFS ${manifest}`);
+    }
+  }
+
   async generateUncrashableEntities(graphSchema: any) {
     const ast = graphql.parse(graphSchema.document);
     const entityDefinitions = ast['definitions'];
@@ -160,7 +226,11 @@ export default class TypeGenerator {
     );
   }
 
-  async generateTypesForSchema(schema: any) {
+  async generateTypesForSchema(
+    schema: any,
+    fileName = 'schema.ts', // Default file name
+    outputDir: string = this.options.outputDir, // Default output directory
+  ) {
     return await withSpinner(
       `Generate types for GraphQL schema`,
       `Failed to generate types for GraphQL schema`,
@@ -180,7 +250,7 @@ export default class TypeGenerator {
           },
         );
 
-        const outputFile = path.join(this.options.outputDir, 'schema.ts');
+        const outputFile = path.join(outputDir, fileName); // Use provided outputDir and fileName
         step(spinner, 'Write types to', displayPath(outputFile));
         await fs.mkdirs(path.dirname(outputFile));
         await fs.writeFile(outputFile, code);

packages/ts/common/collections.ts

@@ -458,6 +458,28 @@ export class Entity extends TypedMap<string, Value> {
   }
 }
 
+/**
+ * Common representation for entity triggers, this wraps the entity
+ * and has fields for the operation type and the entity type.
+ */
+export class EntityTrigger<T extends Entity> {
+  constructor(
+    public operation: EntityOp,
+    public type: string,
+    public data: T, // T is a specific type that extends Entity
+  ) {}
+}
+
+/**
+ * Enum for entity operations.
+ * Create, Modify, Remove
+ */
+export enum EntityOp {
+  Create,
+  Modify,
+  Remove,
+}
+
 /**
  * The result of an operation, with a corresponding value and error type.
  */