From 255005c1577943cce35606cb92bb37247661dc1f Mon Sep 17 00:00:00 2001
From: Matthieu Vachon <matt@streamingfast.io>
Date: Fri, 28 Jan 2022 13:27:54 -0500
Subject: [PATCH 1/2] Moved to one SubgraphManifest schema file per protocol
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This will make it easier to add new network and will improve error messages when someone will be using fields that do not exist on a specific network but does in other (`temlates` for example used in NEAR were given a bunch of errors like the support was possible while it should have displayed an error on the `templates` node itself).

Using `templates` on NEAR before this PR:

```
✔ Apply migrations
✖ Failed to load subgraph from ../graph-node-dev/subgraphs/near/data-source/testnet.yaml: Error in ../graph-node-dev/subgraphs/near/data-source/testnet.yaml:

  Path: templates > 0 > source
  No value provided

  Path: templates > 0 > mapping
  Unexpected key in map: receiptHandlers

  Path: templates > 0 > mapping > abis
  No value provided
```

And with this PR:

```
$ ./bin/graph build ../graph-node-dev/subgraphs/near/data-source/testnet.yaml
✖ Failed to load subgraph from ../graph-node-dev/subgraphs/near/data-source/testnet.yaml: Error in ../graph-node-dev/subgraphs/near/data-source/testnet.yaml:

  Path: /
  Unexpected key in map: templates
```
---
 manifest-schema.graphql                       | 122 ------------------
 schemas/ethereum.graphql                      |  86 ++++++++++++
 schemas/near.graphql                          |  51 ++++++++
 src/protocols/index.js                        |  25 +++-
 src/subgraph.js                               |  31 +++--
 src/validation/manifest.js                    |  20 +--
 .../validation/example-values-found.stderr    |   4 +-
 tests/cli/validation/invalid-manifest.stderr  |  30 +----
 8 files changed, 189 insertions(+), 180 deletions(-)
 delete mode 100644 manifest-schema.graphql
 create mode 100644 schemas/ethereum.graphql
 create mode 100644 schemas/near.graphql

diff --git a/manifest-schema.graphql b/manifest-schema.graphql
deleted file mode 100644
index f4cf25c46..000000000
--- a/manifest-schema.graphql
+++ /dev/null
@@ -1,122 +0,0 @@
-scalar String
-scalar File
-scalar BigInt
-
-type Schema {
-  file: File!
-}
-
-union DataSource =
-  EthereumContractDataSource |
-  NearContractDataSource
-
-union DataSourceTemplate = EthereumContractDataSourceTemplate
-
-type EthereumContractDataSource {
-  kind: String!
-  name: String!
-  network: String
-  source: EthereumContractSource!
-  mapping: EthereumContractMapping!
-}
-
-type NearContractDataSource {
-  kind: String!
-  name: String!
-  network: String
-  source: NearContractSource!
-  mapping: NearContractMapping!
-}
-
-type EthereumContractSource {
-  address: String
-  abi: String!
-  startBlock: BigInt
-}
-
-type NearContractSource {
-  account: String
-  startBlock: BigInt
-}
-
-type EthereumContractMapping {
-  kind: String
-  apiVersion: String!
-  language: String!
-  file: File!
-  entities: [String!]!
-  abis: [EthereumContractAbi!]!
-  blockHandlers: [EthereumBlockHandler!]
-  callHandlers: [EthereumCallHandler!]
-  eventHandlers: [EthereumContractEventHandler!]
-}
-
-type NearContractMapping {
-  apiVersion: String!
-  language: String!
-  file: File!
-  entities: [String!]!
-  blockHandlers: [NearBlockHandler!]
-  receiptHandlers: [NearReceiptHandler!]
-}
-
-type EthereumContractAbi {
-  name: String!
-  file: File!
-}
-
-type EthereumBlockHandler {
-  handler: String!
-  filter: EthereumBlockFilter
-}
-
-type EthereumBlockFilter {
-  kind: String!
-}
-
-type NearBlockHandler {
-  handler: String!
-}
-
-type EthereumCallHandler {
-  function: String!
-  handler: String!
-}
-
-type NearReceiptHandler {
-  handler: String!
-}
-
-type EthereumContractEventHandler {
-  event: String!
-  topic0: String
-  handler: String!
-}
-
-type Graft {
-  base: String!
-  block: BigInt!
-}
-
-type SubgraphManifest {
-  specVersion: String!
-  features: [String!]
-  schema: Schema!
-  description: String
-  repository: String
-  graft: Graft
-  dataSources: [DataSource!]!
-  templates: [DataSourceTemplate!]
-}
-
-type EthereumContractDataSourceTemplate {
-  kind: String!
-  name: String!
-  network: String
-  source: EthereumContractSourceTemplate!
-  mapping: EthereumContractMapping!
-}
-
-type EthereumContractSourceTemplate {
-  abi: String!
-}
diff --git a/schemas/ethereum.graphql b/schemas/ethereum.graphql
new file mode 100644
index 000000000..1b16d31dd
--- /dev/null
+++ b/schemas/ethereum.graphql
@@ -0,0 +1,86 @@
+scalar String
+scalar File
+scalar BigInt
+
+type SubgraphManifest {
+  specVersion: String!
+  features: [String!]
+  schema: Schema!
+  description: String
+  repository: String
+  graft: Graft
+  dataSources: [DataSource!]!
+  templates: [DataSourceTemplate!]
+}
+
+type Schema {
+  file: File!
+}
+
+type DataSource {
+  kind: String!
+  name: String!
+  network: String
+  source: ContractSource!
+  mapping: ContractMapping!
+}
+
+type ContractSource {
+  address: String
+  abi: String!
+  startBlock: BigInt
+}
+
+type ContractMapping {
+  kind: String
+  apiVersion: String!
+  language: String!
+  file: File!
+  entities: [String!]!
+  abis: [ContractAbi!]!
+  blockHandlers: [BlockHandler!]
+  callHandlers: [CallHandler!]
+  eventHandlers: [ContractEventHandler!]
+}
+
+type ContractAbi {
+  name: String!
+  file: File!
+}
+
+type BlockHandler {
+  handler: String!
+  filter: BlockFilter
+}
+
+type BlockFilter {
+  kind: String!
+}
+
+type CallHandler {
+  function: String!
+  handler: String!
+}
+
+type ContractEventHandler {
+  event: String!
+  topic0: String
+  handler: String!
+}
+
+type Graft {
+  base: String!
+  block: BigInt!
+}
+
+type DataSourceTemplate {
+  kind: String!
+  name: String!
+  network: String
+  source: ContractSourceTemplate!
+  mapping: ContractMapping!
+}
+
+type ContractSourceTemplate {
+  abi: String!
+}
diff --git a/schemas/near.graphql b/schemas/near.graphql
new file mode 100644
index 000000000..b40ebd848
--- /dev/null
+++ b/schemas/near.graphql
@@ -0,0 +1,51 @@
+scalar String
+scalar File
+scalar BigInt
+
+type SubgraphManifest {
+  specVersion: String!
+  schema: Schema!
+  description: String
+  repository: String
+  graft: Graft
+  dataSources: [DataSource!]!
+}
+
+type Schema {
+  file: File!
+}
+
+type DataSource {
+  kind: String!
+  name: String!
+  network: String
+  source: ContractSource!
+  mapping: ContractMapping!
+}
+
+type ContractSource {
+  account: String
+  startBlock: BigInt
+}
+
+type ContractMapping {
+  apiVersion: String!
+  language: String!
+  file: File!
+  entities: [String!]!
+  blockHandlers: [BlockHandler!]
+  receiptHandlers: [ReceiptHandler!]
+}
+
+type BlockHandler {
+  handler: String!
+}
+
+type ReceiptHandler {
+  handler: String!
+}
+
+type Graft {
+  base: String!
+  block: BigInt!
+}
diff --git a/src/protocols/index.js b/src/protocols/index.js
index 8bd77ecab..05cc73499 100644
--- a/src/protocols/index.js
+++ b/src/protocols/index.js
@@ -1,6 +1,7 @@
 const immutable = require('immutable')
 const EthereumTypeGenerator = require('./ethereum/type-generator')
 const EthereumTemplateCodeGen = require('./ethereum/codegen/template')
+const NearTemplateCodeGen = require('./near/codegen/template')
 const EthereumABI = require('./ethereum/abi')
 const EthereumSubgraph = require('./ethereum/subgraph')
 const NearSubgraph = require('./near/subgraph')
@@ -60,10 +61,7 @@ module.exports = class Protocol {
         'aurora',
         'aurora-testnet',
       ],
-      near: [
-        'near-mainnet',
-        'near-testnet'
-      ],
+      near: ['near-mainnet', 'near-testnet'],
     })
   }
 
@@ -108,6 +106,15 @@ module.exports = class Protocol {
     }
   }
 
+  hasTemplates() {
+    switch (this.name) {
+      case 'ethereum':
+        return true
+      case 'near':
+        return false
+    }
+  }
+
   getTypeGenerator(options) {
     switch (this.name) {
       case 'ethereum':
@@ -118,13 +125,17 @@ module.exports = class Protocol {
   }
 
   getTemplateCodeGen(template) {
+    if (!this.hasTemplates()) {
+      throw new Error(
+        `Template data sources with kind '${this.name}' are not supported yet`,
+      )
+    }
+
     switch (this.name) {
       case 'ethereum':
         return new EthereumTemplateCodeGen(template)
       default:
-        throw new Error(
-          `Template data sources with kind '${this.name}' are not supported yet`,
-        )
+        throw new Error(`Template data sources with kind '${this.name}' is unknown`)
     }
   }
 
diff --git a/src/subgraph.js b/src/subgraph.js
index 99f61b89a..86dc8c746 100644
--- a/src/subgraph.js
+++ b/src/subgraph.js
@@ -27,7 +27,7 @@ const buildCombinedWarning = (filename, warnings) =>
     ? warnings.reduce(
         (msg, w) =>
           `${msg}
-  
+
     Path: ${w.get('path').size === 0 ? '/' : w.get('path').join(' > ')}
     ${w
       .get('message')
@@ -39,9 +39,21 @@ const buildCombinedWarning = (filename, warnings) =>
 
 module.exports = class Subgraph {
   static async validate(data, protocol, { resolveFile }) {
+    if (protocol.name == null) {
+      return immutable.fromJS([
+        {
+          path: [],
+          message: `Unable to determine for which protocol manifest file is built for. Ensure you have at least one 'dataSources' and/or 'templates' elements defined in your subgraph.`,
+        },
+      ])
+    }
+
     // Parse the default subgraph schema
     let schema = graphql.parse(
-      await fs.readFile(path.join(__dirname, '..', 'manifest-schema.graphql'), 'utf-8'),
+      await fs.readFile(
+        path.join(__dirname, '..', 'schemas', `${protocol.name}.graphql`),
+        'utf-8',
+      ),
     )
 
     // Obtain the root `SubgraphManifest` type from the schema
@@ -146,10 +158,7 @@ At least one such handler must be defined.`,
   }
 
   static validateContractValues(manifest, protocol) {
-    return validation.validateContractValues(
-      manifest,
-      protocol,
-    )
+    return validation.validateContractValues(manifest, protocol)
   }
 
   // Validate that data source names are unique, so they don't overwrite each other.
@@ -200,17 +209,13 @@ More than one template named '${name}', template names must be unique.`,
     return yaml.stringify(manifest.toJS())
   }
 
-  static async load(
-    filename,
-    { protocol, skipValidation } = { skipValidation: false }
-  ) {
+  static async load(filename, { protocol, skipValidation } = { skipValidation: false }) {
     // Load and validate the manifest
     let data = null
 
-    if(filename.match(/.js$/)) {
+    if (filename.match(/.js$/)) {
       data = require(path.resolve(filename))
-    }
-    else {
+    } else {
       data = yaml.parse(await fs.readFile(filename, 'utf-8'))
     }
 
diff --git a/src/validation/manifest.js b/src/validation/manifest.js
index 66c865bf5..1de702a54 100644
--- a/src/validation/manifest.js
+++ b/src/validation/manifest.js
@@ -52,8 +52,7 @@ const validators = immutable.fromJS({
     validators.get(ctx.getIn(['type', 'name', 'value']))(value, ctx),
 
   UnionTypeDefinition: (value, ctx) => {
-    const unionVariants = ctx
-      .getIn(['type', 'types'])
+    const unionVariants = ctx.getIn(['type', 'types'])
 
     let errors = List()
 
@@ -78,7 +77,10 @@ const validators = immutable.fromJS({
 
   NonNullType: (value, ctx) =>
     value !== null && value !== undefined
-      ? validateValue(value, ctx.update('type', type => type.get('type')))
+      ? validateValue(
+          value,
+          ctx.update('type', type => type.get('type')),
+        )
       : immutable.fromJS([
           {
             path: ctx.get('path'),
@@ -126,7 +128,7 @@ const validators = immutable.fromJS({
                     ),
                   )
                 : errors.push(
-                    key == 'templates'
+                    key == 'templates' && ctx.get('protocol').hasTemplates()
                       ? immutable.fromJS({
                           path: ctx.get('path'),
                           message:
@@ -211,7 +213,8 @@ const validateValue = (value, ctx) => {
 }
 
 const validateDataSourceForNetwork = (dataSources, protocol) =>
-  dataSources.filter(dataSource => protocol.isValidKindName(dataSource.kind))
+  dataSources
+    .filter(dataSource => protocol.isValidKindName(dataSource.kind))
     .reduce(
       (networks, dataSource) =>
         networks.update(dataSource.network, dataSources =>
@@ -235,11 +238,11 @@ const validateDataSourceNetworks = (value, protocol) => {
 ${networks
   .map(
     (dataSources, network) =>
-    `  ${
+      `  ${
         network === undefined
           ? 'Data sources and templates having no network set'
-        : `Data sources and templates using '${network}'`
-        }:\n${dataSources.map(ds => `    - ${ds}`).join('\n')}`,
+          : `Data sources and templates using '${network}'`
+      }:\n${dataSources.map(ds => `    - ${ds}`).join('\n')}`,
   )
   .join('\n')}
 Recommendation: Make all data sources and templates use the same network name.`,
@@ -262,6 +265,7 @@ const validateManifest = (value, type, schema, protocol, { resolveFile }) => {
             path: [],
             errors: [],
             resolveFile,
+            protocol,
           }),
         )
       : immutable.fromJS([
diff --git a/tests/cli/validation/example-values-found.stderr b/tests/cli/validation/example-values-found.stderr
index 600bf2fcf..524095619 100644
--- a/tests/cli/validation/example-values-found.stderr
+++ b/tests/cli/validation/example-values-found.stderr
@@ -1,10 +1,10 @@
 - Load subgraph from subgraph.yaml
 ⚠ Warnings while loading subgraph from subgraph.yaml: Warnings in subgraph.yaml:
-  
+
     Path: repository
     The repository is still set to https://github.com/graphprotocol/example-subgraph.
     Please replace it with a link to your subgraph source code.
-  
+
     Path: description
     The description is still the one from the example subgraph.
     Please update it to tell users more about your subgraph.
diff --git a/tests/cli/validation/invalid-manifest.stderr b/tests/cli/validation/invalid-manifest.stderr
index f7794fb77..6c3bc469e 100644
--- a/tests/cli/validation/invalid-manifest.stderr
+++ b/tests/cli/validation/invalid-manifest.stderr
@@ -1,31 +1,5 @@
 - Load subgraph from subgraph.yaml
 ✖ Failed to load subgraph from subgraph.yaml: Error in subgraph.yaml:
 
-  Path: specVersion
-  No value provided
-
-  Path: schema > file
-  File does not exist: non-existent.grapqhl
-
-  Path: dataSources > 0 > kind
-  No value provided
-
-  Path: dataSources > 0 > name
-  Expected string, found number:
-  5
-
-  Path: dataSources > 0 > source
-  No value provided
-
-  Path: dataSources > 0 > mapping
-  Expected map, found list:
-  - 12
-  - 13
-  - 14
-
-  Path: dataSources > 0
-  Unexpected key in map: abis
-
-  Path: templates
-  Expected list, found map:
-  field: foo
+  Path: /
+  Unable to determine for which protocol manifest file is built for. Ensure you have at least one 'dataSources' and/or 'templates' elements defined in your subgraph.

From 00427b7f25db28c3eca1d1ce9e615decf49f2eae Mon Sep 17 00:00:00 2001
From: Matthieu Vachon <matt@streamingfast.io>
Date: Mon, 7 Feb 2022 10:47:26 -0500
Subject: [PATCH 2/2] Updated PR with comments

---
 .../protocols/ethereum/manifest.graphql       |  6 +++++
 src/protocols/index.js                        |  1 -
 .../protocols/near/manifest.graphql           |  6 +++++
 src/subgraph.js                               |  2 +-
 tests/cli/validation.test.js                  |  8 ++++++
 ...alid-manifest-cannot-infer-protocol.stderr |  5 ++++
 .../subgraph.yaml                             | 12 +++++++++
 tests/cli/validation/invalid-manifest.stderr  | 27 +++++++++++++++++--
 .../validation/invalid-manifest/subgraph.yaml |  3 ++-
 9 files changed, 65 insertions(+), 5 deletions(-)
 rename schemas/ethereum.graphql => src/protocols/ethereum/manifest.graphql (84%)
 rename schemas/near.graphql => src/protocols/near/manifest.graphql (75%)
 create mode 100644 tests/cli/validation/invalid-manifest-cannot-infer-protocol.stderr
 create mode 100644 tests/cli/validation/invalid-manifest-cannot-infer-protocol/subgraph.yaml

diff --git a/schemas/ethereum.graphql b/src/protocols/ethereum/manifest.graphql
similarity index 84%
rename from schemas/ethereum.graphql
rename to src/protocols/ethereum/manifest.graphql
index 1b16d31dd..59a697f88 100644
--- a/schemas/ethereum.graphql
+++ b/src/protocols/ethereum/manifest.graphql
@@ -1,3 +1,9 @@
+# Each referenced type's in any of the types below must be listed
+# here either as `scalar` or `type` for the validation code to work
+# properly.
+#
+# That's why `String` is listed as a scalar even though it's built-in
+# GraphQL basic types.
 scalar String
 scalar File
 scalar BigInt
diff --git a/src/protocols/index.js b/src/protocols/index.js
index 05cc73499..293513f79 100644
--- a/src/protocols/index.js
+++ b/src/protocols/index.js
@@ -1,7 +1,6 @@
 const immutable = require('immutable')
 const EthereumTypeGenerator = require('./ethereum/type-generator')
 const EthereumTemplateCodeGen = require('./ethereum/codegen/template')
-const NearTemplateCodeGen = require('./near/codegen/template')
 const EthereumABI = require('./ethereum/abi')
 const EthereumSubgraph = require('./ethereum/subgraph')
 const NearSubgraph = require('./near/subgraph')
diff --git a/schemas/near.graphql b/src/protocols/near/manifest.graphql
similarity index 75%
rename from schemas/near.graphql
rename to src/protocols/near/manifest.graphql
index b40ebd848..a9fd91628 100644
--- a/schemas/near.graphql
+++ b/src/protocols/near/manifest.graphql
@@ -1,3 +1,9 @@
+# Each referenced type's in any of the types below must be listed
+# here either as `scalar` or `type` for the validation code to work
+# properly.
+#
+# That's why `String` is listed as a scalar even though it's built-in
+# GraphQL basic types.
 scalar String
 scalar File
 scalar BigInt
diff --git a/src/subgraph.js b/src/subgraph.js
index 86dc8c746..fc9c66ca8 100644
--- a/src/subgraph.js
+++ b/src/subgraph.js
@@ -51,7 +51,7 @@ module.exports = class Subgraph {
     // Parse the default subgraph schema
     let schema = graphql.parse(
       await fs.readFile(
-        path.join(__dirname, '..', 'schemas', `${protocol.name}.graphql`),
+        path.join(__dirname, 'protocols', protocol.name, `manifest.graphql`),
         'utf-8',
       ),
     )
diff --git a/tests/cli/validation.test.js b/tests/cli/validation.test.js
index 0dff02e1d..ef89def61 100644
--- a/tests/cli/validation.test.js
+++ b/tests/cli/validation.test.js
@@ -9,6 +9,14 @@ describe('Validation', () => {
       exitCode: 1,
     },
   )
+  cliTest(
+    'Invalid subgraph manifest (cannot infer protocol)',
+    ['codegen', '--skip-migrations'],
+    'validation/invalid-manifest-cannot-infer-protocol',
+    {
+      exitCode: 1,
+    },
+  )
   cliTest(
     'ABI not found in data source',
     ['codegen', '--skip-migrations'],
diff --git a/tests/cli/validation/invalid-manifest-cannot-infer-protocol.stderr b/tests/cli/validation/invalid-manifest-cannot-infer-protocol.stderr
new file mode 100644
index 000000000..6c3bc469e
--- /dev/null
+++ b/tests/cli/validation/invalid-manifest-cannot-infer-protocol.stderr
@@ -0,0 +1,5 @@
+- Load subgraph from subgraph.yaml
+✖ Failed to load subgraph from subgraph.yaml: Error in subgraph.yaml:
+
+  Path: /
+  Unable to determine for which protocol manifest file is built for. Ensure you have at least one 'dataSources' and/or 'templates' elements defined in your subgraph.
diff --git a/tests/cli/validation/invalid-manifest-cannot-infer-protocol/subgraph.yaml b/tests/cli/validation/invalid-manifest-cannot-infer-protocol/subgraph.yaml
new file mode 100644
index 000000000..d4fa4ee08
--- /dev/null
+++ b/tests/cli/validation/invalid-manifest-cannot-infer-protocol/subgraph.yaml
@@ -0,0 +1,12 @@
+schema:
+  file: ./non-existent.grapqhl
+dataSources:
+  - name: 5
+    abis:
+      name: Foo
+    mapping:
+      - 12
+      - 13
+      - 14
+templates:
+  field: foo
diff --git a/tests/cli/validation/invalid-manifest.stderr b/tests/cli/validation/invalid-manifest.stderr
index 6c3bc469e..64cc78776 100644
--- a/tests/cli/validation/invalid-manifest.stderr
+++ b/tests/cli/validation/invalid-manifest.stderr
@@ -1,5 +1,28 @@
 - Load subgraph from subgraph.yaml
 ✖ Failed to load subgraph from subgraph.yaml: Error in subgraph.yaml:
 
-  Path: /
-  Unable to determine for which protocol manifest file is built for. Ensure you have at least one 'dataSources' and/or 'templates' elements defined in your subgraph.
+  Path: specVersion
+  No value provided
+
+  Path: schema > file
+  File does not exist: non-existent.grapqhl
+
+  Path: dataSources > 0 > name
+  Expected string, found number:
+  5
+
+  Path: dataSources > 0 > source
+  No value provided
+
+  Path: dataSources > 0 > mapping
+  Expected map, found list:
+  - 12
+  - 13
+  - 14
+
+  Path: dataSources > 0
+  Unexpected key in map: abis
+
+  Path: templates
+  Expected list, found map:
+  field: foo
diff --git a/tests/cli/validation/invalid-manifest/subgraph.yaml b/tests/cli/validation/invalid-manifest/subgraph.yaml
index d4fa4ee08..22d990eb5 100644
--- a/tests/cli/validation/invalid-manifest/subgraph.yaml
+++ b/tests/cli/validation/invalid-manifest/subgraph.yaml
@@ -1,7 +1,8 @@
 schema:
   file: ./non-existent.grapqhl
 dataSources:
-  - name: 5
+  - kind: ethereum/contract
+    name: 5
     abis:
       name: Foo
     mapping: