Create script to check proxy test filtering coverage

README.md

@@ -91,6 +91,7 @@ These include:
   - proxy call filtering works both positively and negatively; in particular, for every proxy type in Polkadot/Kusama relay and system parachains, it is checked that:
       - a proxy of a given type can always execute calls which that proxy type is allowed to execute
       - a proxy of a given type can never execute calls that its proxy type disallowws it from running
+        - see the section below for more
 - E2E suite for vesting
   - normal (signed) and forced (root) vested transfers
   - forced (root) vesting schedule removal
@@ -130,6 +131,15 @@ to roughly `1-10` blocks/second, not all scenarios are testable in practice e.g.
 confirmation, or the unbonding of staked funds.
 Consider placing such tests elsewhere, or using different tools (e.g. XCM emulator).
 
+#### Proxy call filtering checker
+
+The proxy E2E test suite contains checks to proxy types' allowed and disallowed calls - for many chains.
+Because these tests are extensive and hard to manually verify (the test code itself and the snapshots), there exists a
+coverage checking script (`scripts/check-proxy-coverage.ts`)
+It searches for allowed/forbidden call coverage for a chain's proxy types.
+
+Run it with `yarn check-proxy-coverage` to see which proxy types need test coverage.
+
 ### Test Guidelines
 - Write network-agnostic tests where possible
 - Handle minor chain state changes gracefully

package.json

@@ -8,7 +8,8 @@
 		"test:ui": "vitest --ui",
 		"update-env": "tsx scripts/update-env.ts",
 		"update-known-good": "tsx scripts/update-env.ts --update-known-good",
-		"postinstall": "husky install"
+		"postinstall": "husky install",
+		"check-proxy-coverage": "tsx scripts/check-proxy-coverage.ts"
 	},
 	"type": "module",
 	"workspaces": [

packages/shared/src/helpers/proxyTypes.ts

@@ -1,7 +1,10 @@
+/// A map of proxy type names to their corresponding numeric values, for a given network.
+export type ProxyTypeMap = Record<string, number>
+
 /**
  * Proxy types in the Polkadot relay chain.
  */
-export const PolkadotProxyTypes = {
+export const PolkadotProxyTypes: ProxyTypeMap = {
   Any: 0,
   NonTransfer: 1,
   Governance: 2,
@@ -12,7 +15,7 @@ export const PolkadotProxyTypes = {
   ParaRegistration: 9,
 }
 
-export const KusamaProxyTypes = {
+export const KusamaProxyTypes: ProxyTypeMap = {
   Any: 0,
   NonTransfer: 1,
   Governance: 2,
@@ -25,7 +28,7 @@ export const KusamaProxyTypes = {
   ParaRegistration: 10,
 }
 
-export const AssetHubProxyTypes = {
+export const AssetHubProxyTypes: ProxyTypeMap = {
   Any: 0,
   NonTransfer: 1,
   CancelProxy: 2,
@@ -35,7 +38,7 @@ export const AssetHubProxyTypes = {
   Collator: 6,
 }
 
-export const CollectivesProxyTypes = {
+export const CollectivesProxyTypes: ProxyTypeMap = {
   Any: 0,
   NonTransfer: 1,
   CancelProxy: 2,
@@ -45,7 +48,7 @@ export const CollectivesProxyTypes = {
   Ambassador: 6,
 }
 
-export const CoretimeProxyTypes = {
+export const CoretimeProxyTypes: ProxyTypeMap = {
   Any: 0,
   NonTransfer: 1,
   CancelProxy: 2,
@@ -55,7 +58,7 @@ export const CoretimeProxyTypes = {
   Collator: 6,
 }
 
-export const PeopleProxyTypes = {
+export const PeopleProxyTypes: ProxyTypeMap = {
   Any: 0,
   NonTransfer: 1,
   CancelProxy: 2,

scripts/check-proxy-coverage.ts

@@ -0,0 +1,235 @@
+/**
+ * This module checks the coverage of proxy filtering tests across different chains.
+ * Such tests are used to ensure that proxy types:
+ * 1. *can* make calls they are allowed to: (referred to as"allowed" tests in this module)
+ * 2. *cannot* make calls they are forbidden from making: (referred to as "forbidden" tests)
+ *
+ * This [issue](https://github.com/paritytech/polkadot-ecosystem-tests/pull/266) showed that some proxy types were
+ * not covered by proxy filtering tests, despite being present in the test module.
+ * This was due to an oversight when building each proxy type's actions for tests.
+ *
+ * Due to the wide scope of the tests and the considerable size of each snapshot file, it is not effective to manually
+ * check each proxy type's coverage.
+ *
+ * This script is thus used to check the coverage of proxy filtering tests for all proxy types in all chains.
+ *
+ * For each chain (Polkadot, Kusama, etc.), it:
+ * 1. Finds the chain's proxy E2E test snapshot file
+ * 2. Searches for both "allowed" and "forbidden" proxy call tests for each proxy type
+ * 3. Reports which proxy types have tests, and which don't
+ *
+ * This helps ensure that all proxy types have proper test coverage for both allowed and forbidden
+ * proxy call scenarios.
+ */
+
+import { createReadStream, readdirSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import * as readline from 'node:readline'
+import { fileURLToPath } from 'node:url'
+import {
+  AssetHubProxyTypes,
+  CollectivesProxyTypes,
+  CoretimeProxyTypes,
+  KusamaProxyTypes,
+  PeopleProxyTypes,
+  PolkadotProxyTypes,
+  type ProxyTypeMap,
+} from '../packages/shared/src/helpers/proxyTypes.js'
+
+/**
+ * When printing the results for each network's proxy type, pad the output to this length.
+ */
+const PAD_LENGTH = 40
+
+/**
+ * Generate a unique background color for a network's filepath when logged.
+ * A simple hash function is used to convert the network name into a number between 16 and 231
+ * (the range of 6x6x6 color cube in ANSI colors).
+ *
+ * @param networkName The name of the network
+ * @returns ANSI escape code for a background color
+ */
+function getNetworkBackgroundColor(networkName: string): string {
+  // Simple hash function to get a number from a string
+  let hash = 0
+  for (let i = 0; i < networkName.length; i++) {
+    hash = (hash << 5) - hash + networkName.charCodeAt(i)
+  }
+
+  // Map the hash to a color in the ANSI 6 x 6 x 6 color cube (colors from 16 to 231).
+  const color = (Math.abs(hash) % 216) + 16
+
+  return `\x1b[48;5;${color}m`
+}
+
+/**
+ * An object with a chain's name and its proxy types.
+ * The name used must correspond with the name of the chain's snapshot file; for example,
+ * if Polkadot's proxy E2E test snapshots are in `polkadot.proxy.e2e.test.ts.snap`, then the name
+ * should be `polkadot`.
+ */
+interface ChainAndProxyTypes {
+  name: string
+  proxyTypes: ProxyTypeMap
+}
+
+/**
+ * The list of chains that currently have proxy E2E test snapshots, and their proxy types.
+ */
+const networks: ChainAndProxyTypes[] = [
+  { name: 'polkadot', proxyTypes: PolkadotProxyTypes },
+  { name: 'kusama', proxyTypes: KusamaProxyTypes },
+  { name: 'assetHubPolkadot', proxyTypes: AssetHubProxyTypes },
+  { name: 'assetHubKusama', proxyTypes: AssetHubProxyTypes },
+  { name: 'collectivesPolkadot', proxyTypes: CollectivesProxyTypes },
+  { name: 'coretimePolkadot', proxyTypes: CoretimeProxyTypes },
+  { name: 'coretimeKusama', proxyTypes: CoretimeProxyTypes },
+  { name: 'peoplePolkadot', proxyTypes: PeopleProxyTypes },
+  { name: 'peopleKusama', proxyTypes: PeopleProxyTypes },
+]
+
+/**
+ * Represents the test results for a single proxy type, with the status for both allowed and forbidden tests.
+ */
+type TestTypes = {
+  allowed: string
+  forbidden: string
+}
+
+/**
+ * Result of a search for proxy filtering tests for a given chain.
+ *
+ * Each of the outer keys is a string representing a proxy type in that chain.
+ * Each proxy type's corresponding value is an object containing:
+ * - `allowed`: a message indicating whether the snapshot file contained _any_ "allowed" test for that proxy type
+ * - `forbidden`: the same, but for forbidden tests of that proxy type
+ *
+ * In either case, if the test is found, the message will be `✅ (line <line number>)`, where the line number is for
+ * _any_ of the found tests, with no guarantees on match ordinality cardinality.
+ * If not, the message will be `❌ (not found)`.
+ */
+type SearchResult = Record<string, TestTypes>
+
+/**
+ * Creates a new SearchResult with all proxy types initialized to "not found" status.
+ *
+ * @param proxyTypes The proxy types to initialize results for
+ * @returns A SearchResult with all proxy types initialized
+ */
+function createSearchResult(proxyTypes: ProxyTypeMap): SearchResult {
+  return Object.fromEntries(
+    Object.keys(proxyTypes).map((proxyTypeName) => [
+      proxyTypeName,
+      {
+        allowed: `${`${proxyTypeName} allowed tests:`.padEnd(PAD_LENGTH, ' ')} ❌ (not found)`,
+        forbidden: `${`${proxyTypeName} forbidden tests:`.padEnd(PAD_LENGTH, ' ')} ❌ (not found)`,
+      },
+    ]),
+  )
+}
+
+/**
+ * Find proxy filtering tests for all proxy types in a given chain.
+ * The search is done in the given file, which must be an E2E test snapshot file.
+ *
+ * @param chain The chain whose proxy types' tests will be checked.
+ * @param networkSnapshotFilename The path to the chain's proxy E2E test snapshot file.
+ * @returns A promise that resolves to a record of proxy types -> their search results.
+ */
+function findProxyFilteringTests(chain: ChainAndProxyTypes, networkSnapshotFilename: string): Promise<SearchResult> {
+  const proxyTypes = chain.proxyTypes
+  const proxyTestResults = createSearchResult(proxyTypes)
+
+  return new Promise((resolve) => {
+    // Open the chain's snapshot file.
+    const fileStream = readline.createInterface({
+      input: createReadStream(networkSnapshotFilename),
+      crlfDelay: Number.POSITIVE_INFINITY,
+    })
+
+    let lineNumber = 0
+
+    // For each line in the snapshot file, check if it contains any proxy filtering test for any of the proxy types.
+    // If either test type is found for any proxy types, move on to next line, as there is no need to check the rest
+    // of the proxy types.
+    fileStream.on('line', (line) => {
+      lineNumber++
+      for (const proxyTypeName of Object.keys(proxyTypes)) {
+        const allowedPattern = new RegExp(`allowed proxy calls for ${proxyTypeName} `)
+        const forbiddenPattern = new RegExp(`forbidden proxy calls for ${proxyTypeName} `)
+
+        let msg: string
+        if (allowedPattern.test(line)) {
+          msg = `${proxyTypeName} allowed tests:`
+          msg = msg.padEnd(PAD_LENGTH, ' ')
+          proxyTestResults[proxyTypeName]['allowed'] = `${msg} ✅ (line ${lineNumber})`
+          break
+        }
+
+        if (forbiddenPattern.test(line)) {
+          msg = `${proxyTypeName} forbidden tests:`
+          msg = msg.padEnd(PAD_LENGTH, ' ')
+          proxyTestResults[proxyTypeName]['forbidden'] = `${msg} ✅ (line ${lineNumber})`
+          break
+        }
+      }
+    })
+
+    fileStream.on('close', () => {
+      resolve(proxyTestResults)
+    })
+  })
+}
+
+/**
+ * Recursively find all proxy E2E test snapshot files in the given directory.
+ *
+ * @param dir The directory to search for snapshot files.
+ * @returns A list of paths to all proxy E2E test snapshot files.
+ */
+function findProxyTestSnapshots(dir: string): string[] {
+  const files: string[] = []
+  const entries = readdirSync(dir, { withFileTypes: true })
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name)
+    if (entry.isDirectory()) {
+      files.push(...findProxyTestSnapshots(fullPath))
+    } else if (entry.isFile() && entry.name.endsWith('proxy.e2e.test.ts.snap')) {
+      files.push(fullPath)
+    }
+  }
+
+  return files
+}
+
+async function main() {
+  // This script is run from `./scripts`, so going up once leads to the root directory.
+  const rootDir = join(dirname(fileURLToPath(import.meta.url)), '..')
+  const snapshotFiles = findProxyTestSnapshots(rootDir)
+
+  console.log('Proxy Type Test Coverage Checker')
+  console.log('===============================')
+
+  for (const network of networks) {
+    const networkSnapshotFilename = snapshotFiles.find((file) => file.split('/').pop()?.startsWith(network.name))
+    if (!networkSnapshotFilename) {
+      console.log(`No snapshots found for ${network.name}`)
+      continue
+    }
+
+    const searchResults = await findProxyFilteringTests(network, networkSnapshotFilename)
+
+    console.log(`\nProxy call filtering test coverage for network: ${network.name}`)
+    console.log(`Snapshot filepath: ${getNetworkBackgroundColor(network.name)}${networkSnapshotFilename}\x1b[0m`)
+    for (const [_, msgPerTestType] of Object.entries(searchResults)) {
+      for (const [_, searchResult] of Object.entries(msgPerTestType)) {
+        console.log(searchResult)
+      }
+    }
+  }
+}
+
+main()
+  .catch(console.error)
+  .finally(() => process.exit(0))