diff --git a/README.md b/README.md index 99f46559c..047729694 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ These include: - proxy call filtering works both positively and negatively; in particular, for every proxy type in Polkadot/Kusama relay and system parachains, it is checked that: - a proxy of a given type can always execute calls which that proxy type is allowed to execute - a proxy of a given type can never execute calls that its proxy type disallowws it from running + - see the section below for more - E2E suite for vesting - normal (signed) and forced (root) vested transfers - forced (root) vesting schedule removal @@ -130,6 +131,15 @@ to roughly `1-10` blocks/second, not all scenarios are testable in practice e.g. confirmation, or the unbonding of staked funds. Consider placing such tests elsewhere, or using different tools (e.g. XCM emulator). +#### Proxy call filtering checker + +The proxy E2E test suite contains checks to proxy types' allowed and disallowed calls - for many chains. +Because these tests are extensive and hard to manually verify (the test code itself and the snapshots), there exists a +coverage checking script (`scripts/check-proxy-coverage.ts`) +It searches for allowed/forbidden call coverage for a chain's proxy types. + +Run it with `yarn check-proxy-coverage` to see which proxy types need test coverage. + ### Test Guidelines - Write network-agnostic tests where possible - Handle minor chain state changes gracefully diff --git a/package.json b/package.json index d64c64da5..45c811f57 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "test:ui": "vitest --ui", "update-env": "tsx scripts/update-env.ts", "update-known-good": "tsx scripts/update-env.ts --update-known-good", - "postinstall": "husky install" + "postinstall": "husky install", + "check-proxy-coverage": "tsx scripts/check-proxy-coverage.ts" }, "type": "module", "workspaces": [ diff --git a/packages/shared/src/helpers/proxyTypes.ts b/packages/shared/src/helpers/proxyTypes.ts index 22c63b2b4..e1ea4f353 100644 --- a/packages/shared/src/helpers/proxyTypes.ts +++ b/packages/shared/src/helpers/proxyTypes.ts @@ -1,7 +1,10 @@ +/// A map of proxy type names to their corresponding numeric values, for a given network. +export type ProxyTypeMap = Record + /** * Proxy types in the Polkadot relay chain. */ -export const PolkadotProxyTypes = { +export const PolkadotProxyTypes: ProxyTypeMap = { Any: 0, NonTransfer: 1, Governance: 2, @@ -12,7 +15,7 @@ export const PolkadotProxyTypes = { ParaRegistration: 9, } -export const KusamaProxyTypes = { +export const KusamaProxyTypes: ProxyTypeMap = { Any: 0, NonTransfer: 1, Governance: 2, @@ -25,7 +28,7 @@ export const KusamaProxyTypes = { ParaRegistration: 10, } -export const AssetHubProxyTypes = { +export const AssetHubProxyTypes: ProxyTypeMap = { Any: 0, NonTransfer: 1, CancelProxy: 2, @@ -35,7 +38,7 @@ export const AssetHubProxyTypes = { Collator: 6, } -export const CollectivesProxyTypes = { +export const CollectivesProxyTypes: ProxyTypeMap = { Any: 0, NonTransfer: 1, CancelProxy: 2, @@ -45,7 +48,7 @@ export const CollectivesProxyTypes = { Ambassador: 6, } -export const CoretimeProxyTypes = { +export const CoretimeProxyTypes: ProxyTypeMap = { Any: 0, NonTransfer: 1, CancelProxy: 2, @@ -55,7 +58,7 @@ export const CoretimeProxyTypes = { Collator: 6, } -export const PeopleProxyTypes = { +export const PeopleProxyTypes: ProxyTypeMap = { Any: 0, NonTransfer: 1, CancelProxy: 2, diff --git a/scripts/check-proxy-coverage.ts b/scripts/check-proxy-coverage.ts new file mode 100644 index 000000000..f8e788f28 --- /dev/null +++ b/scripts/check-proxy-coverage.ts @@ -0,0 +1,235 @@ +/** + * This module checks the coverage of proxy filtering tests across different chains. + * Such tests are used to ensure that proxy types: + * 1. *can* make calls they are allowed to: (referred to as"allowed" tests in this module) + * 2. *cannot* make calls they are forbidden from making: (referred to as "forbidden" tests) + * + * This [issue](https://github.com/paritytech/polkadot-ecosystem-tests/pull/266) showed that some proxy types were + * not covered by proxy filtering tests, despite being present in the test module. + * This was due to an oversight when building each proxy type's actions for tests. + * + * Due to the wide scope of the tests and the considerable size of each snapshot file, it is not effective to manually + * check each proxy type's coverage. + * + * This script is thus used to check the coverage of proxy filtering tests for all proxy types in all chains. + * + * For each chain (Polkadot, Kusama, etc.), it: + * 1. Finds the chain's proxy E2E test snapshot file + * 2. Searches for both "allowed" and "forbidden" proxy call tests for each proxy type + * 3. Reports which proxy types have tests, and which don't + * + * This helps ensure that all proxy types have proper test coverage for both allowed and forbidden + * proxy call scenarios. + */ + +import { createReadStream, readdirSync } from 'node:fs' +import { dirname, join } from 'node:path' +import * as readline from 'node:readline' +import { fileURLToPath } from 'node:url' +import { + AssetHubProxyTypes, + CollectivesProxyTypes, + CoretimeProxyTypes, + KusamaProxyTypes, + PeopleProxyTypes, + PolkadotProxyTypes, + type ProxyTypeMap, +} from '../packages/shared/src/helpers/proxyTypes.js' + +/** + * When printing the results for each network's proxy type, pad the output to this length. + */ +const PAD_LENGTH = 40 + +/** + * Generate a unique background color for a network's filepath when logged. + * A simple hash function is used to convert the network name into a number between 16 and 231 + * (the range of 6x6x6 color cube in ANSI colors). + * + * @param networkName The name of the network + * @returns ANSI escape code for a background color + */ +function getNetworkBackgroundColor(networkName: string): string { + // Simple hash function to get a number from a string + let hash = 0 + for (let i = 0; i < networkName.length; i++) { + hash = (hash << 5) - hash + networkName.charCodeAt(i) + } + + // Map the hash to a color in the ANSI 6 x 6 x 6 color cube (colors from 16 to 231). + const color = (Math.abs(hash) % 216) + 16 + + return `\x1b[48;5;${color}m` +} + +/** + * An object with a chain's name and its proxy types. + * The name used must correspond with the name of the chain's snapshot file; for example, + * if Polkadot's proxy E2E test snapshots are in `polkadot.proxy.e2e.test.ts.snap`, then the name + * should be `polkadot`. + */ +interface ChainAndProxyTypes { + name: string + proxyTypes: ProxyTypeMap +} + +/** + * The list of chains that currently have proxy E2E test snapshots, and their proxy types. + */ +const networks: ChainAndProxyTypes[] = [ + { name: 'polkadot', proxyTypes: PolkadotProxyTypes }, + { name: 'kusama', proxyTypes: KusamaProxyTypes }, + { name: 'assetHubPolkadot', proxyTypes: AssetHubProxyTypes }, + { name: 'assetHubKusama', proxyTypes: AssetHubProxyTypes }, + { name: 'collectivesPolkadot', proxyTypes: CollectivesProxyTypes }, + { name: 'coretimePolkadot', proxyTypes: CoretimeProxyTypes }, + { name: 'coretimeKusama', proxyTypes: CoretimeProxyTypes }, + { name: 'peoplePolkadot', proxyTypes: PeopleProxyTypes }, + { name: 'peopleKusama', proxyTypes: PeopleProxyTypes }, +] + +/** + * Represents the test results for a single proxy type, with the status for both allowed and forbidden tests. + */ +type TestTypes = { + allowed: string + forbidden: string +} + +/** + * Result of a search for proxy filtering tests for a given chain. + * + * Each of the outer keys is a string representing a proxy type in that chain. + * Each proxy type's corresponding value is an object containing: + * - `allowed`: a message indicating whether the snapshot file contained _any_ "allowed" test for that proxy type + * - `forbidden`: the same, but for forbidden tests of that proxy type + * + * In either case, if the test is found, the message will be `✅ (line )`, where the line number is for + * _any_ of the found tests, with no guarantees on match ordinality cardinality. + * If not, the message will be `❌ (not found)`. + */ +type SearchResult = Record + +/** + * Creates a new SearchResult with all proxy types initialized to "not found" status. + * + * @param proxyTypes The proxy types to initialize results for + * @returns A SearchResult with all proxy types initialized + */ +function createSearchResult(proxyTypes: ProxyTypeMap): SearchResult { + return Object.fromEntries( + Object.keys(proxyTypes).map((proxyTypeName) => [ + proxyTypeName, + { + allowed: `${`${proxyTypeName} allowed tests:`.padEnd(PAD_LENGTH, ' ')} ❌ (not found)`, + forbidden: `${`${proxyTypeName} forbidden tests:`.padEnd(PAD_LENGTH, ' ')} ❌ (not found)`, + }, + ]), + ) +} + +/** + * Find proxy filtering tests for all proxy types in a given chain. + * The search is done in the given file, which must be an E2E test snapshot file. + * + * @param chain The chain whose proxy types' tests will be checked. + * @param networkSnapshotFilename The path to the chain's proxy E2E test snapshot file. + * @returns A promise that resolves to a record of proxy types -> their search results. + */ +function findProxyFilteringTests(chain: ChainAndProxyTypes, networkSnapshotFilename: string): Promise { + const proxyTypes = chain.proxyTypes + const proxyTestResults = createSearchResult(proxyTypes) + + return new Promise((resolve) => { + // Open the chain's snapshot file. + const fileStream = readline.createInterface({ + input: createReadStream(networkSnapshotFilename), + crlfDelay: Number.POSITIVE_INFINITY, + }) + + let lineNumber = 0 + + // For each line in the snapshot file, check if it contains any proxy filtering test for any of the proxy types. + // If either test type is found for any proxy types, move on to next line, as there is no need to check the rest + // of the proxy types. + fileStream.on('line', (line) => { + lineNumber++ + for (const proxyTypeName of Object.keys(proxyTypes)) { + const allowedPattern = new RegExp(`allowed proxy calls for ${proxyTypeName} `) + const forbiddenPattern = new RegExp(`forbidden proxy calls for ${proxyTypeName} `) + + let msg: string + if (allowedPattern.test(line)) { + msg = `${proxyTypeName} allowed tests:` + msg = msg.padEnd(PAD_LENGTH, ' ') + proxyTestResults[proxyTypeName]['allowed'] = `${msg} ✅ (line ${lineNumber})` + break + } + + if (forbiddenPattern.test(line)) { + msg = `${proxyTypeName} forbidden tests:` + msg = msg.padEnd(PAD_LENGTH, ' ') + proxyTestResults[proxyTypeName]['forbidden'] = `${msg} ✅ (line ${lineNumber})` + break + } + } + }) + + fileStream.on('close', () => { + resolve(proxyTestResults) + }) + }) +} + +/** + * Recursively find all proxy E2E test snapshot files in the given directory. + * + * @param dir The directory to search for snapshot files. + * @returns A list of paths to all proxy E2E test snapshot files. + */ +function findProxyTestSnapshots(dir: string): string[] { + const files: string[] = [] + const entries = readdirSync(dir, { withFileTypes: true }) + + for (const entry of entries) { + const fullPath = join(dir, entry.name) + if (entry.isDirectory()) { + files.push(...findProxyTestSnapshots(fullPath)) + } else if (entry.isFile() && entry.name.endsWith('proxy.e2e.test.ts.snap')) { + files.push(fullPath) + } + } + + return files +} + +async function main() { + // This script is run from `./scripts`, so going up once leads to the root directory. + const rootDir = join(dirname(fileURLToPath(import.meta.url)), '..') + const snapshotFiles = findProxyTestSnapshots(rootDir) + + console.log('Proxy Type Test Coverage Checker') + console.log('===============================') + + for (const network of networks) { + const networkSnapshotFilename = snapshotFiles.find((file) => file.split('/').pop()?.startsWith(network.name)) + if (!networkSnapshotFilename) { + console.log(`No snapshots found for ${network.name}`) + continue + } + + const searchResults = await findProxyFilteringTests(network, networkSnapshotFilename) + + console.log(`\nProxy call filtering test coverage for network: ${network.name}`) + console.log(`Snapshot filepath: ${getNetworkBackgroundColor(network.name)}${networkSnapshotFilename}\x1b[0m`) + for (const [_, msgPerTestType] of Object.entries(searchResults)) { + for (const [_, searchResult] of Object.entries(msgPerTestType)) { + console.log(searchResult) + } + } + } +} + +main() + .catch(console.error) + .finally(() => process.exit(0))