|
| 1 | +/** |
| 2 | + * This module checks the coverage of proxy filtering tests across different chains. |
| 3 | + * Such tests are used to ensure that proxy types: |
| 4 | + * 1. *can* make calls they are allowed to: (referred to as"allowed" tests in this module) |
| 5 | + * 2. *cannot* make calls they are forbidden from making: (referred to as "forbidden" tests) |
| 6 | + * |
| 7 | + * This [issue](https://github.com/paritytech/polkadot-ecosystem-tests/pull/266) showed that some proxy types were |
| 8 | + * not covered by proxy filtering tests, despite being present in the test module. |
| 9 | + * This was due to an oversight when building each proxy type's actions for tests. |
| 10 | + * |
| 11 | + * Due to the wide scope of the tests and the considerable size of each snapshot file, it is not effective to manually |
| 12 | + * check each proxy type's coverage. |
| 13 | + * |
| 14 | + * This script is thus used to check the coverage of proxy filtering tests for all proxy types in all chains. |
| 15 | + * |
| 16 | + * For each chain (Polkadot, Kusama, etc.), it: |
| 17 | + * 1. Finds the chain's proxy E2E test snapshot file |
| 18 | + * 2. Searches for both "allowed" and "forbidden" proxy call tests for each proxy type |
| 19 | + * 3. Reports which proxy types have tests, and which don't |
| 20 | + * |
| 21 | + * This helps ensure that all proxy types have proper test coverage for both allowed and forbidden |
| 22 | + * proxy call scenarios. |
| 23 | + */ |
| 24 | + |
| 25 | +import { createReadStream, readdirSync } from 'node:fs' |
| 26 | +import { dirname, join } from 'node:path' |
| 27 | +import * as readline from 'node:readline' |
| 28 | +import { fileURLToPath } from 'node:url' |
| 29 | +import { |
| 30 | + AssetHubProxyTypes, |
| 31 | + CollectivesProxyTypes, |
| 32 | + CoretimeProxyTypes, |
| 33 | + KusamaProxyTypes, |
| 34 | + PeopleProxyTypes, |
| 35 | + PolkadotProxyTypes, |
| 36 | + type ProxyTypeMap, |
| 37 | +} from '../packages/shared/src/helpers/proxyTypes.js' |
| 38 | + |
| 39 | +/** |
| 40 | + * When printing the results for each network's proxy type, pad the output to this length. |
| 41 | + */ |
| 42 | +const PAD_LENGTH = 40 |
| 43 | + |
| 44 | +/** |
| 45 | + * Generate a unique background color for a network's filepath when logged. |
| 46 | + * A simple hash function is used to convert the network name into a number between 16 and 231 |
| 47 | + * (the range of 6x6x6 color cube in ANSI colors). |
| 48 | + * |
| 49 | + * @param networkName The name of the network |
| 50 | + * @returns ANSI escape code for a background color |
| 51 | + */ |
| 52 | +function getNetworkBackgroundColor(networkName: string): string { |
| 53 | + // Simple hash function to get a number from a string |
| 54 | + let hash = 0 |
| 55 | + for (let i = 0; i < networkName.length; i++) { |
| 56 | + hash = (hash << 5) - hash + networkName.charCodeAt(i) |
| 57 | + } |
| 58 | + |
| 59 | + // Map the hash to a color in the ANSI 6 x 6 x 6 color cube (colors from 16 to 231). |
| 60 | + const color = (Math.abs(hash) % 216) + 16 |
| 61 | + |
| 62 | + return `\x1b[48;5;${color}m` |
| 63 | +} |
| 64 | + |
| 65 | +/** |
| 66 | + * An object with a chain's name and its proxy types. |
| 67 | + * The name used must correspond with the name of the chain's snapshot file; for example, |
| 68 | + * if Polkadot's proxy E2E test snapshots are in `polkadot.proxy.e2e.test.ts.snap`, then the name |
| 69 | + * should be `polkadot`. |
| 70 | + */ |
| 71 | +interface ChainAndProxyTypes { |
| 72 | + name: string |
| 73 | + proxyTypes: ProxyTypeMap |
| 74 | +} |
| 75 | + |
| 76 | +/** |
| 77 | + * The list of chains that currently have proxy E2E test snapshots, and their proxy types. |
| 78 | + */ |
| 79 | +const networks: ChainAndProxyTypes[] = [ |
| 80 | + { name: 'polkadot', proxyTypes: PolkadotProxyTypes }, |
| 81 | + { name: 'kusama', proxyTypes: KusamaProxyTypes }, |
| 82 | + { name: 'assetHubPolkadot', proxyTypes: AssetHubProxyTypes }, |
| 83 | + { name: 'assetHubKusama', proxyTypes: AssetHubProxyTypes }, |
| 84 | + { name: 'collectivesPolkadot', proxyTypes: CollectivesProxyTypes }, |
| 85 | + { name: 'coretimePolkadot', proxyTypes: CoretimeProxyTypes }, |
| 86 | + { name: 'coretimeKusama', proxyTypes: CoretimeProxyTypes }, |
| 87 | + { name: 'peoplePolkadot', proxyTypes: PeopleProxyTypes }, |
| 88 | + { name: 'peopleKusama', proxyTypes: PeopleProxyTypes }, |
| 89 | +] |
| 90 | + |
| 91 | +/** |
| 92 | + * Represents the test results for a single proxy type, with the status for both allowed and forbidden tests. |
| 93 | + */ |
| 94 | +type TestTypes = { |
| 95 | + allowed: string |
| 96 | + forbidden: string |
| 97 | +} |
| 98 | + |
| 99 | +/** |
| 100 | + * Result of a search for proxy filtering tests for a given chain. |
| 101 | + * |
| 102 | + * Each of the outer keys is a string representing a proxy type in that chain. |
| 103 | + * Each proxy type's corresponding value is an object containing: |
| 104 | + * - `allowed`: a message indicating whether the snapshot file contained _any_ "allowed" test for that proxy type |
| 105 | + * - `forbidden`: the same, but for forbidden tests of that proxy type |
| 106 | + * |
| 107 | + * In either case, if the test is found, the message will be `✅ (line <line number>)`, where the line number is for |
| 108 | + * _any_ of the found tests, with no guarantees on match ordinality cardinality. |
| 109 | + * If not, the message will be `❌ (not found)`. |
| 110 | + */ |
| 111 | +type SearchResult = Record<string, TestTypes> |
| 112 | + |
| 113 | +/** |
| 114 | + * Creates a new SearchResult with all proxy types initialized to "not found" status. |
| 115 | + * |
| 116 | + * @param proxyTypes The proxy types to initialize results for |
| 117 | + * @returns A SearchResult with all proxy types initialized |
| 118 | + */ |
| 119 | +function createSearchResult(proxyTypes: ProxyTypeMap): SearchResult { |
| 120 | + return Object.fromEntries( |
| 121 | + Object.keys(proxyTypes).map((proxyTypeName) => [ |
| 122 | + proxyTypeName, |
| 123 | + { |
| 124 | + allowed: `${`${proxyTypeName} allowed tests:`.padEnd(PAD_LENGTH, ' ')} ❌ (not found)`, |
| 125 | + forbidden: `${`${proxyTypeName} forbidden tests:`.padEnd(PAD_LENGTH, ' ')} ❌ (not found)`, |
| 126 | + }, |
| 127 | + ]), |
| 128 | + ) |
| 129 | +} |
| 130 | + |
| 131 | +/** |
| 132 | + * Find proxy filtering tests for all proxy types in a given chain. |
| 133 | + * The search is done in the given file, which must be an E2E test snapshot file. |
| 134 | + * |
| 135 | + * @param chain The chain whose proxy types' tests will be checked. |
| 136 | + * @param networkSnapshotFilename The path to the chain's proxy E2E test snapshot file. |
| 137 | + * @returns A promise that resolves to a record of proxy types -> their search results. |
| 138 | + */ |
| 139 | +function findProxyFilteringTests(chain: ChainAndProxyTypes, networkSnapshotFilename: string): Promise<SearchResult> { |
| 140 | + const proxyTypes = chain.proxyTypes |
| 141 | + const proxyTestResults = createSearchResult(proxyTypes) |
| 142 | + |
| 143 | + return new Promise((resolve) => { |
| 144 | + // Open the chain's snapshot file. |
| 145 | + const fileStream = readline.createInterface({ |
| 146 | + input: createReadStream(networkSnapshotFilename), |
| 147 | + crlfDelay: Number.POSITIVE_INFINITY, |
| 148 | + }) |
| 149 | + |
| 150 | + let lineNumber = 0 |
| 151 | + |
| 152 | + // For each line in the snapshot file, check if it contains any proxy filtering test for any of the proxy types. |
| 153 | + // If either test type is found for any proxy types, move on to next line, as there is no need to check the rest |
| 154 | + // of the proxy types. |
| 155 | + fileStream.on('line', (line) => { |
| 156 | + lineNumber++ |
| 157 | + for (const proxyTypeName of Object.keys(proxyTypes)) { |
| 158 | + const allowedPattern = new RegExp(`allowed proxy calls for ${proxyTypeName} `) |
| 159 | + const forbiddenPattern = new RegExp(`forbidden proxy calls for ${proxyTypeName} `) |
| 160 | + |
| 161 | + let msg: string |
| 162 | + if (allowedPattern.test(line)) { |
| 163 | + msg = `${proxyTypeName} allowed tests:` |
| 164 | + msg = msg.padEnd(PAD_LENGTH, ' ') |
| 165 | + proxyTestResults[proxyTypeName]['allowed'] = `${msg} ✅ (line ${lineNumber})` |
| 166 | + break |
| 167 | + } |
| 168 | + |
| 169 | + if (forbiddenPattern.test(line)) { |
| 170 | + msg = `${proxyTypeName} forbidden tests:` |
| 171 | + msg = msg.padEnd(PAD_LENGTH, ' ') |
| 172 | + proxyTestResults[proxyTypeName]['forbidden'] = `${msg} ✅ (line ${lineNumber})` |
| 173 | + break |
| 174 | + } |
| 175 | + } |
| 176 | + }) |
| 177 | + |
| 178 | + fileStream.on('close', () => { |
| 179 | + resolve(proxyTestResults) |
| 180 | + }) |
| 181 | + }) |
| 182 | +} |
| 183 | + |
| 184 | +/** |
| 185 | + * Recursively find all proxy E2E test snapshot files in the given directory. |
| 186 | + * |
| 187 | + * @param dir The directory to search for snapshot files. |
| 188 | + * @returns A list of paths to all proxy E2E test snapshot files. |
| 189 | + */ |
| 190 | +function findProxyTestSnapshots(dir: string): string[] { |
| 191 | + const files: string[] = [] |
| 192 | + const entries = readdirSync(dir, { withFileTypes: true }) |
| 193 | + |
| 194 | + for (const entry of entries) { |
| 195 | + const fullPath = join(dir, entry.name) |
| 196 | + if (entry.isDirectory()) { |
| 197 | + files.push(...findProxyTestSnapshots(fullPath)) |
| 198 | + } else if (entry.isFile() && entry.name.endsWith('proxy.e2e.test.ts.snap')) { |
| 199 | + files.push(fullPath) |
| 200 | + } |
| 201 | + } |
| 202 | + |
| 203 | + return files |
| 204 | +} |
| 205 | + |
| 206 | +async function main() { |
| 207 | + // This script is run from `./scripts`, so going up once leads to the root directory. |
| 208 | + const rootDir = join(dirname(fileURLToPath(import.meta.url)), '..') |
| 209 | + const snapshotFiles = findProxyTestSnapshots(rootDir) |
| 210 | + |
| 211 | + console.log('Proxy Type Test Coverage Checker') |
| 212 | + console.log('===============================') |
| 213 | + |
| 214 | + for (const network of networks) { |
| 215 | + const networkSnapshotFilename = snapshotFiles.find((file) => file.split('/').pop()?.startsWith(network.name)) |
| 216 | + if (!networkSnapshotFilename) { |
| 217 | + console.log(`No snapshots found for ${network.name}`) |
| 218 | + continue |
| 219 | + } |
| 220 | + |
| 221 | + const searchResults = await findProxyFilteringTests(network, networkSnapshotFilename) |
| 222 | + |
| 223 | + console.log(`\nProxy call filtering test coverage for network: ${network.name}`) |
| 224 | + console.log(`Snapshot filepath: ${getNetworkBackgroundColor(network.name)}${networkSnapshotFilename}\x1b[0m`) |
| 225 | + for (const [_, msgPerTestType] of Object.entries(searchResults)) { |
| 226 | + for (const [_, searchResult] of Object.entries(msgPerTestType)) { |
| 227 | + console.log(searchResult) |
| 228 | + } |
| 229 | + } |
| 230 | + } |
| 231 | +} |
| 232 | + |
| 233 | +main() |
| 234 | + .catch(console.error) |
| 235 | + .finally(() => process.exit(0)) |
0 commit comments