diff --git a/package.json b/package.json index 5b837f43..56c67644 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "docs:no-publish": "aegir docs --publish false" }, "devDependencies": { - "aegir": "^47.0.11", + "aegir": "^47.0.21", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js index 18147f31..66184e72 100644 --- a/packages/gateway-conformance/.aegir.js +++ b/packages/gateway-conformance/.aegir.js @@ -10,53 +10,11 @@ export default { }, test: { files: ['./dist/src/*.spec.js'], + build: false, before: async (options) => { if (options.runner !== 'node') { throw new Error('Only node runner is supported') } - - const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js') - const KUBO_PORT = await getPort(3440) - const SERVER_PORT = await getPort(3441) - // The Kubo gateway will be passed to the VerifiedFetch config - const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT) - await controller.start() - const { loadKuboFixtures } = await import('./dist/src/fixtures/kubo-mgmt.js') - const IPFS_NS_MAP = await loadKuboFixtures(repoPath) - const kuboGateway = gatewayUrl - - const { startVerifiedFetchGateway } = await import('./dist/src/fixtures/basic-server.js') - const stopBasicServer = await startVerifiedFetchGateway({ - serverPort: SERVER_PORT, - kuboGateway, - IPFS_NS_MAP - }).catch((err) => { - log.error(err) - }) - - const CONFORMANCE_HOST = 'localhost' - - return { - controller, - stopBasicServer, - env: { - IPFS_NS_MAP, - CONFORMANCE_HOST, - KUBO_PORT: `${KUBO_PORT}`, - SERVER_PORT: `${SERVER_PORT}`, - KUBO_GATEWAY: kuboGateway - } - } - }, - after: async (options, beforeResult) => { - // @ts-expect-error - broken aegir types - await beforeResult.controller.stop() - log('controller stopped') - - // @ts-expect-error - broken aegir types - await beforeResult.stopBasicServer() - log('basic server stopped') - } } } diff --git a/packages/gateway-conformance/README.md b/packages/gateway-conformance/README.md index af7b4d7c..f826f2df 100644 --- a/packages/gateway-conformance/README.md +++ b/packages/gateway-conformance/README.md @@ -68,6 +68,28 @@ $ node dist/src/demo-server.js # in terminal 1 $ curl -v GET http://localhost:3442/ipfs/bafkqabtimvwgy3yk/ # in terminal 2 ``` +## Example - Generating conformance results standalone + +You can generate conformance results independently of the test suite for analysis or reuse: + +```console +# Generate results and keep binary for reuse +$ npm run gen:gwc-report + +# Generate results with custom name and cleanup binary +$ npm run gen:gwc-report -- my-test --cleanup + +# Use the results in your own code +$ node -e " +import { generateConformanceResults } from './dist/src/generate-conformance-report.js'; +import { getReportDetails } from './dist/src/get-report-details.js'; +const result = await generateConformanceResults('custom', { cleanupBinary: true }); +console.log('Report saved to:', result.reportPath); +const details = await getReportDetails(result.reportPath); +console.log('Success rate:', details.successRate); +" +``` + ## Troubleshooting ### Missing file in gateway-conformance-fixtures folder diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json index ae67a945..7075682c 100644 --- a/packages/gateway-conformance/package.json +++ b/packages/gateway-conformance/package.json @@ -135,7 +135,9 @@ "dep-check": "aegir dep-check", "doc-check": "aegir doc-check", "build": "aegir build", - "test": "aegir test -t node", + "test": "npm run gen:gwc-report && aegir test -t node", + "test:only": "aegir test -t node", + "gen:gwc-report": "npm run build && node dist/src/generate-conformance-report.js", "update": "npm run build && node dist/src/update-expected-tests.js", "release": "aegir release" }, @@ -150,7 +152,7 @@ "@libp2p/logger": "^5.1.17", "@libp2p/peer-id": "^5.1.4", "@multiformats/dns": "^1.0.6", - "aegir": "^47.0.11", + "aegir": "^47.0.21", "blockstore-core": "^5.0.2", "datastore-core": "^10.0.2", "execa": "^9.5.3", diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts index b539d550..8a85f42c 100644 --- a/packages/gateway-conformance/src/conformance.spec.ts +++ b/packages/gateway-conformance/src/conformance.spec.ts @@ -1,107 +1,15 @@ /* eslint-env mocha */ import { access, constants } from 'node:fs/promises' -import { homedir } from 'node:os' -import { join } from 'node:path' import { prefixLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' -import { execa } from 'execa' -import { Agent, setGlobalDispatcher } from 'undici' -import { GWC_IMAGE } from './constants.js' import expectedFailingTests from './expected-failing-tests.json' with { type: 'json' } import expectedPassingTests from './expected-passing-tests.json' with { type: 'json' } import { getReportDetails } from './get-report-details.js' -import { getTestsToRun } from './get-tests-to-run.js' -import { getTestsToSkip } from './get-tests-to-skip.js' const logger = prefixLogger('gateway-conformance') -function getGatewayConformanceBinaryPath (): string { - if (process.env.GATEWAY_CONFORMANCE_BINARY != null) { - return process.env.GATEWAY_CONFORMANCE_BINARY - } - const goPath = process.env.GOPATH ?? join(homedir(), 'go') - return join(goPath, 'bin', 'gateway-conformance') -} - -function getConformanceTestArgs (name: string, gwcArgs: string[] = [], goTestArgs: string[] = []): string[] { - return [ - 'test', - `--gateway-url=http://127.0.0.1:${process.env.SERVER_PORT}`, - `--subdomain-url=http://${process.env.CONFORMANCE_HOST}:${process.env.SERVER_PORT}`, - '--verbose', - '--json', `gwc-report-${name}.json`, - ...gwcArgs, - '--', - '-timeout', '5m', - ...goTestArgs - ] -} - describe('@helia/verified-fetch - gateway conformance', function () { - before(async () => { - if (process.env.KUBO_GATEWAY == null) { - throw new Error('KUBO_GATEWAY env var is required') - } - if (process.env.SERVER_PORT == null) { - throw new Error('SERVER_PORT env var is required') - } - if (process.env.CONFORMANCE_HOST == null) { - throw new Error('CONFORMANCE_HOST env var is required') - } - // see https://stackoverflow.com/questions/71074255/use-custom-dns-resolver-for-any-request-in-nodejs - // EVERY undici/fetch request host resolves to local IP. Without this, Node.js does not resolve subdomain requests properly - const staticDnsAgent = new Agent({ - connect: { - lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) } - } - }) - setGlobalDispatcher(staticDnsAgent) - }) - - describe('smokeTests', () => { - [ - ['basic server path request works', `http://localhost:${process.env.SERVER_PORT}/ipfs/bafkqabtimvwgy3yk`], - ['basic server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.SERVER_PORT}`] - ].forEach(([name, url]) => { - it(name, async () => { - const resp = await fetch(url) - expect(resp).to.be.ok() - expect(resp.status).to.equal(200) - const text = await resp.text() - expect(text.trim()).to.equal('hello') - }) - }) - }) - describe('conformance testing', () => { - const binaryPath = getGatewayConformanceBinaryPath() - before(async () => { - const log = logger.forComponent('before') - if (process.env.GATEWAY_CONFORMANCE_BINARY != null) { - log('Using custom gateway-conformance binary at %s', binaryPath) - return - } - const gwcVersion = GWC_IMAGE.split(':').pop() - const { stdout, stderr } = await execa('go', ['install', `github.com/ipfs/gateway-conformance/cmd/gateway-conformance@${gwcVersion}`], { reject: true }) - log(stdout) - log.error(stderr) - }) - - after(async () => { - const log = logger.forComponent('after') - - if (process.env.GATEWAY_CONFORMANCE_BINARY == null) { - try { - await execa('rm', [binaryPath]) - log('gateway-conformance binary successfully uninstalled.') - } catch (error) { - log.error(`Error removing "${binaryPath}"`, error) - } - } else { - log('Not removing custom gateway-conformance binary at %s', binaryPath) - } - }) - /** * You can see what the latest success rate, passing tests, and failing tests are by running the following command: * @@ -110,34 +18,26 @@ describe('@helia/verified-fetch - gateway conformance', function () { * ``` */ describe('gateway conformance', function () { - this.timeout(200000) let successRate: number let failingTests: string[] let passingTests: string[] const log = logger.forComponent('output:all') before(async function () { - const testsToSkip: string[] = getTestsToSkip() - const testsToRun: string[] = getTestsToRun() - const cancelSignal = AbortSignal.timeout(200000) - const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], [ - ...(testsToRun.length > 0 ? ['-run', `${testsToRun.join('|')}`] : []), - ...(testsToSkip.length > 0 ? ['-skip', `${testsToSkip.join('|')}`] : []) - ]), { reject: false, cancelSignal }) - - expect(cancelSignal.aborted).to.be.false() - - log(stdout) - log.error(stderr) await expect(access('gwc-report-all.json', constants.R_OK)).to.eventually.be.fulfilled() const results = await getReportDetails('gwc-report-all.json') successRate = results.successRate failingTests = results.failingTests passingTests = results.passingTests + log.trace('Passing tests:') - passingTests.forEach((test) => { log.trace(`PASS: ${test}`) }) + for (const test of passingTests) { + log.trace(`PASS: ${test}`) + } log.trace('Failing tests:') - failingTests.forEach((test) => { log.trace(`FAIL: ${test}`) }) + for (const test of failingTests) { + log.trace(`FAIL: ${test}`) + } }) for (const test of expectedPassingTests) { diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts index 69bb89eb..2e923bc6 100644 --- a/packages/gateway-conformance/src/fixtures/basic-server.ts +++ b/packages/gateway-conformance/src/fixtures/basic-server.ts @@ -6,7 +6,6 @@ import { dagCborHtmlPreviewPluginFactory, dirIndexHtmlPluginFactory } from '@hel import { logger } from '@libp2p/logger' import { dns } from '@multiformats/dns' import { MemoryBlockstore } from 'blockstore-core' -import { Agent, setGlobalDispatcher } from 'undici' import { createVerifiedFetch } from './create-verified-fetch.js' import { getLocalDnsResolver } from './get-local-dns-resolver.js' import { convertFetchHeadersToNodeJsHeaders, convertNodeJsHeadersToFetchHeaders } from './header-utils.js' @@ -174,12 +173,6 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP } export async function startVerifiedFetchGateway ({ kuboGateway, serverPort, IPFS_NS_MAP }: BasicServerOptions): Promise<() => Promise> { - const staticDnsAgent = new Agent({ - connect: { - lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) } - } - }) - setGlobalDispatcher(staticDnsAgent) kuboGateway = kuboGateway ?? process.env.KUBO_GATEWAY const useSessions = process.env.USE_SESSIONS !== 'false' @@ -224,16 +217,18 @@ export async function startVerifiedFetchGateway ({ kuboGateway, serverPort, IPFS console.log(`Basic server listening on port ${serverPort}`) }) - return async () => { - log('Stopping...') + return async function cleanup () { + log('Stopping basic server...') await new Promise((resolve, reject) => { // no matter what happens, we need to kill the server server.closeAllConnections() log('Closed all connections') server.close((err: any) => { if (err != null) { + log.error('Error closing server - %e', err) reject(err instanceof Error ? err : new Error(err)) } else { + log('Server closed successfully') resolve() } }) diff --git a/packages/gateway-conformance/src/generate-conformance-report.ts b/packages/gateway-conformance/src/generate-conformance-report.ts new file mode 100644 index 00000000..4ff26550 --- /dev/null +++ b/packages/gateway-conformance/src/generate-conformance-report.ts @@ -0,0 +1,242 @@ +#!/usr/bin/env node +import { access, constants } from 'node:fs/promises' +import { homedir } from 'node:os' +import { join } from 'node:path' +import { prefixLogger } from '@libp2p/logger' +import getPort from 'aegir/get-port' +import { execa } from 'execa' +import { Agent, setGlobalDispatcher } from 'undici' +import { GWC_IMAGE } from './constants.ts' +import { startVerifiedFetchGateway } from './fixtures/basic-server.ts' +import { createKuboNode } from './fixtures/create-kubo.ts' +import { loadKuboFixtures } from './fixtures/kubo-mgmt.ts' +import { getTestsToRun } from './get-tests-to-run.ts' +import { getTestsToSkip } from './get-tests-to-skip.ts' + +const logger = prefixLogger('gateway-conformance') +const log = logger.forComponent('generate-conformance-report') +const KUBO_PORT = await getPort(3440) +process.env.KUBO_PORT = `${KUBO_PORT}` +const SERVER_PORT = await getPort(3441) +process.env.SERVER_PORT = `${SERVER_PORT}` +process.env.CONFORMANCE_HOST = 'localhost' + +async function runSmokeTests (): Promise { + const log = logger.forComponent('smoke-tests') + + const testUrls = [ + { name: 'basic server path request', url: `http://localhost:${SERVER_PORT}/ipfs/bafkqabtimvwgy3yk` }, + { name: 'basic server subdomain request', url: `http://bafkqabtimvwgy3yk.ipfs.localhost:${SERVER_PORT}` } + ] + + for (const { name, url } of testUrls) { + log(`Running smoke test: ${name}`) + try { + const resp = await fetch(url) + if (!resp.ok) { + throw new Error(`HTTP ${resp.status}: ${resp.statusText}`) + } + if (resp.status !== 200) { + throw new Error(`Expected status 200, got ${resp.status}`) + } + const text = await resp.text() + if (text.trim() !== 'hello') { + throw new Error(`Expected "hello", got "${text.trim()}"`) + } + log(`✓ ${name} passed`) + } catch (error) { + log.error(`✗ ${name} failed:`, error) + throw error + } + } + + log('All smoke tests passed!') +} + +export interface ConformanceGenerationReport { + reportPath: string + stdout: string + stderr: string +} + +function getGatewayConformanceBinaryPath (): string { + if (process.env.GATEWAY_CONFORMANCE_BINARY != null) { + return process.env.GATEWAY_CONFORMANCE_BINARY + } + const goPath = process.env.GOPATH ?? join(homedir(), 'go') + return join(goPath, 'bin', 'gateway-conformance') +} + +function getConformanceTestArgs (name: string, gwcArgs: string[] = [], goTestArgs: string[] = []): string[] { + return [ + 'test', + `--gateway-url=http://127.0.0.1:${process.env.SERVER_PORT}`, + `--subdomain-url=http://${process.env.CONFORMANCE_HOST}:${process.env.SERVER_PORT}`, + '--verbose', + '--json', `gwc-report-${name}.json`, + ...gwcArgs, + '--', + '-timeout', '5m', + ...goTestArgs + ] +} + +async function installBinary (binaryPath: string): Promise { + const log = logger.forComponent('install-binary') + + if (process.env.GATEWAY_CONFORMANCE_BINARY != null) { + log('Using custom gateway-conformance binary at %s', binaryPath) + return + } + + const gwcVersion = GWC_IMAGE.split(':').pop() + const { stdout, stderr } = await execa('go', ['install', `github.com/ipfs/gateway-conformance/cmd/gateway-conformance@${gwcVersion}`], { reject: true }) + log(stdout) + log.error(stderr) +} + +async function cleanupBinary (binaryPath: string): Promise { + const log = logger.forComponent('cleanup-binary') + + if (process.env.GATEWAY_CONFORMANCE_BINARY == null) { + try { + await execa('rm', [binaryPath]) + log('gateway-conformance binary successfully uninstalled.') + } catch (error) { + log.error(`Error removing "${binaryPath}"`, error) + } + } else { + log('Not removing custom gateway-conformance binary at %s', binaryPath) + } +} + +export async function generateConformanceResults ( + name: string = 'all', + options: { + installBinary?: boolean + cleanupBinary?: boolean + gwcArgs?: string[] + goTestArgs?: string[] + timeout?: number + } = {} +): Promise { + const { + installBinary: shouldInstall = true, + cleanupBinary: shouldCleanup = false, + gwcArgs = [], + goTestArgs = [], + timeout = 200000 + } = options + + const binaryPath = getGatewayConformanceBinaryPath() + let globalAgent: Agent | null = null + // The Kubo gateway will be passed to the VerifiedFetch config + const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT) + await controller.start() + const IPFS_NS_MAP = await loadKuboFixtures(repoPath) + + const stopBasicServer = await startVerifiedFetchGateway({ + serverPort: SERVER_PORT, + kuboGateway: gatewayUrl, + IPFS_NS_MAP + }) + process.env.KUBO_GATEWAY = gatewayUrl + process.env.IPFS_NS_MAP = IPFS_NS_MAP + + try { + // install binary if requested + if (shouldInstall) { + await installBinary(binaryPath) + } + + // see https://stackoverflow.com/questions/71074255/use-custom-dns-resolver-for-any-request-in-nodejs + // EVERY undici/fetch request host resolves to local IP. Without this, Node.js does not resolve subdomain requests properly + globalAgent = new Agent({ + connect: { + lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) } + } + }) + setGlobalDispatcher(globalAgent) + + // Run smoke tests if requested + log('Running smoke tests to ensure basic server is working before conformance tests...') + await runSmokeTests() + + if (process.env.SERVER_PORT == null) { + throw new Error('SERVER_PORT env var is required') + } + if (process.env.CONFORMANCE_HOST == null) { + throw new Error('CONFORMANCE_HOST env var is required') + } + + // Get test configuration + const testsToSkip: string[] = getTestsToSkip() + const testsToRun: string[] = getTestsToRun() + + // Build test arguments + const testArgs = [ + ...(testsToRun.length > 0 ? ['-run', `${testsToRun.join('|')}`] : []), + ...(testsToSkip.length > 0 ? ['-skip', `${testsToSkip.join('|')}`] : []), + ...goTestArgs + ] + + // Run conformance tests + const reportPath = `gwc-report-${name}.json` + const cancelSignal = AbortSignal.timeout(timeout) + + const { stderr, stdout } = await execa( + binaryPath, + getConformanceTestArgs(name, gwcArgs, testArgs), + { reject: false, cancelSignal } + ) + + if (cancelSignal.aborted) { + throw new Error('Conformance tests timed out') + } + + log(stdout) + log.error(stderr) + + // Verify report was generated + await access(reportPath, constants.R_OK) + log('report generated and file exists') + + return { + reportPath, + stdout, + stderr + } + } finally { + // Cleanup binary if requested + if (shouldCleanup) { + await cleanupBinary(binaryPath) + } + await globalAgent?.close() + await stopBasicServer() + await controller.stop() + } +} + +// CLI interface for standalone usage +if (import.meta.url === `file://${process.argv[1]}`) { + const name = process.argv[2] ?? 'all' + const shouldCleanup = process.argv.includes('--cleanup') + + generateConformanceResults(name, { cleanupBinary: shouldCleanup }) + .then((result) => { + log(`Report generated: ${result.reportPath}`) + log('stdout:', result.stdout) + if (result.stderr) { + log('stderr:', result.stderr) + } + }) + .catch((error) => { + log.error('Failed to generate conformance results - %e', error) + process.exit(1) + }).finally(() => { + setTimeout(() => { + log('killing process that would have hung') + process.exit(0) + }, 1000) + }) +} diff --git a/packages/interop/package.json b/packages/interop/package.json index d2d4be2d..2dfe1306 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -146,7 +146,7 @@ "dependencies": { "@helia/delegated-routing-v1-http-api-server": "^4.0.6", "@helia/verified-fetch": "^2.0.0", - "aegir": "^47.0.11", + "aegir": "^47.0.21", "execa": "^9.5.3", "glob": "^11.0.2", "ipfsd-ctl": "^15.0.2", diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index ddeaa242..32a55e58 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -206,7 +206,7 @@ "@helia/json": "^4.0.5", "@libp2p/crypto": "^5.1.3", "@types/sinon": "^17.0.4", - "aegir": "^47.0.11", + "aegir": "^47.0.21", "blockstore-core": "^5.0.2", "browser-readablestream-to-it": "^2.0.9", "datastore-core": "^10.0.2", diff --git a/packages/verified-fetch/test/fixtures/create-offline-helia.ts b/packages/verified-fetch/test/fixtures/create-offline-helia.ts index 27dc0543..a89c205c 100644 --- a/packages/verified-fetch/test/fixtures/create-offline-helia.ts +++ b/packages/verified-fetch/test/fixtures/create-offline-helia.ts @@ -2,9 +2,9 @@ import { createHeliaHTTP } from '@helia/http' import { MemoryBlockstore } from 'blockstore-core' import { IdentityBlockstore } from 'blockstore-core/identity' import { MemoryDatastore } from 'datastore-core' -import type { HeliaInit } from 'helia' +import type { HeliaHTTPInit } from '@helia/http' -export async function createHelia (init: Partial = {}): Promise> { +export async function createHelia (init: Partial = {}): Promise> { const datastore = new MemoryDatastore() const blockstore = new IdentityBlockstore(new MemoryBlockstore())