test: conformance testing more modular, no hangs (#266)

* test: conformance testing more modular, no hangs

* chore: fix offline helia types

* chore: fix lint errors

Author: SgtPooki

package.json

@@ -38,7 +38,7 @@
     "docs:no-publish": "aegir docs --publish false"
   },
   "devDependencies": {
-    "aegir": "^47.0.11",
+    "aegir": "^47.0.21",
     "npm-run-all": "^4.1.5"
   },
   "type": "module",

packages/gateway-conformance/.aegir.js

@@ -10,53 +10,11 @@ export default {
   },
   test: {
     files: ['./dist/src/*.spec.js'],
+    build: false,
     before: async (options) => {
       if (options.runner !== 'node') {
         throw new Error('Only node runner is supported')
       }
-
-      const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js')
-      const KUBO_PORT = await getPort(3440)
-      const SERVER_PORT = await getPort(3441)
-      // The Kubo gateway will be passed to the VerifiedFetch config
-      const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT)
-      await controller.start()
-      const { loadKuboFixtures } = await import('./dist/src/fixtures/kubo-mgmt.js')
-      const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
-      const kuboGateway = gatewayUrl
-
-      const { startVerifiedFetchGateway } = await import('./dist/src/fixtures/basic-server.js')
-      const stopBasicServer = await startVerifiedFetchGateway({
-        serverPort: SERVER_PORT,
-        kuboGateway,
-        IPFS_NS_MAP
-      }).catch((err) => {
-        log.error(err)
-      })
-
-      const CONFORMANCE_HOST = 'localhost'
-
-      return {
-        controller,
-        stopBasicServer,
-        env: {
-          IPFS_NS_MAP,
-          CONFORMANCE_HOST,
-          KUBO_PORT: `${KUBO_PORT}`,
-          SERVER_PORT: `${SERVER_PORT}`,
-          KUBO_GATEWAY: kuboGateway
-        }
-      }
-    },
-    after: async (options, beforeResult) => {
-      // @ts-expect-error - broken aegir types
-      await beforeResult.controller.stop()
-      log('controller stopped')
-
-      // @ts-expect-error - broken aegir types
-      await beforeResult.stopBasicServer()
-      log('basic server stopped')
-
     }
   }
 }

packages/gateway-conformance/README.md

@@ -68,6 +68,28 @@ $ node dist/src/demo-server.js # in terminal 1
 $ curl -v GET http://localhost:3442/ipfs/bafkqabtimvwgy3yk/  # in terminal 2
 ```
 
+## Example - Generating conformance results standalone
+
+You can generate conformance results independently of the test suite for analysis or reuse:
+
+```console
+# Generate results and keep binary for reuse
+$ npm run gen:gwc-report
+
+# Generate results with custom name and cleanup binary
+$ npm run gen:gwc-report -- my-test --cleanup
+
+# Use the results in your own code
+$ node -e "
+import { generateConformanceResults } from './dist/src/generate-conformance-report.js';
+import { getReportDetails } from './dist/src/get-report-details.js';
+const result = await generateConformanceResults('custom', { cleanupBinary: true });
+console.log('Report saved to:', result.reportPath);
+const details = await getReportDetails(result.reportPath);
+console.log('Success rate:', details.successRate);
+"
+```
+
 ## Troubleshooting
 
 ### Missing file in gateway-conformance-fixtures folder

packages/gateway-conformance/package.json

@@ -135,7 +135,9 @@
     "dep-check": "aegir dep-check",
     "doc-check": "aegir doc-check",
     "build": "aegir build",
-    "test": "aegir test -t node",
+    "test": "npm run gen:gwc-report && aegir test -t node",
+    "test:only": "aegir test -t node",
+    "gen:gwc-report": "npm run build && node dist/src/generate-conformance-report.js",
     "update": "npm run build && node dist/src/update-expected-tests.js",
     "release": "aegir release"
   },
@@ -150,7 +152,7 @@
     "@libp2p/logger": "^5.1.17",
     "@libp2p/peer-id": "^5.1.4",
     "@multiformats/dns": "^1.0.6",
-    "aegir": "^47.0.11",
+    "aegir": "^47.0.21",
     "blockstore-core": "^5.0.2",
     "datastore-core": "^10.0.2",
     "execa": "^9.5.3",

packages/gateway-conformance/src/conformance.spec.ts

@@ -1,107 +1,15 @@
 /* eslint-env mocha */
 import { access, constants } from 'node:fs/promises'
-import { homedir } from 'node:os'
-import { join } from 'node:path'
 import { prefixLogger } from '@libp2p/logger'
 import { expect } from 'aegir/chai'
-import { execa } from 'execa'
-import { Agent, setGlobalDispatcher } from 'undici'
-import { GWC_IMAGE } from './constants.js'
 import expectedFailingTests from './expected-failing-tests.json' with { type: 'json' }
 import expectedPassingTests from './expected-passing-tests.json' with { type: 'json' }
 import { getReportDetails } from './get-report-details.js'
-import { getTestsToRun } from './get-tests-to-run.js'
-import { getTestsToSkip } from './get-tests-to-skip.js'
 
 const logger = prefixLogger('gateway-conformance')
 
-function getGatewayConformanceBinaryPath (): string {
-  if (process.env.GATEWAY_CONFORMANCE_BINARY != null) {
-    return process.env.GATEWAY_CONFORMANCE_BINARY
-  }
-  const goPath = process.env.GOPATH ?? join(homedir(), 'go')
-  return join(goPath, 'bin', 'gateway-conformance')
-}
-
-function getConformanceTestArgs (name: string, gwcArgs: string[] = [], goTestArgs: string[] = []): string[] {
-  return [
-    'test',
-    `--gateway-url=http://127.0.0.1:${process.env.SERVER_PORT}`,
-    `--subdomain-url=http://${process.env.CONFORMANCE_HOST}:${process.env.SERVER_PORT}`,
-    '--verbose',
-    '--json', `gwc-report-${name}.json`,
-    ...gwcArgs,
-    '--',
-    '-timeout', '5m',
-    ...goTestArgs
-  ]
-}
-
 describe('@helia/verified-fetch - gateway conformance', function () {
-  before(async () => {
-    if (process.env.KUBO_GATEWAY == null) {
-      throw new Error('KUBO_GATEWAY env var is required')
-    }
-    if (process.env.SERVER_PORT == null) {
-      throw new Error('SERVER_PORT env var is required')
-    }
-    if (process.env.CONFORMANCE_HOST == null) {
-      throw new Error('CONFORMANCE_HOST env var is required')
-    }
-    // see https://stackoverflow.com/questions/71074255/use-custom-dns-resolver-for-any-request-in-nodejs
-    // EVERY undici/fetch request host resolves to local IP. Without this, Node.js does not resolve subdomain requests properly
-    const staticDnsAgent = new Agent({
-      connect: {
-        lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) }
-      }
-    })
-    setGlobalDispatcher(staticDnsAgent)
-  })
-
-  describe('smokeTests', () => {
-    [
-      ['basic server path request works', `http://localhost:${process.env.SERVER_PORT}/ipfs/bafkqabtimvwgy3yk`],
-      ['basic server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.SERVER_PORT}`]
-    ].forEach(([name, url]) => {
-      it(name, async () => {
-        const resp = await fetch(url)
-        expect(resp).to.be.ok()
-        expect(resp.status).to.equal(200)
-        const text = await resp.text()
-        expect(text.trim()).to.equal('hello')
-      })
-    })
-  })
-
   describe('conformance testing', () => {
-    const binaryPath = getGatewayConformanceBinaryPath()
-    before(async () => {
-      const log = logger.forComponent('before')
-      if (process.env.GATEWAY_CONFORMANCE_BINARY != null) {
-        log('Using custom gateway-conformance binary at %s', binaryPath)
-        return
-      }
-      const gwcVersion = GWC_IMAGE.split(':').pop()
-      const { stdout, stderr } = await execa('go', ['install', `github.com/ipfs/gateway-conformance/cmd/gateway-conformance@${gwcVersion}`], { reject: true })
-      log(stdout)
-      log.error(stderr)
-    })
-
-    after(async () => {
-      const log = logger.forComponent('after')
-
-      if (process.env.GATEWAY_CONFORMANCE_BINARY == null) {
-        try {
-          await execa('rm', [binaryPath])
-          log('gateway-conformance binary successfully uninstalled.')
-        } catch (error) {
-          log.error(`Error removing "${binaryPath}"`, error)
-        }
-      } else {
-        log('Not removing custom gateway-conformance binary at %s', binaryPath)
-      }
-    })
-
     /**
      * You can see what the latest success rate, passing tests, and failing tests are by running the following command:
      *
@@ -110,34 +18,26 @@ describe('@helia/verified-fetch - gateway conformance', function () {
      * ```
      */
     describe('gateway conformance', function () {
-      this.timeout(200000)
       let successRate: number
       let failingTests: string[]
       let passingTests: string[]
       const log = logger.forComponent('output:all')
 
       before(async function () {
-        const testsToSkip: string[] = getTestsToSkip()
-        const testsToRun: string[] = getTestsToRun()
-        const cancelSignal = AbortSignal.timeout(200000)
-        const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], [
-          ...(testsToRun.length > 0 ? ['-run', `${testsToRun.join('|')}`] : []),
-          ...(testsToSkip.length > 0 ? ['-skip', `${testsToSkip.join('|')}`] : [])
-        ]), { reject: false, cancelSignal })
-
-        expect(cancelSignal.aborted).to.be.false()
-
-        log(stdout)
-        log.error(stderr)
         await expect(access('gwc-report-all.json', constants.R_OK)).to.eventually.be.fulfilled()
         const results = await getReportDetails('gwc-report-all.json')
         successRate = results.successRate
         failingTests = results.failingTests
         passingTests = results.passingTests
+
         log.trace('Passing tests:')
-        passingTests.forEach((test) => { log.trace(`PASS: ${test}`) })
+        for (const test of passingTests) {
+          log.trace(`PASS: ${test}`)
+        }
         log.trace('Failing tests:')
-        failingTests.forEach((test) => { log.trace(`FAIL: ${test}`) })
+        for (const test of failingTests) {
+          log.trace(`FAIL: ${test}`)
+        }
       })
 
       for (const test of expectedPassingTests) {

packages/gateway-conformance/src/fixtures/basic-server.ts

@@ -6,7 +6,6 @@ import { dagCborHtmlPreviewPluginFactory, dirIndexHtmlPluginFactory } from '@hel
 import { logger } from '@libp2p/logger'
 import { dns } from '@multiformats/dns'
 import { MemoryBlockstore } from 'blockstore-core'
-import { Agent, setGlobalDispatcher } from 'undici'
 import { createVerifiedFetch } from './create-verified-fetch.js'
 import { getLocalDnsResolver } from './get-local-dns-resolver.js'
 import { convertFetchHeadersToNodeJsHeaders, convertNodeJsHeadersToFetchHeaders } from './header-utils.js'
@@ -174,12 +173,6 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP
 }
 
 export async function startVerifiedFetchGateway ({ kuboGateway, serverPort, IPFS_NS_MAP }: BasicServerOptions): Promise<() => Promise<void>> {
-  const staticDnsAgent = new Agent({
-    connect: {
-      lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) }
-    }
-  })
-  setGlobalDispatcher(staticDnsAgent)
   kuboGateway = kuboGateway ?? process.env.KUBO_GATEWAY
   const useSessions = process.env.USE_SESSIONS !== 'false'
 
@@ -224,16 +217,18 @@ export async function startVerifiedFetchGateway ({ kuboGateway, serverPort, IPFS
     console.log(`Basic server listening on port ${serverPort}`)
   })
 
-  return async () => {
-    log('Stopping...')
+  return async function cleanup () {
+    log('Stopping basic server...')
     await new Promise<void>((resolve, reject) => {
       // no matter what happens, we need to kill the server
       server.closeAllConnections()
       log('Closed all connections')
       server.close((err: any) => {
         if (err != null) {
+          log.error('Error closing server - %e', err)
           reject(err instanceof Error ? err : new Error(err))
         } else {
+          log('Server closed successfully')
           resolve()
         }
       })