diff --git a/package.json b/package.json index 478f334..40bd255 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "devDependencies": { "@ai-sdk/provider": "catalog:dev", "@ai-sdk/provider-utils": "catalog:dev", + "@fast-check/vitest": "catalog:dev", "@hono/mcp": "catalog:dev", "@types/node": "catalog:dev", "@typescript/native-preview": "catalog:dev", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5fc647..ea530fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,9 @@ catalogs: '@clack/prompts': specifier: ^0.11.0 version: 0.11.0 + '@fast-check/vitest': + specifier: ^0.2.0 + version: 0.2.4 '@hono/mcp': specifier: ^0.1.4 version: 0.1.5 @@ -129,6 +132,9 @@ importers: '@ai-sdk/provider-utils': specifier: catalog:dev version: 3.0.18(zod@4.1.13) + '@fast-check/vitest': + specifier: catalog:dev + version: 0.2.4(vitest@4.0.15(@opentelemetry/api@1.9.0)(@types/node@22.19.1)(jiti@2.6.1)(msw@2.12.3(@types/node@22.19.1)(typescript@5.9.3))(tsx@4.21.0)(yaml@2.8.2)) '@hono/mcp': specifier: catalog:dev version: 0.1.5(@modelcontextprotocol/sdk@1.24.3(zod@4.1.13))(hono@4.10.7) @@ -621,6 +627,11 @@ packages: cpu: [x64] os: [win32] + '@fast-check/vitest@0.2.4': + resolution: {integrity: sha512-Ilcr+JAIPhb1s6FRm4qoglQYSGXXrS+zAupZeNuWAA3qHVGDA1d1Gb84Hb/+otL3GzVZjFJESg5/1SfIvrgssA==} + peerDependencies: + vitest: ^1 || ^2 || ^3 || ^4 + '@hono/mcp@0.1.5': resolution: {integrity: sha512-q6Yurx9VUwVEpqnwVXtzIYaq4kgQgWWq9lYLM7NFS2W0sg1RzL+RdKh6jO4/dGyvBLKrahPd2v+NC6rr0XWBvQ==} peerDependencies: @@ -1610,6 +1621,10 @@ packages: resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} engines: {node: '>= 18'} + fast-check@4.5.2: + resolution: {integrity: sha512-tOzL01LMrDIWPLfvMiGUMH0AjqnOelHQPmgvYkW/aRO4Yaw+pBQqWmyebNzAEbKOigoCN8HkRWUZXFkjmiaXMQ==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2130,6 +2145,9 @@ packages: engines: {node: '>=18'} hasBin: true + pure-rand@7.0.1: + resolution: {integrity: sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==} + qs@6.14.0: resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==} engines: {node: '>=0.6'} @@ -2809,6 +2827,11 @@ snapshots: '@esbuild/win32-x64@0.27.0': optional: true + '@fast-check/vitest@0.2.4(vitest@4.0.15(@opentelemetry/api@1.9.0)(@types/node@22.19.1)(jiti@2.6.1)(msw@2.12.3(@types/node@22.19.1)(typescript@5.9.3))(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + fast-check: 4.5.2 + vitest: 4.0.15(@opentelemetry/api@1.9.0)(@types/node@22.19.1)(jiti@2.6.1)(msw@2.12.3(@types/node@22.19.1)(typescript@5.9.3))(tsx@4.21.0)(yaml@2.8.2) + '@hono/mcp@0.1.5(@modelcontextprotocol/sdk@1.24.3(zod@4.1.13))(hono@4.10.7)': dependencies: '@modelcontextprotocol/sdk': 1.24.3(zod@4.1.13) @@ -3626,6 +3649,10 @@ snapshots: transitivePeerDependencies: - supports-color + fast-check@4.5.2: + dependencies: + pure-rand: 7.0.1 + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: @@ -4046,6 +4073,8 @@ snapshots: picocolors: 1.1.1 sade: 1.8.1 + pure-rand@7.0.1: {} + qs@6.14.0: dependencies: side-channel: 1.1.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b90558e..76d9e30 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,7 @@ catalogs: dev: '@ai-sdk/openai': ^2.0.80 '@ai-sdk/provider': ^2.0.0 + '@fast-check/vitest': ^0.2.0 '@ai-sdk/provider-utils': ^3.0.18 '@clack/prompts': ^0.11.0 '@hono/mcp': ^0.1.4 diff --git a/src/requestBuilder.test.ts b/src/requestBuilder.test.ts index 8996d5e..4f4a52f 100644 --- a/src/requestBuilder.test.ts +++ b/src/requestBuilder.test.ts @@ -1,3 +1,4 @@ +import { fc, test as fcTest } from '@fast-check/vitest'; import { http, HttpResponse } from 'msw'; import { server } from '../mocks/node'; import { type HttpExecuteConfig, type JsonObject, ParameterLocation } from './types'; @@ -307,23 +308,6 @@ describe('RequestBuilder', () => { }); describe('Security and Performance Improvements', () => { - it('should throw error when recursion depth limit is exceeded', async () => { - // Create a deeply nested object that exceeds the default depth limit of 10 - let deepObject: JsonObject = { value: 'test' }; - for (let i = 0; i < 12; i++) { - deepObject = { nested: deepObject }; - } - - const params = { - pathParam: 'test-value', - deepFilter: deepObject, - } satisfies JsonObject; - - await expect(builder.execute(params, { dryRun: true })).rejects.toThrow( - 'Maximum nesting depth (10) exceeded for parameter serialization', - ); - }); - it('should throw error when circular reference is detected', async () => { // Test runtime behavior when circular reference is passed // Note: This tests error handling for malformed input at runtime @@ -341,20 +325,6 @@ describe('RequestBuilder', () => { ); }); - it('should validate parameter keys and reject invalid characters', async () => { - const params = { - pathParam: 'test-value', - filter: { - valid_key: 'test', - 'invalid key with spaces': 'test', // Should trigger validation error - }, - }; - - await expect(builder.execute(params, { dryRun: true })).rejects.toThrow( - 'Invalid parameter key: invalid key with spaces', - ); - }); - it('should handle special types correctly at runtime', async () => { // Test runtime behavior when non-JSON types are passed // Note: Date and RegExp are not valid JsonValue types, but we test @@ -522,3 +492,287 @@ describe('RequestBuilder', () => { }); }); }); + +/** + * Property-Based Tests for RequestBuilder + * + * These tests verify invariants that must hold for ANY valid input, + * replacing/supplementing example-based tests: + * + * Parameter Key Validation (replaces "should validate parameter keys and reject invalid characters"): + * - Valid: "user_id", "filter.name", "x-custom-field" => accepted + * - Invalid: "invalid key with spaces", "key@special!" => throws "Invalid parameter key" + * + * Value Serialization (supplements "should handle arrays correctly within objects" - kept for clarity): + * - { arrayField: [1, 2, 3] } => filter[arrayField]="[1,2,3]" + * - { stringArray: ["a", "b"] } => filter[stringArray]='["a","b"]' + * + * Deep Object Nesting (replaces "should throw error when recursion depth limit is exceeded"): + * - { nested: { nested: { value: "ok" } } } (depth 3) => accepted + * - { nested: { nested: { ... 12 levels ... } } } => throws "Maximum nesting depth (10) exceeded" + */ +describe('RequestBuilder - Property-Based Tests', () => { + const baseConfig = { + kind: 'http', + method: 'GET', + url: 'https://api.example.com/test', + bodyType: 'json', + params: [{ name: 'filter', location: ParameterLocation.QUERY, type: 'object' }], + } satisfies HttpExecuteConfig; + + // Arbitrary for valid parameter keys (alphanumeric, underscore, dot, hyphen) + const validKeyArbitrary = fc.stringMatching(/^[a-zA-Z][a-zA-Z0-9_.-]{0,19}$/); + + // Arbitrary for invalid parameter keys (contains spaces or special chars) + const invalidKeyArbitrary = fc + .string({ minLength: 1, maxLength: 20 }) + .filter((s) => /[^a-zA-Z0-9_.-]/.test(s) && s.trim().length > 0); + + /** + * Parameter Key Validation + * + * Examples of valid keys: "user_id", "filter.name", "X-Custom-Header" + * Examples of invalid keys: "invalid key", "special@char", "has spaces" + */ + describe('Parameter Key Validation', () => { + // Example: { filter: { user_id: "123" } } => ?filter[user_id]=123 (no error) + fcTest.prop([validKeyArbitrary, fc.string()], { numRuns: 100 })( + 'accepts valid parameter keys', + async (key, value) => { + const builder = new RequestBuilder(baseConfig); + const params = { + filter: { [key]: value }, + }; + + // Should not throw for valid keys + const result = await builder.execute(params, { dryRun: true }); + expect(result.url).toBeDefined(); + }, + ); + + // Example: { filter: { "invalid key with spaces": "test" } } => throws Error + fcTest.prop([invalidKeyArbitrary, fc.string()], { numRuns: 100 })( + 'rejects invalid parameter keys', + async (key, value) => { + const builder = new RequestBuilder(baseConfig); + const params = { + filter: { [key]: value }, + }; + + await expect(builder.execute(params, { dryRun: true })).rejects.toThrow( + /Invalid parameter key/, + ); + }, + ); + }); + + /** + * Header Management + * + * Examples: + * - new RequestBuilder(config, { "Auth": "token" }).setHeaders({ "X-Api": "key" }) + * => getHeaders() returns { "Auth": "token", "X-Api": "key" } + * - prepareHeaders() always includes "User-Agent: stackone-ai-node" + */ + describe('Header Management', () => { + // Arbitrary for header key-value pairs + const headerArbitrary = fc.dictionary( + fc.stringMatching(/^[a-zA-Z][a-zA-Z0-9-]{0,29}$/), + fc.string({ minLength: 1, maxLength: 100 }), + { minKeys: 1, maxKeys: 5 }, + ); + + // Example: init with {"A": "1"}, setHeaders({"B": "2"}) => {"A": "1", "B": "2"} + fcTest.prop([headerArbitrary, headerArbitrary], { numRuns: 50 })( + 'setHeaders accumulates headers without losing existing ones', + (headers1, headers2) => { + const builder = new RequestBuilder(baseConfig, headers1); + builder.setHeaders(headers2); + + const result = builder.getHeaders(); + + // All headers from both sets should be present (with headers2 overriding duplicates) + for (const [key, value] of Object.entries(headers2)) { + expect(result[key]).toBe(value); + } + for (const [key, value] of Object.entries(headers1)) { + if (!(key in headers2)) { + expect(result[key]).toBe(value); + } + } + }, + ); + + // Example: prepareHeaders() => { "User-Agent": "stackone-ai-node", ...customHeaders } + fcTest.prop([headerArbitrary], { numRuns: 50 })( + 'prepareHeaders always includes User-Agent', + (headers) => { + const builder = new RequestBuilder(baseConfig, headers); + const prepared = builder.prepareHeaders(); + + expect(prepared['User-Agent']).toBe('stackone-ai-node'); + }, + ); + + // Example: const h = getHeaders(); h["X"] = "Y"; getHeaders()["X"] is still undefined + fcTest.prop([headerArbitrary], { numRuns: 50 })( + 'getHeaders returns a copy, not the original', + (headers) => { + const builder = new RequestBuilder(baseConfig, headers); + const retrieved = builder.getHeaders(); + + // Mutating the returned object should not affect internal state + retrieved['Mutated-Header'] = 'mutated'; + + expect(builder.getHeaders()['Mutated-Header']).toBeUndefined(); + }, + ); + }); + + /** + * Value Serialization + * + * Examples: + * - { key: "hello" } => ?filter[key]=hello + * - { key: 42 } => ?filter[key]=42 + * - { key: true } => ?filter[key]=true + * - { key: [1, 2, 3] } => ?filter[key]=[1,2,3] + * - { key: ["a", "b"] } => ?filter[key]=["a","b"] + */ + describe('Value Serialization', () => { + // Example: { filter: { key: "hello world" } } => ?filter[key]=hello%20world + fcTest.prop([fc.string()], { numRuns: 100 })( + 'string values serialize to themselves', + async (str) => { + const builder = new RequestBuilder(baseConfig); + const params = { filter: { key: str } }; + + const result = await builder.execute(params, { dryRun: true }); + const url = new URL(result.url as string); + + expect(url.searchParams.get('filter[key]')).toBe(str); + }, + ); + + // Example: { filter: { key: 42 } } => ?filter[key]=42 + fcTest.prop([fc.integer()], { numRuns: 100 })( + 'integer values serialize to string', + async (num) => { + const builder = new RequestBuilder(baseConfig); + const params = { filter: { key: num } }; + + const result = await builder.execute(params, { dryRun: true }); + const url = new URL(result.url as string); + + expect(url.searchParams.get('filter[key]')).toBe(String(num)); + }, + ); + + // Example: { filter: { key: true } } => ?filter[key]=true + fcTest.prop([fc.boolean()], { numRuns: 10 })( + 'boolean values serialize to string', + async (bool) => { + const builder = new RequestBuilder(baseConfig); + const params = { filter: { key: bool } }; + + const result = await builder.execute(params, { dryRun: true }); + const url = new URL(result.url as string); + + expect(url.searchParams.get('filter[key]')).toBe(String(bool)); + }, + ); + + // Example: { filter: { key: [1, 2, 3] } } => ?filter[key]=[1,2,3] + // Example: { filter: { key: ["a", "b"] } } => ?filter[key]=["a","b"] + fcTest.prop( + [fc.array(fc.oneof(fc.string(), fc.integer(), fc.boolean()), { minLength: 1, maxLength: 5 })], + { + numRuns: 50, + }, + )('arrays serialize to JSON string', async (arr) => { + const builder = new RequestBuilder(baseConfig); + const params = { filter: { key: arr } }; + + const result = await builder.execute(params, { dryRun: true }); + const url = new URL(result.url as string); + + expect(url.searchParams.get('filter[key]')).toBe(JSON.stringify(arr)); + }); + }); + + /** + * Deep Object Nesting + * + * Examples: + * - { nested: { value: "ok" } } (depth 2) => accepted + * - { a: { b: { c: { d: { e: { f: { g: { h: { i: { j: { k: "too deep" } } } } } } } } } } } + * (depth 11) => throws "Maximum nesting depth (10) exceeded" + */ + describe('Deep Object Nesting', () => { + // Example: depth 5 => { nested: { nested: { nested: { nested: { nested: { value: "test" } } } } } } + fcTest.prop([fc.integer({ min: 1, max: 9 })], { numRuns: 20 })( + 'accepts objects within depth limit', + async (depth) => { + const builder = new RequestBuilder(baseConfig); + const deepObject = {}; + let current: Record = deepObject; + for (let i = 0; i < depth; i++) { + current.nested = {}; + current = current.nested as Record; + } + current.value = 'test'; + + const params = { filter: deepObject } as JsonObject; + const result = await builder.execute(params, { dryRun: true }); + + expect(result.url).toBeDefined(); + }, + ); + + // Example: 12 levels of nesting => throws error + test('rejects objects exceeding depth limit of 10', async () => { + const builder = new RequestBuilder(baseConfig); + let deepObject: Record = { value: 'test' }; + for (let i = 0; i < 12; i++) { + deepObject = { nested: deepObject }; + } + + const params = { filter: deepObject } as JsonObject; + + await expect(builder.execute(params, { dryRun: true })).rejects.toThrow( + /Maximum nesting depth.*exceeded/, + ); + }); + }); + + /** + * Body Type Handling + * + * Examples: + * - bodyType: "json" => Content-Type: application/json, body: '{"test":"value"}' + * - bodyType: "form" => Content-Type: application/x-www-form-urlencoded, body: "test=value" + * - bodyType: "multipart-form" => body is FormData instance + */ + describe('Body Type Handling', () => { + const bodyTypes = ['json', 'form', 'multipart-form'] as const; + + // Example: buildFetchOptions({ test: "value" }) with bodyType "json" => valid options + fcTest.prop( + [ + fc.constantFrom(...bodyTypes), + fc.dictionary(fc.string(), fc.string(), { minKeys: 1, maxKeys: 3 }), + ], + { + numRuns: 30, + }, + )('all valid body types produce valid fetch options', (bodyType, bodyParams) => { + const config = { ...baseConfig, bodyType }; + const builder = new RequestBuilder(config); + + const options = builder.buildFetchOptions(bodyParams); + + expect(options.method).toBe('GET'); + expect(options.body).toBeDefined(); + }); + }); +}); diff --git a/src/utils/array.test.ts b/src/utils/array.test.ts index 254ea0c..f2d6f9b 100644 --- a/src/utils/array.test.ts +++ b/src/utils/array.test.ts @@ -1,16 +1,48 @@ +import { fc, test as fcTest } from '@fast-check/vitest'; import { toArray } from './array'; -describe('toArray', () => { - it.each([ - [undefined, []], - [null, []], - [false, [false]], - [0, [0]], - ['', ['']], - [[], []], - ['foo', ['foo']], - [['foo'], ['foo']], - ])('%s => %s', (input, expected) => { - expect(toArray(input)).toEqual(expected); +/** + * Property-Based Tests for toArray utility + * + * These tests verify the function's behavior for ANY valid input, + * replacing example-based tests like: + * + * - toArray([1, 2, 3]) === [1, 2, 3] (same reference) + * - toArray("hello") === ["hello"] + * - toArray(42) === [42] + * - toArray({ key: "value" }) === [{ key: "value" }] + * - toArray(null) === [] + * - toArray(undefined) === [] + */ +describe('toArray - Property-Based Tests', () => { + // Example: toArray([1, 2, 3]) returns the exact same array instance, not a copy + fcTest.prop([fc.array(fc.anything())], { numRuns: 100 })( + 'array input returns the same array reference', + (arr) => { + expect(toArray(arr)).toBe(arr); + }, + ); + + // Example: toArray("hello") => ["hello"], toArray(42) => [42], toArray({a:1}) => [{a:1}] + fcTest.prop([fc.anything().filter((x) => !Array.isArray(x) && x != null)], { numRuns: 100 })( + 'non-array non-nullish input returns single-element array', + (value) => { + const result = toArray(value); + expect(result).toHaveLength(1); + expect(result[0]).toBe(value); + }, + ); + + // Example: toArray(null) => [], toArray(undefined) => [] + fcTest.prop([fc.constantFrom(null, undefined)], { numRuns: 10 })( + 'null or undefined returns empty array', + (value) => { + expect(toArray(value)).toEqual([]); + }, + ); + + // Invariant: no matter what input, output is always an array + fcTest.prop([fc.anything()], { numRuns: 100 })('result is always an array', (value) => { + expect(Array.isArray(toArray(value))).toBe(true); }); }); diff --git a/src/utils/tfidf-index.test.ts b/src/utils/tfidf-index.test.ts index 685ca5c..0ca003a 100644 --- a/src/utils/tfidf-index.test.ts +++ b/src/utils/tfidf-index.test.ts @@ -1,3 +1,4 @@ +import { fc, test as fcTest } from '@fast-check/vitest'; import { TfidfIndex } from './tfidf-index'; describe('TF-IDF Index - Core Functionality', () => { @@ -31,115 +32,20 @@ describe('TF-IDF Index - Core Functionality', () => { expect(result?.score ?? 0).toBeGreaterThan(0); }); - test('returns no matches when query shares no terms with the corpus', () => { - const index = new TfidfIndex(); - index.build([ - { id: 'doc1', text: 'generate billing statement' }, - { id: 'doc2', text: 'update user profile' }, - ]); - - const results = index.search('predict weather forecast'); - - expect(results).toHaveLength(0); - }); -}); - -describe('TF-IDF Index - Score Validation', () => { - test('returns scores within [0, 1] range', () => { - const index = new TfidfIndex(); - index.build([ - { id: 'doc1', text: 'employee management system' }, - { id: 'doc2', text: 'employee database records' }, - { id: 'doc3', text: 'candidate tracking application' }, - ]); - - const results = index.search('employee', 10); - - expect(results.length).toBeGreaterThan(0); - for (const result of results) { - expect(result.score).toBeGreaterThanOrEqual(0); - expect(result.score).toBeLessThanOrEqual(1); - } - }); - - test('sorts results by score in descending order', () => { - const index = new TfidfIndex(); - index.build([ - { id: 'doc1', text: 'create employee' }, - { id: 'doc2', text: 'employee employee' }, - { id: 'doc3', text: 'list employee data' }, - ]); - - const results = index.search('employee', 10); - - expect(results.length).toBeGreaterThan(1); - for (let i = 0; i < results.length - 1; i++) { - expect(results[i]?.score ?? 0).toBeGreaterThanOrEqual(results[i + 1]?.score ?? 0); - } - }); -}); - -describe('TF-IDF Index - Edge Cases', () => { - test('handles empty query', () => { - const index = new TfidfIndex(); - index.build([{ id: 'doc1', text: 'some text' }]); - - const results = index.search(''); - - expect(results).toHaveLength(0); - }); - - test('handles empty corpus', () => { - const index = new TfidfIndex(); - index.build([]); - - const results = index.search('test query'); - - expect(results).toHaveLength(0); - }); - - test('handles single document corpus', () => { - const index = new TfidfIndex(); - index.build([{ id: 'doc1', text: 'unique document' }]); - - const results = index.search('document'); - - expect(results).toHaveLength(1); - expect(results[0]?.id).toBe('doc1'); - expect(results[0]?.score ?? 0).toBeGreaterThan(0); - }); - - test('handles query with only stopwords', () => { - const index = new TfidfIndex(); - index.build([ - { id: 'doc1', text: 'important content here' }, - { id: 'doc2', text: 'another document' }, - ]); - - const results = index.search('the and or but'); - - expect(results).toHaveLength(0); - }); -}); - -describe('TF-IDF Index - Case Sensitivity', () => { - test('performs case-insensitive search', () => { + test('assigns higher IDF to rare terms', () => { const index = new TfidfIndex(); index.build([ - { id: 'doc1', text: 'EMPLOYEE record' }, - { id: 'doc2', text: 'candidate profile' }, + { id: 'doc1', text: 'common term appears everywhere' }, + { id: 'doc2', text: 'common term appears here' }, + { id: 'doc3', text: 'common term and rare word' }, ]); - const resultsLower = index.search('employee'); - const resultsUpper = index.search('EMPLOYEE'); - const resultsMixed = index.search('EmPlOyEe'); + const rareResults = index.search('rare'); + const commonResults = index.search('common'); - expect(resultsLower.length).toBeGreaterThan(0); - expect(resultsUpper.length).toBe(resultsLower.length); - expect(resultsMixed.length).toBe(resultsLower.length); - expect(resultsLower[0]?.id).toBe('doc1'); - expect(resultsUpper[0]?.id).toBe('doc1'); - expect(resultsMixed[0]?.id).toBe('doc1'); + expect(rareResults.length).toBeGreaterThan(0); + expect(commonResults.length).toBeGreaterThan(0); + expect(rareResults[0]?.score ?? 0).toBeGreaterThan(0); }); }); @@ -152,32 +58,13 @@ describe('TF-IDF Index - Tool Name Scenarios', () => { { id: 'workday_create_candidate', text: 'workday_create_candidate create candidate workday' }, ]); - // Search for terms that appear in tool names const results = index.search('bamboohr create employee'); expect(results.length).toBeGreaterThan(0); - // The BambooHR create employee tool should be highly ranked const topIds = results.slice(0, 2).map((r) => r.id); expect(topIds).toContain('bamboohr_create_employee'); }); - test('finds relevant tools with multiple query terms', () => { - const index = new TfidfIndex(); - index.build([ - { id: 'bamboohr_create_employee', text: 'create employee bamboohr system' }, - { id: 'bamboohr_list_employees', text: 'list employees bamboohr system' }, - { id: 'workday_create_candidate', text: 'create candidate workday system' }, - { id: 'salesforce_list_contacts', text: 'list contacts salesforce system' }, - ]); - - const results = index.search('employee bamboohr'); - - expect(results.length).toBeGreaterThan(0); - // BambooHR employee tools should be top ranked - const topIds = results.slice(0, 2).map((r) => r.id); - expect(topIds.some((id) => id.includes('bamboohr') && id.includes('employee'))).toBe(true); - }); - test('ranks by action type (create, list, etc)', () => { const index = new TfidfIndex(); index.build([ @@ -190,60 +77,130 @@ describe('TF-IDF Index - Tool Name Scenarios', () => { const results = index.search('create employee'); expect(results.length).toBeGreaterThan(0); - // create_employee should be top result expect(results[0]?.id).toBe('bamboohr_create_employee'); }); }); -describe('TF-IDF Index - Search Limits', () => { - test('respects k parameter limit', () => { +/** + * Property-Based Tests for TfidfIndex + * + * These tests verify invariants that must hold for ANY valid input, + * replacing the following example-based tests: + * + * - Score Validation: scores like 0.7071, 0.5, 1.0 are always in [0, 1] + * - Edge Cases: empty query "" returns [], query with no matches returns [] + * - Case Sensitivity: "Alpha" and "ALPHA" and "alpha" return same results + * - Search Limits: search("term", 5) returns at most 5 results + */ +describe('TF-IDF Index - Property-Based Tests', () => { + const documentArbitrary = fc.record({ + id: fc.string({ minLength: 1, maxLength: 20 }).filter((s) => s.trim().length > 0), + text: fc.string({ minLength: 1, maxLength: 200 }), + }); + + const corpusArbitrary = fc.array(documentArbitrary, { minLength: 1, maxLength: 20 }); + + const queryArbitrary = fc + .array(fc.stringMatching(/^[a-zA-Z][a-zA-Z0-9]*$/), { minLength: 1, maxLength: 5 }) + .map((words) => words.join(' ')); + + // Example: search("alpha") on any corpus returns scores like 0.0, 0.5, 1.0 - never 1.5 or -0.1 + fcTest.prop([corpusArbitrary, queryArbitrary], { numRuns: 100 })( + 'scores are always within [0, 1] range', + (corpus, query) => { + const index = new TfidfIndex(); + index.build(corpus); + const results = index.search(query, 100); + + for (const result of results) { + expect(result.score).toBeGreaterThanOrEqual(0); + expect(result.score).toBeLessThanOrEqual(1); + } + }, + ); + + // Example: [{ score: 0.9 }, { score: 0.7 }, { score: 0.3 }] - always descending + fcTest.prop([corpusArbitrary, queryArbitrary], { numRuns: 100 })( + 'results are always sorted by score in descending order', + (corpus, query) => { + const index = new TfidfIndex(); + index.build(corpus); + const results = index.search(query, 100); + + for (let i = 0; i < results.length - 1; i++) { + expect(results[i]?.score ?? 0).toBeGreaterThanOrEqual(results[i + 1]?.score ?? 0); + } + }, + ); + + // Example: search("term", 3) with 10 matching docs returns only 3 results + fcTest.prop([corpusArbitrary, queryArbitrary, fc.integer({ min: 1, max: 50 })], { numRuns: 100 })( + 'search returns at most k results', + (corpus, query, k) => { + const index = new TfidfIndex(); + index.build(corpus); + const results = index.search(query, k); + + expect(results.length).toBeLessThanOrEqual(k); + }, + ); + + // Example: search("Alpha"), search("ALPHA"), search("alpha") all return identical results + fcTest.prop([corpusArbitrary, queryArbitrary], { numRuns: 100 })( + 'search is case-insensitive', + (corpus, query) => { + const index = new TfidfIndex(); + index.build(corpus); + + const lowerResults = index.search(query.toLowerCase()); + const upperResults = index.search(query.toUpperCase()); + + expect(lowerResults.length).toBe(upperResults.length); + for (let i = 0; i < lowerResults.length; i++) { + expect(lowerResults[i]?.id).toBe(upperResults[i]?.id); + expect(lowerResults[i]?.score).toBeCloseTo(upperResults[i]?.score ?? 0, 10); + } + }, + ); + + // Example: index.build([]) then search("anything") returns [] + fcTest.prop([queryArbitrary], { numRuns: 50 })('empty corpus returns empty results', (query) => { const index = new TfidfIndex(); - index.build([ - { id: 'doc1', text: 'alpha' }, - { id: 'doc2', text: 'alpha beta' }, - { id: 'doc3', text: 'alpha gamma' }, - { id: 'doc4', text: 'alpha delta' }, - { id: 'doc5', text: 'alpha epsilon' }, - ]); - - const results = index.search('alpha', 2); - - expect(results.length).toBeLessThanOrEqual(2); - }); - - test('returns all matches when k exceeds corpus size', () => { - const index = new TfidfIndex(); - index.build([ - { id: 'doc1', text: 'test document' }, - { id: 'doc2', text: 'test file' }, - ]); - - const results = index.search('test', 100); + index.build([]); + const results = index.search(query); - // Should return at most 2 results (corpus size) - expect(results.length).toBeLessThanOrEqual(2); + expect(results).toHaveLength(0); }); -}); - -describe('TF-IDF Index - IDF Calculation', () => { - test('assigns higher IDF to rare terms', () => { - const index = new TfidfIndex(); - index.build([ - { id: 'doc1', text: 'common term appears everywhere' }, - { id: 'doc2', text: 'common term appears here' }, - { id: 'doc3', text: 'common term and rare word' }, - ]); - - // Search for the rare term - const rareResults = index.search('rare'); - // Search for the common term - const commonResults = index.search('common'); - - // Both should return results - expect(rareResults.length).toBeGreaterThan(0); - expect(commonResults.length).toBeGreaterThan(0); - // The document with "rare" should have a good score because it's unique - expect(rareResults[0]?.score ?? 0).toBeGreaterThan(0); - }); + // Example: corpus has ids ["doc1", "doc2"], results only contain "doc1" or "doc2" + fcTest.prop([corpusArbitrary, queryArbitrary], { numRuns: 100 })( + 'result IDs are from the indexed corpus', + (corpus, query) => { + const index = new TfidfIndex(); + index.build(corpus); + const results = index.search(query, 100); + + const corpusIds = new Set(corpus.map((doc) => doc.id)); + for (const result of results) { + expect(corpusIds.has(result.id)).toBe(true); + } + }, + ); + + // Example: same corpus + same query always produces identical results + fcTest.prop([corpusArbitrary, queryArbitrary], { numRuns: 50 })( + 'search is deterministic', + (corpus, query) => { + const index1 = new TfidfIndex(); + const index2 = new TfidfIndex(); + + index1.build(corpus); + index2.build(corpus); + + const results1 = index1.search(query, 10); + const results2 = index2.search(query, 10); + + expect(results1).toEqual(results2); + }, + ); });