Skip to content

Commit a822cf7

Browse files
bh2smithbh2smith
authored
Testing Spec Validation (#87)
Closes #86 - originally it looked like duplicated parsing, but its actually duplicate fetching. There is also some weird trickery with `accountId` in the `fetchAndValidateSpec`. This PR tests the spec validation while also adding explicit error strings for logging to the user on validation failure.
1 parent 980b1fa commit a822cf7

File tree

3 files changed

+179
-15
lines changed

3 files changed

+179
-15
lines changed

‎src/config/types.ts‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
export type XMbSpec = {
2-
"account-id": string;
32
assistant: {
43
name: string;
54
description: string;
@@ -11,8 +10,10 @@ export type XMbSpec = {
1110
repo?: string;
1211
};
1312
email?: string;
13+
"account-id"?: string;
1414
};
1515

16+
// TODO(bh2smith): Should this be an enum? Or union of all supported tools?
1617
type XMbSpecTools = {
1718
type: string;
1819
};
@@ -26,16 +27,18 @@ export type VerifyData = {
2627
chainIds?: number[];
2728
};
2829

29-
export function isXMbSpec(xMbSpec: unknown): xMbSpec is XMbSpec {
30+
type ValidationResult = { valid: true } | { valid: false; error: string };
31+
32+
function validateXMbSpecHelper(xMbSpec: unknown): ValidationResult {
3033
if (!xMbSpec || typeof xMbSpec !== "object") {
31-
return false;
34+
return { valid: false, error: "x-mb spec must be an object" };
3235
}
3336

3437
const spec = xMbSpec as Record<string, unknown>;
3538

3639
// Validate required fields
3740
if (!spec.assistant || typeof spec.assistant !== "object") {
38-
return false;
41+
return { valid: false, error: "x-mb spec must contain assistant object" };
3942
}
4043

4144
const assistant = spec.assistant as Record<string, unknown>;
@@ -44,21 +47,24 @@ export function isXMbSpec(xMbSpec: unknown): xMbSpec is XMbSpec {
4447
const requiredStringFields = ["name", "description", "instructions"] as const;
4548
for (const field of requiredStringFields) {
4649
if (!assistant[field] || typeof assistant[field] !== "string") {
47-
return false;
50+
return {
51+
valid: false,
52+
error: `assistant must contain ${field} as string`,
53+
};
4854
}
4955
}
5056

5157
// Validate optional fields
5258
if (assistant.tools !== undefined) {
5359
if (!Array.isArray(assistant.tools)) {
54-
return false;
60+
return { valid: false, error: "tools must be an array" };
5561
}
5662
for (const tool of assistant.tools) {
5763
if (!tool || typeof tool !== "object") {
58-
return false;
64+
return { valid: false, error: "each tool must be an object" };
5965
}
6066
if (!("type" in tool) || typeof tool.type !== "string") {
61-
return false;
67+
return { valid: false, error: "each tool must have a type string" };
6268
}
6369
}
6470
}
@@ -68,7 +74,7 @@ export function isXMbSpec(xMbSpec: unknown): xMbSpec is XMbSpec {
6874
!Array.isArray(assistant.chainIds) ||
6975
!assistant.chainIds.every((id) => typeof id === "number")
7076
) {
71-
return false;
77+
return { valid: false, error: "chainIds must be an array of numbers" };
7278
}
7379
}
7480

@@ -77,7 +83,7 @@ export function isXMbSpec(xMbSpec: unknown): xMbSpec is XMbSpec {
7783
!Array.isArray(assistant.categories) ||
7884
!assistant.categories.every((cat) => typeof cat === "string")
7985
) {
80-
return false;
86+
return { valid: false, error: "categories must be an array of strings" };
8187
}
8288
}
8389

@@ -88,13 +94,32 @@ export function isXMbSpec(xMbSpec: unknown): xMbSpec is XMbSpec {
8894
assistant[field] !== undefined &&
8995
typeof assistant[field] !== "string"
9096
) {
91-
return false;
97+
return { valid: false, error: `${field} must be a string` };
9298
}
9399
}
94100

95101
if (spec.email !== undefined && typeof spec.email !== "string") {
96-
return false;
102+
return { valid: false, error: "email must be a string" };
103+
}
104+
105+
return { valid: true };
106+
}
107+
108+
// Type guard (for use in if statements)
109+
export function isXMbSpec(xMbSpec: unknown): xMbSpec is XMbSpec {
110+
return validateXMbSpecHelper(xMbSpec).valid;
111+
}
112+
113+
// Assertion function (for throwing errors)
114+
export function validateXMbSpec(xMbSpec: unknown): asserts xMbSpec is XMbSpec {
115+
const result = validateXMbSpecHelper(xMbSpec);
116+
if (!result.valid) {
117+
throw new Error(result.error);
97118
}
119+
}
98120

99-
return true;
121+
// Helper function to get validation error without throwing
122+
export function getXMbSpecValidationError(xMbSpec: unknown): string | null {
123+
const result = validateXMbSpecHelper(xMbSpec);
124+
return result.valid ? null : result.error;
100125
}

‎src/utils/openapi.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import SwaggerParser from "@apidevtools/swagger-parser";
22

3-
import { isXMbSpec, type XMbSpec } from "../config/types";
3+
import {
4+
getXMbSpecValidationError,
5+
isXMbSpec,
6+
type XMbSpec,
7+
} from "../config/types";
48

59
const MAX_RETRIES = 3;
610
const RETRY_DELAY = 1000;
@@ -54,7 +58,8 @@ export async function validateAndParseOpenApiSpec(
5458
if (isXMbSpec(xMbSpec)) {
5559
return xMbSpec;
5660
}
57-
throw new Error("Invalid OpenAPI spec");
61+
console.error("Invalid x-mb spec: ", getXMbSpecValidationError(xMbSpec));
62+
return undefined;
5863
} catch (error) {
5964
console.error(
6065
"Unexpected error:",

‎tests/config/types.spec.ts‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import { describe, it, expect } from "vitest";
2+
3+
import { isXMbSpec, validateXMbSpec } from "../../src/config/types";
4+
5+
type ArbitraryObject = { [key: string]: unknown };
6+
describe("src/config", () => {
7+
it("validateXMbSpec complete flow", () => {
8+
expect(() => validateXMbSpec(null)).toThrow("x-mb spec must be an object");
9+
// Start with empty spec
10+
let spec: ArbitraryObject = {};
11+
expect(() => validateXMbSpec(spec)).toThrow(
12+
"x-mb spec must contain assistant object",
13+
);
14+
15+
let assistant: ArbitraryObject = {};
16+
spec.assistant = assistant;
17+
expect(() => validateXMbSpec(spec)).toThrow(
18+
"assistant must contain name as string",
19+
);
20+
assistant.name = "assistantName";
21+
expect(() => validateXMbSpec(spec)).toThrow(
22+
"assistant must contain description as string",
23+
);
24+
assistant.description = "assistantDescription";
25+
expect(() => validateXMbSpec(spec)).toThrow(
26+
"assistant must contain instructions as string",
27+
);
28+
assistant.instructions = "assistantInstructions";
29+
30+
// Checkpoint Valid XMbSpec
31+
expect(spec).toStrictEqual({
32+
assistant: {
33+
name: "assistantName",
34+
description: "assistantDescription",
35+
instructions: "assistantInstructions",
36+
},
37+
});
38+
expect(validateXMbSpec(spec)).toBeUndefined();
39+
expect(isXMbSpec(spec)).toBe(true);
40+
41+
// Optional fields
42+
assistant.tools = null;
43+
expect(() => validateXMbSpec(spec)).toThrow("tools must be an array");
44+
assistant.tools = [];
45+
expect(validateXMbSpec(spec)).toBeUndefined();
46+
assistant.tools = [1];
47+
expect(() => validateXMbSpec(spec)).toThrow("each tool must be an object");
48+
assistant.tools = [{}];
49+
expect(() => validateXMbSpec(spec)).toThrow(
50+
"each tool must have a type string",
51+
);
52+
assistant.tools = [{ type: 1 }];
53+
expect(() => validateXMbSpec(spec)).toThrow(
54+
"each tool must have a type string",
55+
);
56+
assistant.tools = [{ type: "function" }];
57+
// TODO(bh2smith): Should the tool type be an enum?
58+
expect(validateXMbSpec(spec)).toBeUndefined();
59+
60+
assistant.chainIds = null;
61+
expect(() => validateXMbSpec(spec)).toThrow("chainIds must be an array");
62+
assistant.chainIds = ["ethereum"];
63+
expect(() => validateXMbSpec(spec)).toThrow(
64+
"chainIds must be an array of number",
65+
);
66+
assistant.chainIds = [1, 2];
67+
expect(validateXMbSpec(spec)).toBeUndefined();
68+
69+
assistant.categories = null;
70+
expect(() => validateXMbSpec(spec)).toThrow("categories must be an array");
71+
assistant.categories = [null];
72+
expect(() => validateXMbSpec(spec)).toThrow(
73+
"categories must be an array of strings",
74+
);
75+
assistant.categories = ["ethereum", "solana"];
76+
expect(validateXMbSpec(spec)).toBeUndefined();
77+
78+
assistant.version = null;
79+
expect(() => validateXMbSpec(spec)).toThrow("version must be a string");
80+
assistant.version = "1.0.0";
81+
expect(validateXMbSpec(spec)).toBeUndefined();
82+
83+
assistant.repo = null;
84+
expect(() => validateXMbSpec(spec)).toThrow("repo must be a string");
85+
86+
assistant.repo = "https://github.com/example/repo";
87+
expect(validateXMbSpec(spec)).toBeUndefined();
88+
89+
spec.email = 123;
90+
expect(() => validateXMbSpec(spec)).toThrow("email must be a string");
91+
92+
spec.email = "[email protected]";
93+
expect(validateXMbSpec(spec)).toBeUndefined();
94+
95+
// Full Example Spec:
96+
expect(spec).toStrictEqual({
97+
assistant: {
98+
name: "assistantName",
99+
description: "assistantDescription",
100+
instructions: "assistantInstructions",
101+
tools: [{ type: "function" }],
102+
chainIds: [1, 2],
103+
categories: ["ethereum", "solana"],
104+
version: "1.0.0",
105+
repo: "https://github.com/example/repo",
106+
},
107+
email: "[email protected]",
108+
});
109+
});
110+
111+
it("validateXMbSpec real example (CoW Agent)", () => {
112+
const cowSpec = {
113+
"account-id": "max-normal.near",
114+
assistant: {
115+
name: "CoWSwap Assistant",
116+
description:
117+
"An assistant that generates EVM transaction data for CoW Protocol Interactions",
118+
instructions:
119+
"Encodes transactions as signature requests on EVM networks. This assistant is only for EVM networks. Passes the the transaction fields of the response to generate-evm-tx tool for signing and displays the meta content of the response to the user after signing. For selling native assets, such as ETH, xDAI, POL, BNB it uses 0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE as the sellToken. It does not infer the chainId. Do not infer the token decimals. Use Token Units for sellAmountBeforeFee. Uses token symbols for sellToken and buyToken unless addresses are provided. Always passes evmAddress as the safeAddress on any request requiring safeAddress. The only supported chains for the cowswap endpoint are Ethereum, Gnosis, Arbitrum and Base. All network support for balance, weth and erc20 endpoints.",
120+
tools: [
121+
{
122+
type: "generate-evm-tx",
123+
},
124+
],
125+
image: "https://near-cow-agent.vercel.app/cowswap.svg",
126+
categories: ["defi"],
127+
chainIds: [1, 100, 8453, 42161, 11155111],
128+
},
129+
image: "https://near-cow-agent.vercel.app/cowswap.svg",
130+
};
131+
expect(validateXMbSpec(cowSpec)).toBeUndefined();
132+
expect(isXMbSpec(cowSpec)).toBe(true);
133+
});
134+
});

0 commit comments

Comments
 (0)