Skip to content

Commit 727bde2

Browse files
gregbergegregberge
authored
Refactor and optimize OpenAPI parsing (#2830)
1 parent 9f30af1 commit 727bde2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+2239
-407
lines changed

‎.changeset/rotten-cups-judge.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@gitbook/openapi-parser': major
3+
'@gitbook/react-openapi': major
4+
'gitbook': patch
5+
---
6+
7+
Improve and split OpenAPI parser into its own package

‎bun.lock

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
"@gitbook/cache-do": "workspace:*",
3838
"@gitbook/emoji-codepoints": "workspace:*",
3939
"@gitbook/icons": "workspace:*",
40+
"@gitbook/openapi-parser": "workspace:*",
4041
"@gitbook/react-contentkit": "workspace:*",
4142
"@gitbook/react-math": "workspace:*",
4243
"@gitbook/react-openapi": "workspace:*",
@@ -148,6 +149,23 @@
148149
"react": "*",
149150
},
150151
},
152+
"packages/openapi-parser": {
153+
"name": "@gitbook/openapi-parser",
154+
"version": "0.0.0",
155+
"dependencies": {
156+
"@scalar/openapi-parser": "^0.10.4",
157+
"@scalar/openapi-types": "^0.1.6",
158+
"swagger2openapi": "^7.0.8",
159+
"yaml": "1.10.2",
160+
},
161+
"devDependencies": {
162+
"@tsconfig/node20": "^20.1.4",
163+
"@tsconfig/strictest": "^2.0.5",
164+
"@types/swagger2openapi": "^7.0.4",
165+
"bun-types": "^1.1.20",
166+
"typescript": "^5.5.3",
167+
},
168+
},
151169
"packages/proxy": {
152170
"name": "@gitbook/proxy",
153171
"version": "0.1.0",
@@ -191,20 +209,16 @@
191209
"name": "@gitbook/react-openapi",
192210
"version": "0.7.1",
193211
"dependencies": {
212+
"@gitbook/openapi-parser": "workspace:*",
194213
"@scalar/api-client-react": "1.0.87",
195214
"@scalar/oas-utils": "^0.2.101",
196-
"@scalar/openapi-parser": "^0.10.4",
197-
"@scalar/openapi-types": "^0.1.6",
198-
"classnames": "^2.5.1",
215+
"clsx": "^2.1.1",
199216
"flatted": "^3.2.9",
200217
"react-aria": "^3.37.0",
201218
"react-aria-components": "^1.6.0",
202-
"swagger2openapi": "^7.0.8",
203219
"usehooks-ts": "^3.1.0",
204-
"yaml": "1.10.2",
205220
},
206221
"devDependencies": {
207-
"@types/swagger2openapi": "^7.0.4",
208222
"bun-types": "^1.1.20",
209223
"typescript": "^5.5.3",
210224
},
@@ -589,6 +603,8 @@
589603

590604
"@gitbook/icons": ["@gitbook/icons@workspace:packages/icons"],
591605

606+
"@gitbook/openapi-parser": ["@gitbook/openapi-parser@workspace:packages/openapi-parser"],
607+
592608
"@gitbook/proxy": ["@gitbook/proxy@workspace:packages/proxy"],
593609

594610
"@gitbook/react-contentkit": ["@gitbook/react-contentkit@workspace:packages/react-contentkit"],
@@ -1327,6 +1343,10 @@
13271343

13281344
"@tsconfig/node18": ["@tsconfig/[email protected]", "", {}, "sha512-RbwvSJQsuN9TB04AQbGULYfOGE/RnSFk/FLQ5b0NmDf5Kx2q/lABZbHQPKCO1vZ6Fiwkplu+yb9pGdLy1iGseQ=="],
13291345

1346+
"@tsconfig/node20": ["@tsconfig/[email protected]", "", {}, "sha512-sqgsT69YFeLWf5NtJ4Xq/xAF8p4ZQHlmGW74Nu2tD4+g5fAsposc4ZfaaPixVu4y01BEiDCWLRDCvDM5JOsRxg=="],
1347+
1348+
"@tsconfig/strictest": ["@tsconfig/[email protected]", "", {}, "sha512-ec4tjL2Rr0pkZ5hww65c+EEPYwxOi4Ryv+0MtjeaSQRJyq322Q27eOQiFbuNgw2hpL4hB1/W/HBGk3VKS43osg=="],
1349+
13301350
"@types/connect": ["@types/[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w=="],
13311351

13321352
"@types/debug": ["@types/[email protected]", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="],
@@ -1701,7 +1721,7 @@
17011721

17021722
"clone-response": ["[email protected]", "", { "dependencies": { "mimic-response": "^1.0.0" } }, "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA=="],
17031723

1704-
"clsx": ["clsx@2.0.0", "", {}, "sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q=="],
1724+
"clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
17051725

17061726
"code-block-writer": ["[email protected]", "", {}, "sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg=="],
17071727

@@ -4219,6 +4239,10 @@
42194239

42204240
"@radix-ui/react-visually-hidden/@radix-ui/react-primitive": ["@radix-ui/[email protected]", "", { "dependencies": { "@radix-ui/react-slot": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-sHCWTtxwNn3L3fH8qAfnF3WbUZycW93SM1j3NFDzXBiz8D6F5UTTy8G1+WFEaiCdvCVRJWj6N2R4Xq6HdiHmDg=="],
42214241

4242+
"@react-aria/focus/clsx": ["[email protected]", "", {}, "sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q=="],
4243+
4244+
"@react-aria/utils/clsx": ["[email protected]", "", {}, "sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q=="],
4245+
42224246
"@rollup/plugin-commonjs/glob": ["[email protected]", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg=="],
42234247

42244248
"@scalar/api-client/@scalar/oas-utils": ["@scalar/[email protected]", "", { "dependencies": { "@hyperjump/json-schema": "^1.9.6", "@scalar/object-utils": "1.1.12", "@scalar/openapi-types": "0.1.5", "@scalar/themes": "0.9.48", "@scalar/types": "0.0.19", "flatted": "^3.3.1", "microdiff": "^1.4.0", "nanoid": "^5.0.7", "yaml": "^2.4.5", "zod": "^3.23.8" } }, "sha512-deBH359aA9hO+QzdcdJkd0KpZFlSf9xYf+58nl+sVOQL+uEay/LAn1kO+iiE0JAkL78wlW+nqje9sVBKEAdVdw=="],
@@ -4551,6 +4575,8 @@
45514575

45524576
"convict/yargs-parser": ["[email protected]", "", {}, "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w=="],
45534577

4578+
"cva/clsx": ["[email protected]", "", {}, "sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q=="],
4579+
45544580
"decamelize-keys/map-obj": ["[email protected]", "", {}, "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg=="],
45554581

45564582
"deep-equal/isarray": ["[email protected]", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="],

‎package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
},
1616
"private": true,
1717
"scripts": {
18-
"dev": "turbo run dev --ui=stream",
19-
"dev:v2": "turbo run dev:v2 --ui=stream",
18+
"dev": "turbo run dev",
19+
"dev:v2": "turbo run dev:v2",
2020
"build": "turbo run build",
2121
"build:v2": "turbo run build:v2",
2222
"clean-deps": "rm -rf node_modules && rm -rf packages/*/node_modules",

‎packages/gitbook/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"@gitbook/cache-do": "workspace:*",
2121
"@gitbook/emoji-codepoints": "workspace:*",
2222
"@gitbook/icons": "workspace:*",
23+
"@gitbook/openapi-parser": "workspace:*",
2324
"@gitbook/react-contentkit": "workspace:*",
2425
"@gitbook/react-math": "workspace:*",
2526
"@gitbook/react-openapi": "workspace:*",

‎packages/gitbook/src/lib/openapi.ts

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import { ContentRef, DocumentBlockOpenAPI } from '@gitbook/api';
2+
import { parseOpenAPI, OpenAPIParseError, traverse } from '@gitbook/openapi-parser';
23
import {
34
OpenAPIOperationData,
45
fetchOpenAPIOperation,
56
OpenAPIFetcher,
6-
parseOpenAPI,
7-
OpenAPIParseError,
87
} from '@gitbook/react-openapi';
98

109
import { cache, noCacheFetchOptions, CacheFunctionOptions } from '@/lib/cache';
@@ -49,7 +48,7 @@ export async function fetchOpenAPIBlock(
4948

5049
const fetcher: OpenAPIFetcher = {
5150
fetch: cache({
52-
name: 'openapi.fetch.v4',
51+
name: 'openapi.fetch.v5',
5352
get: async (url: string, options: CacheFunctionOptions) => {
5453
// Wrap the raw string to prevent invalid URLs from being passed to fetch.
5554
// This can happen if the URL has whitespace, which is currently handled differently by Cloudflare's implementation of fetch:
@@ -66,13 +65,30 @@ const fetcher: OpenAPIFetcher = {
6665
}
6766

6867
const text = await response.text();
69-
const data = await parseOpenAPI({ url, value: text, parseMarkdown });
68+
const filesystem = await parseOpenAPI({ url, value: text });
69+
const cache: Map<string, Promise<string>> = new Map();
70+
const transformedFs = await traverse(filesystem, async (node) => {
71+
if (
72+
'description' in node &&
73+
typeof node.description === 'string' &&
74+
node.description
75+
) {
76+
if (cache.has(node.description)) {
77+
node['x-description-html'] = await cache.get(node.description);
78+
} else {
79+
const promise = parseMarkdown(node.description);
80+
cache.set(node.description, promise);
81+
node['x-description-html'] = await promise;
82+
}
83+
}
84+
return node;
85+
});
7086
return {
7187
// Cache for 4 hours
7288
ttl: 24 * 60 * 60,
7389
// Revalidate every 2 hours
7490
revalidateBefore: 22 * 60 * 60,
75-
data,
91+
data: transformedFs,
7692
};
7793
},
7894
}),

‎packages/gitbook/tsconfig.json

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,10 @@
2626
]
2727
},
2828
"include": ["next-env.d.ts", "cf-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
29-
"exclude": ["node_modules", "packages/react-openapi", "packages/react-math"]
29+
"exclude": [
30+
"node_modules",
31+
"packages/openapi-parser",
32+
"packages/react-openapi",
33+
"packages/react-math"
34+
]
3035
}

‎packages/openapi-parser/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dist

‎packages/openapi-parser/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# `@gitbook/openapi-parser`
2+
3+
Modern OpenAPI parser written in TypeScript with support for OpenAPI 3.1, OpenAPI 3.0 and Swagger 2.0.

‎packages/openapi-parser/package.json

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"name": "@gitbook/openapi-parser",
3+
"description": "Modern OpenAPI parser written in TypeScript with support for OpenAPI 3.1, OpenAPI 3.0 and Swagger 2.0.",
4+
"type": "module",
5+
"exports": {
6+
".": {
7+
"types": "./dist/index.d.ts",
8+
"development": "./src/index.ts",
9+
"default": "./dist/index.js"
10+
}
11+
},
12+
"version": "0.0.0",
13+
"sideEffects": false,
14+
"dependencies": {
15+
"@scalar/openapi-parser": "^0.10.4",
16+
"@scalar/openapi-types": "^0.1.6",
17+
"swagger2openapi": "^7.0.8",
18+
"yaml": "1.10.2"
19+
},
20+
"devDependencies": {
21+
"@tsconfig/strictest": "^2.0.5",
22+
"@tsconfig/node20": "^20.1.4",
23+
"@types/swagger2openapi": "^7.0.4",
24+
"bun-types": "^1.1.20",
25+
"typescript": "^5.5.3"
26+
},
27+
"scripts": {
28+
"build": "tsc --project tsconfig.build.json",
29+
"typecheck": "tsc --noEmit",
30+
"unit": "bun test",
31+
"dev": "bun run build -- --watch",
32+
"clean": "rm -rf ./dist"
33+
},
34+
"files": [
35+
"dist",
36+
"src",
37+
"README.md",
38+
"CHANGELOG.md"
39+
]
40+
}

‎packages/react-openapi/src/parser/error.ts renamed to ‎packages/openapi-parser/src/error.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
/**
2+
* Error thrown when the OpenAPI document is invalid.
3+
*/
14
export class OpenAPIParseError extends Error {
2-
public name = 'OpenAPIParseError';
5+
public override name = 'OpenAPIParseError';
36

47
constructor(
58
message: string,

0 commit comments

Comments
 (0)