Skip to content

Commit 82e6d80

Browse files
eddeee888eddeee888
authored
Fix website deployment (#10536)
* Test * Generate packages-info * Ignore generated packages info file * Update website build/deploy script to generate packages info first * Fix lint/format/data issue * Add FIXME * Add random sleep when fetching packages to avoid getting rate limited * Ensure packages info are generated * Move all scripts to scripts * Fix pathing
1 parent f588d91 commit 82e6d80

File tree

10 files changed

+455
-346
lines changed

10 files changed

+455
-346
lines changed

‎website/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# NPM packages info fetched before building.
2+
# Without this, every page build process will hit NPM and may get rate limited.
3+
# Run `yarn generate-packages-info` to regenerate.
4+
src/lib/packages-info.generated.ts

‎website/package.json‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@
55
"type": "module",
66
"scripts": {
77
"start": "next start",
8-
"build": "yarn generate-json-config && next build && next-sitemap",
8+
"build": "yarn generate-packages-info && yarn generate-json-config && next build && next-sitemap",
99
"dev": "next",
10-
"generate-json-config": "tsx generate-config-json-schema.ts"
10+
"generate-json-config": "tsx scripts/generate-config-json-schema.ts",
11+
"generate-packages-info": "tsx scripts/generate-packages-info.ts"
1112
},
1213
"devDependencies": {
1314
"@theguild/tailwind-config": "0.6.3",
@@ -20,7 +21,9 @@
2021
"postcss-import": "^16.1.0",
2122
"postcss-lightningcss": "^1.0.1",
2223
"prettier-plugin-tailwindcss": "0.2.8",
23-
"tailwindcss": "^3.4.14"
24+
"tailwindcss": "^3.4.14",
25+
"semver": "7.7.3",
26+
"@types/semver": "7.7.1"
2427
},
2528
"dependencies": {
2629
"@graphql-codegen/add": "6.0.0",

‎website/public/config.schema.json‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,10 @@
665665
},
666666
"importExtension": {
667667
"description": "Append this extension to all imports.\nUseful for ESM environments that require file extensions in import statements.",
668-
"type": "string"
668+
"anyOf": [
669+
{ "type": "array", "items": { "type": "string" } },
670+
{ "enum": [""], "type": "string" }
671+
]
669672
},
670673
"extractAllFieldsToTypes": {
671674
"description": "Extract all field types to their own types, instead of inlining them.\nThis helps to reduce type duplication, and makes type errors more readable.\nIt can also significantly reduce the size of the generated code, the generation time,\nand the typechecking time.\nDefault value: \"false\"",
@@ -862,7 +865,10 @@
862865
},
863866
"importExtension": {
864867
"description": "Append this extension to all imports.\nUseful for ESM environments that require file extensions in import statements.",
865-
"type": "string"
868+
"anyOf": [
869+
{ "type": "array", "items": { "type": "string" } },
870+
{ "enum": [""], "type": "string" }
871+
]
866872
},
867873
"extractAllFieldsToTypes": {
868874
"description": "Extract all field types to their own types, instead of inlining them.\nThis helps to reduce type duplication, and makes type errors more readable.\nIt can also significantly reduce the size of the generated code, the generation time,\nand the typechecking time.\nDefault value: \"false\"",
@@ -1685,7 +1691,10 @@
16851691
},
16861692
"importExtension": {
16871693
"description": "Append this extension to all imports.\nUseful for ESM environments that require file extensions in import statements.",
1688-
"type": "string"
1694+
"anyOf": [
1695+
{ "type": "array", "items": { "type": "string" } },
1696+
{ "enum": [""], "type": "string" }
1697+
]
16891698
},
16901699
"extractAllFieldsToTypes": {
16911700
"description": "Extract all field types to their own types, instead of inlining them.\nThis helps to reduce type duplication, and makes type errors more readable.\nIt can also significantly reduce the size of the generated code, the generation time,\nand the typechecking time.\nDefault value: \"false\"",
@@ -2814,7 +2823,10 @@
28142823
},
28152824
"importExtension": {
28162825
"description": "Append this extension to all imports.\nUseful for ESM environments that require file extensions in import statements.",
2817-
"type": "string"
2826+
"anyOf": [
2827+
{ "type": "array", "items": { "type": "string" } },
2828+
{ "enum": [""], "type": "string" }
2829+
]
28182830
},
28192831
"extractAllFieldsToTypes": {
28202832
"description": "Extract all field types to their own types, instead of inlining them.\nThis helps to reduce type duplication, and makes type errors more readable.\nIt can also significantly reduce the size of the generated code, the generation time,\nand the typechecking time.\nDefault value: \"false\"",

website/generate-config-json-schema.ts renamed to website/scripts/generate-config-json-schema.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { writeFile } from 'node:fs/promises';
22
import { join } from 'node:path';
33
import jsonPath from 'jsonpath';
44
import prettier from 'prettier';
5-
import { transformDocs } from './src/lib/transform';
5+
import { transformDocs } from '../src/lib/transform';
66

77
const MARKDOWN_JSDOC_KEY = 'exampleMarkdown';
88
const DEFAULT_JSDOC_KEY = 'default';

‎website/scripts/generate-packages-info.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import { writeFileSync } from 'node:fs';
2+
import path from 'node:path';
3+
import semver from 'semver';
4+
import { PACKAGES } from '../src/lib/plugins/packages.js';
5+
6+
interface PackageInfo {
7+
readme: string;
8+
createdAt: string;
9+
updatedAt: string;
10+
description: string;
11+
weeklyNPMDownloads: number;
12+
}
13+
14+
// FIXME: fetchPackageInfo is a simpler version of `@theguild/components`'s fetchPackageInfo
15+
// We cannot use the shared package version because it has imports that Next.js understands
16+
// but not a simple script like this. So, inlining this version helps unblock deployment
17+
const fetchPackageInfo = async (packageName: string): Promise<PackageInfo> => {
18+
const NO_NPM_README_PLACEHOLDER = 'ERROR: No README data found!';
19+
20+
const encodedName = encodeURIComponent(packageName);
21+
console.debug(`Loading NPM package info: ${packageName}`);
22+
const [packageInfo, { downloads }] = await Promise.all([
23+
fetch(`https://registry.npmjs.org/${encodedName}`).then(response => response.json()),
24+
fetch(`https://api.npmjs.org/downloads/point/last-week/${encodedName}`).then(response => response.json()),
25+
]);
26+
const { readme, time, description } = packageInfo;
27+
const latestVersion = packageInfo['dist-tags'].latest;
28+
29+
return {
30+
readme:
31+
(readme !== NO_NPM_README_PLACEHOLDER && readme) || // for some reason top level "readme" can be empty string, so we get the latest version readme
32+
Object.values(packageInfo.versions as { readme?: string; version: string }[])
33+
.reverse()
34+
.find(curr => {
35+
const isReadmeExist = curr.readme && curr.readme !== NO_NPM_README_PLACEHOLDER;
36+
if (isReadmeExist) {
37+
return semver.lte(curr.version, latestVersion);
38+
}
39+
})?.readme ||
40+
'',
41+
createdAt: time.created,
42+
updatedAt: time.modified,
43+
description,
44+
weeklyNPMDownloads: downloads,
45+
};
46+
};
47+
48+
const result: Record<string, PackageInfo> = {};
49+
for (let [identifier, pkg] of Object.entries(PACKAGES)) {
50+
const packageInfo = await fetchPackageInfo(pkg.npmPackage);
51+
result[identifier] = packageInfo;
52+
53+
await (async function randomSleep({ minMs, maxMs }: { minMs: number; maxMs: number }): Promise<void> {
54+
function sleep(ms: number): Promise<void> {
55+
return new Promise(resolve => setTimeout(resolve, ms));
56+
}
57+
58+
const delay = Math.floor(Math.random() * (maxMs - minMs + 1)) + minMs;
59+
await sleep(delay);
60+
})({ minMs: 100, maxMs: 5000 });
61+
}
62+
63+
writeFileSync(
64+
path.join(process.cwd(), 'src', 'lib', 'packages-info.generated.ts'),
65+
`export const packagesInfo= ${JSON.stringify(result)}`,
66+
{
67+
encoding: 'utf8',
68+
}
69+
);

‎website/src/components/plugins-marketplace-search.tsx‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import { useMemo } from 'react';
2-
import { fetchPackageInfo, IMarketplaceSearchProps, MarketplaceSearch } from '@theguild/components';
2+
import { IMarketplaceSearchProps, MarketplaceSearch } from '@theguild/components';
33
import { compareDesc } from 'date-fns';
44
import { CategoryToPackages } from '@/category-to-packages.mjs';
55
import { ALL_TAGS, Icon, icons, PACKAGES } from '@/lib/plugins';
6+
import { packagesInfo } from '@/lib/packages-info.generated';
67

78
export type Plugin = {
89
title: string;
@@ -18,24 +19,30 @@ export type Plugin = {
1819

1920
export const getPluginsStaticProps = async () => {
2021
const categoryEntries = Object.entries(CategoryToPackages);
21-
const plugins: Plugin[] = await Promise.all(
22-
Object.entries(PACKAGES).map(async ([identifier, { npmPackage, title, icon, tags }]) => {
23-
const { readme, createdAt, updatedAt, description, weeklyNPMDownloads = 0 } = await fetchPackageInfo(npmPackage);
24-
const [category] = categoryEntries.find(([, pluginName]) => pluginName.includes(identifier)) || [];
2522

26-
return {
27-
title,
28-
readme,
29-
createdAt,
30-
updatedAt,
31-
description,
32-
linkHref: `/plugins/${category}/${identifier}`,
33-
weeklyNPMDownloads,
34-
icon,
35-
tags,
36-
};
37-
})
38-
);
23+
const plugins: Plugin[] = Object.entries(PACKAGES).map(([identifier, { title, icon, tags }]) => {
24+
const packageInfo = packagesInfo[identifier as keyof typeof packagesInfo];
25+
if (!packageInfo) {
26+
throw new Error(`Unknown "${identifier}" plugin identifier`);
27+
}
28+
29+
const { readme, createdAt, updatedAt, description, weeklyNPMDownloads = 0 } = packageInfo;
30+
31+
const [category] = categoryEntries.find(([, pluginName]) => pluginName.includes(identifier)) || [];
32+
33+
return {
34+
title,
35+
readme,
36+
createdAt,
37+
updatedAt,
38+
description,
39+
linkHref: `/plugins/${category}/${identifier}`,
40+
weeklyNPMDownloads,
41+
icon,
42+
tags,
43+
};
44+
});
45+
3946
return {
4047
props: {
4148
// We add an `ssg` field to the page props,

‎website/src/lib/plugin-get-static-props.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import { parse } from 'node:path';
2-
import { fetchPackageInfo } from '@theguild/components';
32
import { defaultRemarkPlugins } from '@theguild/components/next.config';
43
import { format } from 'date-fns';
54
import { PACKAGES } from '@/lib/plugins';
65
import { transformDocs } from '@/lib/transform';
76
import { buildDynamicMDX } from 'nextra/remote';
7+
import { packagesInfo } from '@/lib/packages-info.generated';
88

99
// Can't be used in plugin.tsx due incorrect tree shaking:
1010
// Module not found: Can't resolve 'fs'
@@ -20,8 +20,9 @@ export const pluginGetStaticProps =
2020
if (!plugin) {
2121
throw new Error(`Unknown "${identifier}" plugin identifier`);
2222
}
23+
2324
const { npmPackage } = plugin;
24-
const { readme, updatedAt } = await fetchPackageInfo(npmPackage);
25+
const { readme, updatedAt } = packagesInfo[identifier as keyof typeof packagesInfo];
2526

2627
const generatedDocs = transformDocs();
2728
const source = generatedDocs.docs[identifier] || readme.replaceAll('```yml', '```yaml') || '';

0 commit comments

Comments
 (0)