svelte.dev/packages (#1523)

* Implementation of svelte.dev/packages only with no package search

* remove unused code

* fix bad data

* add missing sv add-ons

* add vite-plugin-devtools-json

* add back alias

* keep one packages-meta

* add sv cmd display

* add link to title of pkg

* rmv icon for border left

* update main redirect (hompage or sv)

* homelink & sv link

* remove little used routers

* add router from svelte faq

* move svelte-chartjs. lowercase titles

* remove unused field from registry

* update sv add-ons title

* show some categories as alternatives to sveltekit functionality

* cleanup list of packages and descriptions

* better display of add-ons

* svelte-check

* category descriptions

* update descriptions to mention tailwind

* prettier

* cleanup

* scroll y

* more cleanup

* add 2 missing json

* rmv unused json

* update metadata

* repo_rul back

* update metadata script

* update melt to new package

* add testing and devtools section

* i18n category

* cleanup media category

* update metadata

* rmv duplicate

* add links in descriptions

* bug fix: don't mutate existing data

* change wuchale to '@wuchale/svelte'

* format

* update browser extension listing

* remove a couple more entries

* update stats

* addin a report at the end

* add github token (preparing action) & update stars

* add some reports

* moving things around for stats & better understanding

* pretty

* update with svelte_range & kit_range

* stable order

* fetcher

* address deprecations

* add sveltepress

* no filter npm stats

* CI sync-packages v001

* CI sync-packages v002

* CI sync-packages v003

* CI sync-packages v004

* CI sync-packages v005

* CI sync-packages v006

* CI sync-packages v007

* CI sync-packages v008

* CI sync-packages v009

* CI sync-packages v011

* update metadata

* adding a readme

* update stats

* switch to full ts first (description)

* better CRUD management

* Don't use @html description (and strip out makdown links). Description can be overrritten

* more package cleanup

* remove comment

* add deno sveltekit adapter

* reorder categories

* move ark down a few places out of the default viewport

* Update .github/workflows/sync-packages.yml

Co-authored-by: Dominik G. <[email protected]>

* Update .github/workflows/sync-packages.yml

* update setup-node action

* don't use tsx for new script

* add a link to sveltesociety

* format

* more-less

* tweak styles

* style tweaks

* tweak

* fix scroll

* dark mode

* tweaks

* more useful title

* make the entire card a link

* tidy up

* tweak wording

* move stuff to top

* remove link to sveltesociety.dev

* add note

* tweak styles

* unused

* design cleanup (#1568)

* description overrides

* use <details>

* tweak

* unused

* sync script update

* rmc dots at the end of desc

* cleanup description will not end by "." & links will not end with #readme

* update data

* cursor: pointer all the way

* always link

* omit links element for sv add-ons

* remove stats from sv add-ons

* more description updates

* a few more descriptions

* logos

* hack

* fix playwright logo cropping

* tweak some wording slightly

* slim down some descriptions

* lucia description

* bunch more

* more

* more

* more

* more

* fix lucide

* finish removing package

* more

* fix

* focus ring on links

* small tweak

* fix

* text-decoration to make it clear it links

* tweak

* add permalink

* sharing links are open

* rmv auto, add static, tweak order

* semi

* sync packages

* remove kit-docs as unmaintained

* remove amp package

---------

Co-authored-by: jycouet <[email protected]>
Co-authored-by: Tee Ming <[email protected]>
Co-authored-by: Dominik G. <[email protected]>
Co-authored-by: Rich Harris <[email protected]>

Author: 5 people

.github/workflows/sync-packages.yml

@@ -0,0 +1,37 @@
+name: Sync packages
+
+on:
+  schedule:
+    - cron: '0 0 * * 0' # At 00:00 on Sunday.
+  workflow_dispatch: # Allow manual triggering
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v5
+        with:
+          node-version: 24
+      - run: pnpm install --frozen-lockfile --ignore-scripts
+
+      - name: Sync packages
+        run: cd apps/svelte.dev && pnpm sync-packages
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create or update pull request
+        uses: peter-evans/create-pull-request@599a7e63a6240886b1b61fe984db1de9e0b05bc4 # v7.0.8
+        with:
+          commit-message: 'chore(packages): Update metadata'
+          title: 'chore(packages): Update metadata'
+          body: Automatically fetch latest packages metadata from NPM & GitHub.
+          branch: ci/update-packages-metadata
+          delete-branch: true

apps/svelte.dev/package.json

@@ -13,7 +13,8 @@
 		"check": "node scripts/update.js && svelte-kit sync && svelte-check",
 		"format": "prettier --write .",
 		"lint": "prettier --check .",
-		"sync-docs": "tsx scripts/sync-docs/index.ts"
+		"sync-docs": "tsx scripts/sync-docs/index.ts",
+		"sync-packages": "node scripts/sync-packages/index.ts"
 	},
 	"dependencies": {
 		"@jridgewell/sourcemap-codec": "^1.4.15",
@@ -40,6 +41,7 @@
 		"flexsearch": "^0.7.43",
 		"flru": "^1.0.2",
 		"icons": "workspace:*",
+		"logos": "workspace:*",
 		"port-authority": "^2.0.1",
 		"topojson-client": "^3.1.0",
 		"vitest": "^3.2.4",

apps/svelte.dev/scripts/sync-docs/index.ts

@@ -4,11 +4,11 @@ import path from 'node:path';
 import fs from 'node:fs';
 import { parseArgs } from 'node:util';
 import ts from 'typescript';
-import glob from 'tiny-glob/sync';
+import glob from 'tiny-glob/sync.js';
 import chokidar from 'chokidar';
 import { fileURLToPath } from 'node:url';
-import { clone_repo, migrate_meta_json } from './utils';
-import { get_types, read_d_ts_file, read_types } from './types';
+import { clone_repo, migrate_meta_json } from './utils.ts';
+import { get_types, read_d_ts_file, read_types } from './types.ts';
 import type { Modules } from '@sveltejs/site-kit/markdown';
 
 interface Package {

apps/svelte.dev/scripts/sync-docs/utils.ts

@@ -1,6 +1,6 @@
 import { execSync, spawn, type SpawnOptions } from 'node:child_process';
 import fs from 'node:fs';
-import glob from 'tiny-glob/sync';
+import glob from 'tiny-glob/sync.js';
 
 export async function clone_repo(repo: string, name: string, branch: string, cwd: string) {
 	const dir = `${cwd}/${name}`;

apps/svelte.dev/scripts/sync-packages/README.md

@@ -0,0 +1,19 @@
+# sync-packages
+
+This script syncs the packages metadata from NPM & GitHub.
+
+## Usage
+
+```bash
+pnpm sync-packages
+```
+
+## Notes
+
+- All packages `names` are in `FEATURED` of [packages-meta.ts](apps/svelte.dev/src/lib/packages-meta.ts) file.
+
+If you want to add or remove a package, you need to update `FEATURED` objects and run the script again (it will update the json files in the `src/lib/server/generated/registry` directory).
+
+- [sync-packages.yml](/.github/workflows/sync-packages.yml) is responsible for running the script regularly and update all metadata (it can also be triggered manually).
+
+- Ambassadors and maintainers are curating the list of packages.

apps/svelte.dev/scripts/sync-packages/index.ts

@@ -0,0 +1,280 @@
+import { PACKAGES_META } from '../../src/lib/packages-meta.ts';
+import type { PackageKey, PackageNpm, PackageGithub } from '../../src/lib/server/content.ts';
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+
+let skipGithubStars = false;
+
+const start = performance.now();
+console.log('[sync-packages] start');
+
+let logsAtTheEnd: {
+	type:
+		| 'no_repo_url'
+		| 'low_downloads'
+		| 'low_github_stars'
+		| 'new_json_file'
+		| 'deleted_unused_json_file'
+		| 'outdated'
+		| 'deprecated'
+		| 'npm_error';
+	pkg: string;
+	extra: string;
+}[] = [];
+
+const packages = PACKAGES_META.FEATURED.flatMap((pkg) => pkg.packages).map((pkg) => pkg.name);
+
+const registryFolder = 'src/lib/server/generated/registry';
+
+// PART 1: create missing json files
+for (const pkg of packages) {
+	const cleanPkg = pkg.replace('@', '').replace('/', '-');
+	const jsonPath = path.join(registryFolder, `${cleanPkg}.json`);
+	if (!fs.existsSync(jsonPath)) {
+		const p = await getNpmAndGitHubData(pkg);
+		writeJsonData(jsonPath, p);
+		logsAtTheEnd.push({ type: 'new_json_file', pkg, extra: `created -> ${jsonPath}` });
+	}
+}
+
+// PART 2: delete unused json files
+let registryJsonFiles = fs.readdirSync(registryFolder);
+const jsonUsed: string[] = [];
+for (const pkg of packages) {
+	const cleanPkg = pkg.replace('@', '').replace('/', '-');
+	const cleanPkgFile = `${cleanPkg}.json`;
+	const jsonPath = path.join(registryFolder, cleanPkgFile);
+	if (fs.existsSync(jsonPath)) {
+		jsonUsed.push(cleanPkgFile);
+	}
+}
+const jsonNotNeeded = registryJsonFiles.filter((pkg) => !jsonUsed.includes(pkg));
+if (jsonNotNeeded.length > 0) {
+	// delete json files
+	for (const pkg of jsonNotNeeded) {
+		const jsonPath = path.join(registryFolder, pkg);
+		fs.unlinkSync(jsonPath);
+		logsAtTheEnd.push({ type: 'deleted_unused_json_file', pkg, extra: `deleted -> ${jsonPath}` });
+	}
+
+	// Let's continue
+	// theEnd(1);
+}
+
+// PART 3: refresh data
+registryJsonFiles = fs.readdirSync(registryFolder); //.slice(0, 20);
+
+const batch = 10;
+for (let i = 0; i < registryJsonFiles.length; i += batch) {
+	const batchFiles = registryJsonFiles.slice(i, i + batch);
+	await Promise.all(
+		batchFiles.map(async (pkg) => {
+			await refreshJsonFile(path.join(registryFolder, pkg));
+		})
+	);
+}
+
+theEnd(0);
+
+// HELPERS
+
+function theEnd(val: number) {
+	const msg = ['[sync-packages]'];
+	if (val > 0) {
+		msg.push(`exit(${val}) - `);
+	}
+	msg.push(`took: ${(performance.now() - start).toFixed(0)}ms`);
+	console.log(msg.join(' '));
+	if (logsAtTheEnd.length > 0) {
+		console.log('[sync-packages] report:');
+		const typePrints: Record<(typeof logsAtTheEnd)[number]['type'], string> = {
+			no_repo_url: 'No GitHub URL',
+			low_downloads: 'Low Downloads',
+			low_github_stars: 'Low Stars',
+			new_json_file: 'NEW JSON',
+			deleted_unused_json_file: 'DEL JSON',
+			outdated: 'Outdated',
+			deprecated: 'Deprecated',
+			npm_error: 'NPM Error'
+		};
+		console.log(
+			`  - ${logsAtTheEnd.map((l) => `${typePrints[l.type].padEnd(15)} | ${l.pkg.padEnd(35)} | ${l.extra}`).join('\n  - ')}`
+		);
+	}
+	process.exit(val);
+}
+
+async function getNpmAndGitHubData(pkg: string): Promise<PackageKey & PackageNpm & PackageGithub> {
+	const [npmInfo, npmDlInfo] = await Promise.all([
+		fetchJson(`https://registry.npmjs.org/${pkg}`),
+		fetchJson(`https://api.npmjs.org/downloads/point/last-week/${pkg}`)
+	]);
+
+	if (npmInfo.error) {
+		logsAtTheEnd.push({ type: 'npm_error', pkg, extra: npmInfo.error });
+		theEnd(1);
+	}
+
+	// delete npmInfo.readme;
+	// delete npmInfo.versions;
+	// console.log(`npmInfo`, npmInfo);
+
+	const npm_description = npmInfo.description;
+	const raw_repo_url = npmInfo.repository?.url ?? '';
+	const repo_url = raw_repo_url?.replace(/^git\+/, '').replace(/\.git$/, '');
+	if (!repo_url) {
+		// console.error(`repo_url not found for ${pkg}`);
+		logsAtTheEnd.push({ type: 'no_repo_url', pkg, extra: `not found` });
+	}
+	const git_org = repo_url?.split('/')[3];
+	const git_repo = repo_url?.split('/')[4];
+
+	const authors = npmInfo.maintainers?.map((m: { name: string }) => m.name);
+	const homepage = npmInfo.homepage;
+	const downloads = npmDlInfo.downloads;
+	const version = npmInfo['dist-tags'].latest;
+	const updated = npmInfo.time[version];
+	const pkgInfo = npmInfo.versions[version];
+	const deprecated_reason = pkgInfo.deprecated;
+
+	const svelte_range =
+		pkgInfo.peerDependencies?.svelte ??
+		pkgInfo.dependencies?.svelte ??
+		pkgInfo.devDependencies?.svelte;
+
+	const kit_range =
+		pkgInfo.peerDependencies?.['@sveltejs/kit'] ??
+		pkgInfo.dependencies?.['@sveltejs/kit'] ??
+		pkgInfo.devDependencies?.['@sveltejs/kit'];
+
+	// GitHub
+	let github_stars: number | undefined = undefined;
+	if (git_org && git_repo && !skipGithubStars) {
+		const token = process.env.GITHUB_TOKEN;
+		const headers = token ? new Headers({ authorization: 'Bearer ' + token }) : {};
+		const res = await fetchJson(`https://api.github.com/repos/${git_org}/${git_repo}`, { headers });
+		if (res?.message && res?.message.startsWith('API rate limit exceeded')) {
+			skipGithubStars = true;
+		} else {
+			github_stars = res.stargazers_count;
+		}
+	}
+
+	return {
+		name: pkg,
+		npm_description,
+		repo_url,
+		authors,
+		homepage,
+		version,
+		deprecated_reason,
+		downloads,
+		updated,
+		svelte_range,
+		kit_range,
+
+		// GitHub
+		github_stars
+	};
+}
+
+async function refreshJsonFile(fullPath: string) {
+	console.log(`Refreshing:`, fullPath);
+
+	const currentJson = JSON.parse(fs.readFileSync(fullPath, 'utf-8'));
+	const newData = await getNpmAndGitHubData(currentJson.name);
+
+	// remove all undefined values
+	for (const key in newData) {
+		if (newData[key] === undefined) {
+			delete newData[key];
+		}
+	}
+
+	// Let's not filter npm downloads changes
+	// // filter changes of downloads
+	// if (newData.downloads && currentJson.downloads) {
+	// 	const dlDelta = Math.abs(
+	// 		((newData.downloads - currentJson.downloads) / currentJson.downloads) * 100
+	// 	);
+	// 	if (dlDelta < 2.5) delete newData.downloads;
+	// }
+
+	const data = { ...currentJson, ...newData };
+
+	// Some stats infos to log
+	if (data.downloads && data.downloads < 255) {
+		logsAtTheEnd.push({ type: 'low_downloads', pkg: data.name, extra: `${data.downloads}` });
+	}
+
+	if (data.github_stars && data.github_stars < 42) {
+		logsAtTheEnd.push({
+			type: 'low_github_stars',
+			pkg: data.name,
+			extra: `${data.github_stars}`
+		});
+	}
+
+	if (data.updated && PACKAGES_META.is_outdated(data.updated)) {
+		logsAtTheEnd.push({ type: 'outdated', pkg: data.name, extra: `${data.updated}` });
+	}
+
+	if (data.deprecated_reason) {
+		logsAtTheEnd.push({ type: 'deprecated', pkg: data.name, extra: `${data.deprecated_reason}` });
+	}
+
+	writeJsonData(fullPath, data);
+}
+
+function writeJsonData(path: string, data: any) {
+	const keysOrder: (keyof PackageKey | keyof PackageNpm | keyof PackageGithub)[] = [
+		'name',
+		'npm_description',
+		'repo_url',
+		'authors',
+		'homepage',
+		'version',
+		'deprecated_reason',
+		'downloads',
+		'github_stars',
+		'updated',
+		'svelte_range',
+		'kit_range',
+		'typescript',
+		'runes',
+		'last_rune_check_version'
+	];
+
+	const sortedData: Record<string, any> = {};
+	for (const key of keysOrder) {
+		if (data[key] !== undefined) {
+			sortedData[key] = data[key];
+		}
+	}
+	// all all the remaining keys
+	for (const key in data) {
+		if (!keysOrder.includes(key as keyof PackageKey | keyof PackageNpm | keyof PackageGithub)) {
+			sortedData[key] = data[key];
+		}
+	}
+
+	fs.writeFileSync(path, JSON.stringify(sortedData, null, 2));
+	execSync(`prettier --write ${path}`);
+}
+
+async function fetchJson(url: string, options: RequestInit = {}): Promise<any> {
+	const headers = new Headers({ ...options.headers, 'User-Agent': 'svelte.dev/packages_v0.0.1' });
+
+	for (let i = 0; i < 5; i++) {
+		try {
+			const res = await fetch(url, { ...options, headers });
+			return await res.json();
+		} catch (e) {
+			console.error(`Failed to fetch ${url} after ${i + 1} retries`);
+		}
+
+		await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, i + 1)));
+	}
+}