feat(packages): get package descriptions

Author: WarningImHack3r

src/lib/server/github-cache.ts

@@ -8,6 +8,8 @@ export type GitHubRelease = Awaited<
 	ReturnType<InstanceType<typeof Octokit>["rest"]["repos"]["listReleases"]>
 >["data"][number];
 
+type KeyType = "releases" | "descriptions";
+
 /**
  * The maximum items amount to get per-page
  * when fetching from GitHub API.
@@ -57,11 +59,12 @@ export class GitHubCache {
 	 *
 	 * @param owner the GitHub repository owner
 	 * @param repo the GitHub repository name
+	 * @param type the kind of cache to use
 	 * @returns the pure computed key
 	 * @private
 	 */
-	#getRepoKey(owner: string, repo: string) {
-		return `repo:${owner}/${repo}:releases`;
+	#getRepoKey(owner: string, repo: string, type: KeyType) {
+		return `repo:${owner}/${repo}:${type}`;
 	}
 
 	/**
@@ -71,15 +74,15 @@ export class GitHubCache {
 	 * @returns the releases, either cached or fetched
 	 */
 	async getReleases(repository: Repository) {
-		const cacheKey = this.#getRepoKey(repository.owner, repository.repoName);
+		const cacheKey = this.#getRepoKey(repository.owner, repository.repoName, "releases");
 
 		const cachedReleases = await this.#redis.json.get<GitHubRelease[]>(cacheKey);
 		if (cachedReleases) {
-			console.log(`Cache hit for ${cacheKey}`);
+			console.log(`Cache hit for releases for ${cacheKey}`);
 			return cachedReleases;
 		}
 
-		console.log(`Cache miss for ${cacheKey}, fetching from GitHub API`);
+		console.log(`Cache miss for releases for ${cacheKey}, fetching from GitHub API`);
 
 		const releases = await this.#fetchReleases(repository);
 
@@ -213,6 +216,72 @@ export class GitHubCache {
 		);
 	}
 
+	/**
+	 * Get a map that contains the descriptions
+	 * of all the packages in the given repository.
+	 * Irrelevant paths (e.g., tests) or empty descriptions
+	 * are excluded.
+	 *
+	 * @param repository the repository to fetch the
+	 * descriptions in
+	 * @returns a map of paths to descriptions.
+	 * @private
+	 */
+	async getDescriptions(repository: Repository) {
+		const cacheKey = this.#getRepoKey(repository.owner, repository.repoName, "descriptions");
+
+		const cachedDescriptions = await this.#redis.json.get<{ [key: string]: string }>(cacheKey);
+		if (cachedDescriptions) {
+			console.log(`Cache hit for descriptions for ${cacheKey}`);
+			return cachedDescriptions;
+		}
+
+		console.log(`Cache miss for releases for ${cacheKey}, fetching from GitHub API`);
+
+		const { owner, repoName: repo } = repository;
+
+		const { data: allFiles } = await this.#octokit.rest.git.getTree({
+			owner,
+			repo,
+			tree_sha: "HEAD",
+			recursive: "true"
+		});
+
+		const allPackageJson = allFiles.tree
+			.map(({ path }) => path)
+			.filter(path => path !== undefined)
+			.filter(
+				path =>
+					!path.includes("/test/") && (path === "package.json" || path.endsWith("/package.json"))
+			);
+
+		const descriptions = new Map<string, string>();
+		for (const path of allPackageJson) {
+			const { data: packageJson } = await this.#octokit.rest.repos.getContent({
+				owner,
+				repo,
+				path
+			});
+
+			if (!("content" in packageJson)) continue; // filter out empty or multiple results
+			const { content, encoding, type } = packageJson;
+			if (type !== "file" || !content) continue; // filter out directories and empty files
+			const packageFile =
+				encoding === "base64" ? Buffer.from(content, "base64").toString() : content;
+
+			try {
+				const { description } = JSON.parse(packageFile) as { description: string };
+				if (description) descriptions.set(path, description);
+			} catch {
+				// ignore
+			}
+		}
+
+		await this.#redis.json.set(cacheKey, "$", Object.fromEntries(descriptions));
+
+		return Object.fromEntries(descriptions);
+	}
+
 	/**
 	 * Checks if releases are present in the cache for the
 	 * given GitHub info
@@ -221,10 +290,11 @@ export class GitHubCache {
 	 * existence in the cache for
 	 * @param repo the name of the GitHub repository to check the
 	 * existence in the cache for
+	 * @param type the kind of cache to target
 	 * @returns whether the repository is cached or not
 	 */
-	async exists(owner: string, repo: string) {
-		const cacheKey = this.#getRepoKey(owner, repo);
+	async exists(owner: string, repo: string, type: KeyType) {
+		const cacheKey = this.#getRepoKey(owner, repo, type);
 		const result = await this.#redis.exists(cacheKey);
 		return result === 1;
 	}
@@ -236,9 +306,10 @@ export class GitHubCache {
 	 * from the cache
 	 * @param repo the name of the GitHub repository to remove
 	 * from the cache
+	 * @param type the kind of cache to target
 	 */
-	async deleteEntry(owner: string, repo: string) {
-		const cacheKey = this.#getRepoKey(owner, repo);
+	async deleteEntry(owner: string, repo: string, type: KeyType) {
+		const cacheKey = this.#getRepoKey(owner, repo, type);
 		await this.#redis.del(cacheKey);
 	}
 }

src/lib/server/package-discoverer.ts

@@ -2,15 +2,20 @@ import type { Prettify } from "$lib/types";
 import { GitHubCache, gitHubCache } from "./github-cache";
 import { publicRepos, type Repository } from "$lib/repositories";
 
+type Package = {
+	name: string;
+	description: string;
+};
+
 export type DiscoveredPackage = Prettify<
 	Repository & {
-		packages: string[];
+		packages: Package[];
 	}
 >;
 
 export type CategorizedPackage = Prettify<
 	Pick<Repository, "category"> & {
-		packages: (Omit<Repository, "category"> & { packageName: string })[];
+		packages: (Omit<Repository, "category"> & { pkg: Package })[];
 	}
 >;
 
@@ -33,6 +38,7 @@ export class PackageDiscoverer {
 		this.#packages = await Promise.all(
 			this.#repos.map(async repo => {
 				const releases = await this.#cache.getReleases(repo);
+				const descriptions = await this.#cache.getDescriptions(repo);
 				const packages = [
 					...new Set(
 						releases
@@ -46,11 +52,48 @@ export class PackageDiscoverer {
 				console.log(
 					`Discovered ${packages.length} packages for ${repo.owner}/${repo.repoName}: ${packages.join(", ")}`
 				);
-				return { ...repo, packages };
+				return {
+					...repo,
+					packages: packages.map(pkg => {
+						const ghName = this.#gitHubDirectoryFromName(pkg);
+						return {
+							name: pkg,
+							description:
+								descriptions[`packages/${ghName}/package.json`] ??
+								descriptions[
+									`packages/${ghName.substring(ghName.lastIndexOf("/") + 1)}/package.json`
+								] ??
+								descriptions["package.json"] ??
+								""
+						};
+					})
+				};
 			})
 		);
 	}
 
+	/**
+	 * Returns the directory on GitHub from the name
+	 * of the package.
+	 * Useful to retrieve the correct `package.json` file.
+	 *
+	 * @param name the package name
+	 * @returns the directory name in GitHub for that package
+	 * @private
+	 */
+	#gitHubDirectoryFromName(name: string): string {
+		switch (name) {
+			case "extensions":
+				return "svelte-vscode";
+			case "sv":
+				return "cli";
+			case "svelte-migrate":
+				return "migrate";
+			default:
+				return name;
+		}
+	}
+
 	/**
 	 * Returns the saved packages if they're not empty,
 	 * otherwise calls {@link discoverAll} then returns the
@@ -76,9 +119,9 @@ export class PackageDiscoverer {
 	async getOrDiscoverCategorized() {
 		return (await this.getOrDiscover()).reduce<CategorizedPackage[]>(
 			(acc, { category, ...rest }) => {
-				const formattedPackages = rest.packages.map(packageName => ({
+				const formattedPackages = rest.packages.map(pkg => ({
 					...rest,
-					packageName
+					pkg
 				}));
 
 				for (const [i, item] of acc.entries()) {
@@ -91,9 +134,9 @@ export class PackageDiscoverer {
 				// If the category doesn't exist in the accumulator, create it
 				acc.push({
 					category,
-					packages: rest.packages.map(packageName => ({
+					packages: rest.packages.map(pkg => ({
 						...rest,
-						packageName
+						pkg
 					}))
 				});
 

src/routes/+layout.server.ts

@@ -11,8 +11,8 @@ export async function load() {
 			packages: uniq(
 				res.packages
 					.map(({ dataFilter, metadataFromTag, changelogContentsReplacer, ...rest }) => rest)
-					.toSorted((a, b) => a.packageName.localeCompare(b.packageName)),
-				item => item.packageName
+					.toSorted((a, b) => a.pkg.name.localeCompare(b.pkg.name)),
+				item => item.pkg.name
 			)
 		}))
 	};

src/routes/package/[...package]/+page.server.ts

@@ -9,18 +9,17 @@ export async function load({ params }) {
 	const categorizedPackages = await discoverer.getOrDiscoverCategorized();
 
 	let currentPackage:
-		| (Omit<Repository, "dataFilter" | "metadataFromTag" | "changelogContentsReplacer"> & {
-				packageName: string;
-		  })
+		| (Omit<Repository, "dataFilter" | "metadataFromTag" | "changelogContentsReplacer"> &
+				Pick<(typeof categorizedPackages)[number]["packages"][number], "pkg">)
 		| undefined = undefined;
 	const foundVersions = new Set<string>();
 	const releases: (GitHubRelease & { cleanVersion: string })[] = [];
 
 	// Discover releases
 	console.log("Starting loading releases...");
 	for (const { category, packages } of categorizedPackages) {
-		for (const { packageName, ...repo } of packages) {
-			if (packageName.toLowerCase() === slugPackage.toLowerCase()) {
+		for (const { pkg, ...repo } of packages) {
+			if (pkg.name.toLowerCase() === slugPackage.toLowerCase()) {
 				// 1. Get releases
 				const cachedReleases = await gitHubCache.getReleases({ ...repo, category });
 				console.log(
@@ -65,7 +64,7 @@ export async function load({ params }) {
 						);
 						currentPackage = {
 							category,
-							packageName,
+							pkg,
 							...rest
 						};
 					}

src/routes/package/[...package]/+page.svelte

@@ -23,10 +23,13 @@
 		{@render loading()}
 	{:then}
 		<div class="my-8">
-			<h1 class="text-5xl font-semibold text-primary">{data.currentPackage.packageName}</h1>
+			<h1 class="text-5xl font-semibold text-primary">{data.currentPackage.pkg.name}</h1>
 			<h2 class="text-xl text-muted-foreground">
 				{data.currentPackage.owner}/{data.currentPackage.repoName}
 			</h2>
+			{#if data.currentPackage.pkg.description}
+				<h3 class="mt-4 italic">{data.currentPackage.pkg.description}</h3>
+			{/if}
 		</div>
 		<div class="flex gap-8">
 			<Accordion.Root
@@ -41,14 +44,14 @@
 			>
 				{#each data.releases as release (release.id)}
 					<ReleaseCard
-						packageName={data.currentPackage.packageName}
+						packageName={data.currentPackage.pkg.name}
 						repo={{ owner: data.currentPackage.owner, name: data.currentPackage.repoName }}
 						{release}
 					/>
 				{/each}
 			</Accordion.Root>
 			<SidePanel
-				packageName={data.currentPackage.packageName}
+				packageName={data.currentPackage.pkg.name}
 				allPackages={data.displayablePackages}
 				class="h-fit w-2/5"
 			/>

src/routes/package/[...package]/SidePanel.svelte

@@ -35,10 +35,10 @@
 					<li class="space-y-2">
 						<h3 class="text-xl font-bold text-primary">{category.name}</h3>
 						<ul class="space-y-2">
-							{#each packages as { packageName } (packageName)}
+							{#each packages as { pkg } (pkg.name)}
 								<li>
-									<a href="/package/{packageName}" class="underline-offset-4 hover:underline">
-										{packageName}
+									<a href="/package/{pkg.name}" class="underline-offset-4 hover:underline">
+										{pkg.name}
 									</a>
 								</li>
 							{/each}

src/routes/packages/+page.svelte

@@ -10,17 +10,17 @@
 		<li>
 			<h3 class="text-2xl font-bold text-primary">{category.name}</h3>
 			<ul class="mt-2">
-				{#each packages as { owner, repoName, packageName }, index (packageName)}
+				{#each packages as { owner, repoName, pkg }, index (pkg.name)}
 					{#if index > 0}
 						<Separator class="mx-auto my-1 w-[95%]" />
 					{/if}
 					<li>
 						<a
-							href="/package/{packageName}"
+							href="/package/{pkg.name}"
 							class="group flex items-center rounded-lg px-4 py-3 transition-colors hover:bg-neutral-800"
 						>
 							<div class="flex flex-col">
-								<h4 class="font-medium">{packageName}</h4>
+								<h4 class="font-medium">{pkg.name}</h4>
 								{#if category.slug === "others"}
 									<span class="text-muted-foreground">
 										{owner}/{repoName}