Update npm stats

Author: PuruVJ

apps/svelte.dev/scripts/registry/npm.ts

@@ -314,11 +314,11 @@ function format(date: Date, format_str: string): string {
 	return format_str.replace('yyyy', year.toString()).replace('MM', month).replace('dd', day);
 }
 
-const API_BASE_URL = 'https://api.npmjs.org/downloads';
+const API_BASE_URL = 'https://api.npmjs.org/';
 const REGISTRY_BASE_URL = 'https://registry.npmjs.org/';
 
 const END_DATE = format(new Date(), 'yyyy-MM-dd');
-const START_DATE = format(sub_days(new Date(), 30), 'yyyy-MM-dd');
+const START_DATE = format(sub_days(new Date(), 7), 'yyyy-MM-dd');
 
 const PAGE_SIZE = 100;
 
@@ -337,7 +337,7 @@ export async function fetch_downloads_for_package(pkg: string): Promise<number>
 /**
  * Gets details for a package from the npm registry
  */
-export async function fetch_details_for_package(pkg: string) {
+export async function fetch_details_for_package(pkg: string): Promise<any> {
 	const registry_data = await superfetch(`${REGISTRY_BASE_URL}${pkg}`, { headers: HEADERS }).then(
 		(r) => r.json()
 	);

apps/svelte.dev/scripts/registry/update-registry.ts

@@ -1,16 +1,15 @@
 // Credit for original: Astro team
 import { createOpenRouter } from '@openrouter/ai-sdk-provider';
-import { extract_frontmatter, transform } from '@sveltejs/site-kit/markdown';
 import { generateText } from 'ai';
 import dotenv from 'dotenv';
 import fs, { mkdirSync } from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import glob from 'tiny-glob/sync.js';
 import registry from '../../src/lib/registry.json' with { type: 'json' };
-import type { Package } from '../../src/lib/server/content.js';
 import {
 	fetch_details_for_package,
+	fetch_downloads_for_package,
 	HEADERS,
 	PackageCache,
 	request_queue,
@@ -20,6 +19,7 @@ import {
 	superfetch,
 	type StructuredInterimPackage
 } from './npm.js';
+import type { Package } from '../../src/lib/server/content.js';
 
 dotenv.config({ path: '.env.local' });
 
@@ -452,17 +452,98 @@ async function update_all_github_stars(ignore_if_exists = false) {
 	}
 }
 
-// Gets the cached files, goes through npm for each, and udpate
-// all relevant npm information from that
+/**
+ * Updates package cache with latest npm data
+ * Uses existing request_queue from superfetch for concurrency control
+ */
 async function update_cache_from_npm() {
-	// for (const [pkg_file_name, data] of PackageCache.entries()) {
-	// console.log(await fetch_details_for_package(data.name));
-	// }
-	const detail = await fetch_details_for_package('svelte-drag');
+	// Track progress
+	let processedCount = 0;
+	const updatedPackages: {
+		pkg_name: string;
+		data: Package;
+	}[] = [];
+
+	// Collect all package entries to count total
+	const packages = [];
+	for await (const [pkg_name, data] of PackageCache.entries()) {
+		packages.push({ pkg_name, data });
+	}
+
+	const totalPackages = packages.length;
+	console.log(`Starting update for ${totalPackages} packages`);
+
+	// Create array of promises for package updates
+	const promises = packages.map(({ pkg_name, data }) => {
+		// These fetch functions already use request_queue internally via superfetch
+		return Promise.all([
+			fetch_details_for_package(data.name),
+			fetch_downloads_for_package(data.name)
+		])
+			.then(([details, downloads]) => {
+				try {
+					if (!details) {
+						console.warn(`No details found for package: ${data.name}`);
+						return null;
+					}
+
+					const latest_version = details['dist-tags']?.latest;
+					if (!latest_version) {
+						console.warn(`No latest version found for package: ${data.name}`);
+						return null;
+					}
+
+					// Get last update date and deprecation status
+					const updated = details.time?.[latest_version]
+						? new Date(details.time[latest_version])
+						: undefined;
+					const deprecated = details.versions?.[latest_version]?.deprecated || false;
+
+					// Update the package data
+					const updatedData = {
+						...data,
+						downloads,
+						updated: updated?.toISOString(),
+						deprecated: deprecated || undefined // Only include if true
+					};
+
+					// Save updated data back to cache
+					PackageCache.set(data.name, updatedData);
+
+					// Track progress
+					processedCount++;
+					if (processedCount % 10 === 0 || processedCount === totalPackages) {
+						console.log(
+							`Progress: ${processedCount}/${totalPackages} (${Math.round((processedCount / totalPackages) * 100)}%)`
+						);
+					}
+
+					// Store updated package info
+					const packageInfo: Package = {
+						...data,
+						downloads,
+						updated: updated?.toISOString(),
+						deprecated
+					};
+
+					updatedPackages.push({ data: packageInfo as any, pkg_name: data.name });
+					return packageInfo;
+				} catch (error) {
+					console.error(`Error processing package ${data.name}:`, error);
+					return null;
+				}
+			})
+			.catch((error) => {
+				console.error(`Error fetching package ${data.name}:`, error);
+				return null;
+			});
+	});
 
-	const latestVersion = detail['dist-tags']?.latest;
+	// Wait for all updates to complete
+	await Promise.all(promises);
 
-	console.log(detail.versions[latestVersion]);
+	console.log(`Update completed for ${processedCount} packages`);
+	return updatedPackages.filter(Boolean);
 }
 
 async function delete_untagged() {
@@ -491,6 +572,6 @@ async function* create_map_batch_generator(
 
 // const svelte_packages = await process_batches_through_llm();
 
-// update_cache_from_npm();
-update_all_github_stars();
+update_cache_from_npm();
+// update_all_github_stars();
 // delete_untagged();

apps/svelte.dev/src/lib/server/generated/registry/0xclearview-svelte-tiny-virtual-table.md

@@ -4,7 +4,7 @@ description: "A tiny but mighty list virtualization component for svelte, with z
 repo_url: "https://github.com/0xclearview/svelte-tiny-virtual-table"
 author: "Marcos Gutiérrez"
 homepage: "https://github.com/0xclearview/svelte-tiny-virtual-table#readme"
-downloads: 6
+downloads: 9
 updated: "2024-11-16T13:11:37.125Z"
 tags: 
   - ui

apps/svelte.dev/src/lib/server/generated/registry/3--svelte-preprocess.md

@@ -4,7 +4,7 @@ description: "A Svelte preprocessor wrapper with baked-in support for commonly u
 repo_url: "https://github.com/i18n-fork/svelte-preprocess"
 author: "Christian Kaisermann"
 homepage: "https://github.com/i18n-fork/svelte-preprocess#readme"
-downloads: 2
+downloads: 16
 updated: "2024-10-21T06:10:48.516Z"
 tags: 
   - preprocessor

apps/svelte.dev/src/lib/server/generated/registry/3xpo-svelte-colour-picker.md

@@ -4,7 +4,7 @@ description: "A highly customizable color picker component library"
 repo_url: "https://github.com/Exponential-Workload/svelte-colour-picker"
 author: "Maxime Dupont"
 homepage: "https://gh.expo.moe/svelte-colour-picker/"
-downloads: 2
+downloads: 8
 updated: "2024-03-01T19:10:08.365Z"
 tags: 
   - ui

apps/svelte.dev/src/lib/server/generated/registry/3xpo-svelte-slider.md

@@ -4,7 +4,7 @@ description: "A highly customizable slider component library"
 repo_url: "https://github.com/Exponential-Workload/svelte-colour-picker"
 author: "Expo"
 homepage: "https://gh.expo.moe/svelte-slider/"
-downloads: 2
+downloads: 4
 dependents: 1
 updated: "2024-01-04T08:25:23.273Z"
 tags: 

apps/svelte.dev/src/lib/server/generated/registry/6edesign-svelte-sortablejs.md

@@ -1,6 +1,6 @@
 ---
 name: "@6edesign/svelte-sortablejs"
-description: "<p align="center">   <img  src="https://github.com/solidsnail/svelte-sortablejs/raw/master/docs/logo.png"> </p>"
+description: """<p align="center">   <img  src="https://github.com/solidsnail/svelte-sortablejs/raw/master/docs/logo.png"> </p>"""
 author: "6edesign"
 downloads: 2
 updated: "2020-10-26T21:07:23.556Z"

apps/svelte.dev/src/lib/server/generated/registry/6edesign-svelte-three.md

@@ -2,7 +2,7 @@
 name: "@6edesign/svelte-three"
 description: "Demo"
 author: "6edesign"
-downloads: 4
+downloads: 23
 updated: "2021-01-20T04:40:39.253Z"
 tags: 
   - ui

apps/svelte.dev/src/lib/server/generated/registry/a-luna-svelte-simple-tables.md

@@ -4,7 +4,7 @@ description: "Accessible, sortable, paginated table component"
 repo_url: "https://github.com/a-luna/svelte-simple-tables"
 author: "Aaron Luna"
 homepage: "https://github.com/a-luna/svelte-simple-tables"
-downloads: 27
+downloads: 47
 updated: "2022-07-19T09:27:18.684Z"
 github_stars: 6
 tags: 

apps/svelte.dev/src/lib/server/generated/registry/abineo-svelte-forms.md

@@ -4,7 +4,7 @@ description: "Well tested form validation for svelte & sveltekit (js/ts)"
 repo_url: "https://github.com/abineo-ag/svelte-forms"
 author: "christoph-jeanluc-schneider"
 homepage: "https://github.com/abineo-ag/svelte-forms#readme"
-downloads: 1
+downloads: 14
 updated: "2023-06-27T14:48:12.451Z"
 tags: 
   - utility