|
| 1 | +import { PACKAGES_META } from '../../src/lib/packages-meta.ts'; |
| 2 | +import type { PackageKey, PackageNpm, PackageGithub } from '../../src/lib/server/content.ts'; |
| 3 | +import { execSync } from 'node:child_process'; |
| 4 | +import fs from 'node:fs'; |
| 5 | +import path from 'node:path'; |
| 6 | +import process from 'node:process'; |
| 7 | + |
| 8 | +let skipGithubStars = false; |
| 9 | + |
| 10 | +const start = performance.now(); |
| 11 | +console.log('[sync-packages] start'); |
| 12 | + |
| 13 | +let logsAtTheEnd: { |
| 14 | + type: |
| 15 | + | 'no_repo_url' |
| 16 | + | 'low_downloads' |
| 17 | + | 'low_github_stars' |
| 18 | + | 'new_json_file' |
| 19 | + | 'deleted_unused_json_file' |
| 20 | + | 'outdated' |
| 21 | + | 'deprecated' |
| 22 | + | 'npm_error'; |
| 23 | + pkg: string; |
| 24 | + extra: string; |
| 25 | +}[] = []; |
| 26 | + |
| 27 | +const packages = PACKAGES_META.FEATURED.flatMap((pkg) => pkg.packages).map((pkg) => pkg.name); |
| 28 | + |
| 29 | +const registryFolder = 'src/lib/server/generated/registry'; |
| 30 | + |
| 31 | +// PART 1: create missing json files |
| 32 | +for (const pkg of packages) { |
| 33 | + const cleanPkg = pkg.replace('@', '').replace('/', '-'); |
| 34 | + const jsonPath = path.join(registryFolder, `${cleanPkg}.json`); |
| 35 | + if (!fs.existsSync(jsonPath)) { |
| 36 | + const p = await getNpmAndGitHubData(pkg); |
| 37 | + writeJsonData(jsonPath, p); |
| 38 | + logsAtTheEnd.push({ type: 'new_json_file', pkg, extra: `created -> ${jsonPath}` }); |
| 39 | + } |
| 40 | +} |
| 41 | + |
| 42 | +// PART 2: delete unused json files |
| 43 | +let registryJsonFiles = fs.readdirSync(registryFolder); |
| 44 | +const jsonUsed: string[] = []; |
| 45 | +for (const pkg of packages) { |
| 46 | + const cleanPkg = pkg.replace('@', '').replace('/', '-'); |
| 47 | + const cleanPkgFile = `${cleanPkg}.json`; |
| 48 | + const jsonPath = path.join(registryFolder, cleanPkgFile); |
| 49 | + if (fs.existsSync(jsonPath)) { |
| 50 | + jsonUsed.push(cleanPkgFile); |
| 51 | + } |
| 52 | +} |
| 53 | +const jsonNotNeeded = registryJsonFiles.filter((pkg) => !jsonUsed.includes(pkg)); |
| 54 | +if (jsonNotNeeded.length > 0) { |
| 55 | + // delete json files |
| 56 | + for (const pkg of jsonNotNeeded) { |
| 57 | + const jsonPath = path.join(registryFolder, pkg); |
| 58 | + fs.unlinkSync(jsonPath); |
| 59 | + logsAtTheEnd.push({ type: 'deleted_unused_json_file', pkg, extra: `deleted -> ${jsonPath}` }); |
| 60 | + } |
| 61 | + |
| 62 | + // Let's continue |
| 63 | + // theEnd(1); |
| 64 | +} |
| 65 | + |
| 66 | +// PART 3: refresh data |
| 67 | +registryJsonFiles = fs.readdirSync(registryFolder); //.slice(0, 20); |
| 68 | + |
| 69 | +const batch = 10; |
| 70 | +for (let i = 0; i < registryJsonFiles.length; i += batch) { |
| 71 | + const batchFiles = registryJsonFiles.slice(i, i + batch); |
| 72 | + await Promise.all( |
| 73 | + batchFiles.map(async (pkg) => { |
| 74 | + await refreshJsonFile(path.join(registryFolder, pkg)); |
| 75 | + }) |
| 76 | + ); |
| 77 | +} |
| 78 | + |
| 79 | +theEnd(0); |
| 80 | + |
| 81 | +// HELPERS |
| 82 | + |
| 83 | +function theEnd(val: number) { |
| 84 | + const msg = ['[sync-packages]']; |
| 85 | + if (val > 0) { |
| 86 | + msg.push(`exit(${val}) - `); |
| 87 | + } |
| 88 | + msg.push(`took: ${(performance.now() - start).toFixed(0)}ms`); |
| 89 | + console.log(msg.join(' ')); |
| 90 | + if (logsAtTheEnd.length > 0) { |
| 91 | + console.log('[sync-packages] report:'); |
| 92 | + const typePrints: Record<(typeof logsAtTheEnd)[number]['type'], string> = { |
| 93 | + no_repo_url: 'No GitHub URL', |
| 94 | + low_downloads: 'Low Downloads', |
| 95 | + low_github_stars: 'Low Stars', |
| 96 | + new_json_file: 'NEW JSON', |
| 97 | + deleted_unused_json_file: 'DEL JSON', |
| 98 | + outdated: 'Outdated', |
| 99 | + deprecated: 'Deprecated', |
| 100 | + npm_error: 'NPM Error' |
| 101 | + }; |
| 102 | + console.log( |
| 103 | + ` - ${logsAtTheEnd.map((l) => `${typePrints[l.type].padEnd(15)} | ${l.pkg.padEnd(35)} | ${l.extra}`).join('\n - ')}` |
| 104 | + ); |
| 105 | + } |
| 106 | + process.exit(val); |
| 107 | +} |
| 108 | + |
| 109 | +async function getNpmAndGitHubData(pkg: string): Promise<PackageKey & PackageNpm & PackageGithub> { |
| 110 | + const [npmInfo, npmDlInfo] = await Promise.all([ |
| 111 | + fetchJson(`https://registry.npmjs.org/${pkg}`), |
| 112 | + fetchJson(`https://api.npmjs.org/downloads/point/last-week/${pkg}`) |
| 113 | + ]); |
| 114 | + |
| 115 | + if (npmInfo.error) { |
| 116 | + logsAtTheEnd.push({ type: 'npm_error', pkg, extra: npmInfo.error }); |
| 117 | + theEnd(1); |
| 118 | + } |
| 119 | + |
| 120 | + // delete npmInfo.readme; |
| 121 | + // delete npmInfo.versions; |
| 122 | + // console.log(`npmInfo`, npmInfo); |
| 123 | + |
| 124 | + const npm_description = npmInfo.description; |
| 125 | + const raw_repo_url = npmInfo.repository?.url ?? ''; |
| 126 | + const repo_url = raw_repo_url?.replace(/^git\+/, '').replace(/\.git$/, ''); |
| 127 | + if (!repo_url) { |
| 128 | + // console.error(`repo_url not found for ${pkg}`); |
| 129 | + logsAtTheEnd.push({ type: 'no_repo_url', pkg, extra: `not found` }); |
| 130 | + } |
| 131 | + const git_org = repo_url?.split('/')[3]; |
| 132 | + const git_repo = repo_url?.split('/')[4]; |
| 133 | + |
| 134 | + const authors = npmInfo.maintainers?.map((m: { name: string }) => m.name); |
| 135 | + const homepage = npmInfo.homepage; |
| 136 | + const downloads = npmDlInfo.downloads; |
| 137 | + const version = npmInfo['dist-tags'].latest; |
| 138 | + const updated = npmInfo.time[version]; |
| 139 | + const pkgInfo = npmInfo.versions[version]; |
| 140 | + const deprecated_reason = pkgInfo.deprecated; |
| 141 | + |
| 142 | + const svelte_range = |
| 143 | + pkgInfo.peerDependencies?.svelte ?? |
| 144 | + pkgInfo.dependencies?.svelte ?? |
| 145 | + pkgInfo.devDependencies?.svelte; |
| 146 | + |
| 147 | + const kit_range = |
| 148 | + pkgInfo.peerDependencies?.['@sveltejs/kit'] ?? |
| 149 | + pkgInfo.dependencies?.['@sveltejs/kit'] ?? |
| 150 | + pkgInfo.devDependencies?.['@sveltejs/kit']; |
| 151 | + |
| 152 | + // GitHub |
| 153 | + let github_stars: number | undefined = undefined; |
| 154 | + if (git_org && git_repo && !skipGithubStars) { |
| 155 | + const token = process.env.GITHUB_TOKEN; |
| 156 | + const headers = token ? new Headers({ authorization: 'Bearer ' + token }) : {}; |
| 157 | + const res = await fetchJson(`https://api.github.com/repos/${git_org}/${git_repo}`, { headers }); |
| 158 | + if (res?.message && res?.message.startsWith('API rate limit exceeded')) { |
| 159 | + skipGithubStars = true; |
| 160 | + } else { |
| 161 | + github_stars = res.stargazers_count; |
| 162 | + } |
| 163 | + } |
| 164 | + |
| 165 | + return { |
| 166 | + name: pkg, |
| 167 | + npm_description, |
| 168 | + repo_url, |
| 169 | + authors, |
| 170 | + homepage, |
| 171 | + version, |
| 172 | + deprecated_reason, |
| 173 | + downloads, |
| 174 | + updated, |
| 175 | + svelte_range, |
| 176 | + kit_range, |
| 177 | + |
| 178 | + // GitHub |
| 179 | + github_stars |
| 180 | + }; |
| 181 | +} |
| 182 | + |
| 183 | +async function refreshJsonFile(fullPath: string) { |
| 184 | + console.log(`Refreshing:`, fullPath); |
| 185 | + |
| 186 | + const currentJson = JSON.parse(fs.readFileSync(fullPath, 'utf-8')); |
| 187 | + const newData = await getNpmAndGitHubData(currentJson.name); |
| 188 | + |
| 189 | + // remove all undefined values |
| 190 | + for (const key in newData) { |
| 191 | + if (newData[key] === undefined) { |
| 192 | + delete newData[key]; |
| 193 | + } |
| 194 | + } |
| 195 | + |
| 196 | + // Let's not filter npm downloads changes |
| 197 | + // // filter changes of downloads |
| 198 | + // if (newData.downloads && currentJson.downloads) { |
| 199 | + // const dlDelta = Math.abs( |
| 200 | + // ((newData.downloads - currentJson.downloads) / currentJson.downloads) * 100 |
| 201 | + // ); |
| 202 | + // if (dlDelta < 2.5) delete newData.downloads; |
| 203 | + // } |
| 204 | + |
| 205 | + const data = { ...currentJson, ...newData }; |
| 206 | + |
| 207 | + // Some stats infos to log |
| 208 | + if (data.downloads && data.downloads < 255) { |
| 209 | + logsAtTheEnd.push({ type: 'low_downloads', pkg: data.name, extra: `${data.downloads}` }); |
| 210 | + } |
| 211 | + |
| 212 | + if (data.github_stars && data.github_stars < 42) { |
| 213 | + logsAtTheEnd.push({ |
| 214 | + type: 'low_github_stars', |
| 215 | + pkg: data.name, |
| 216 | + extra: `${data.github_stars}` |
| 217 | + }); |
| 218 | + } |
| 219 | + |
| 220 | + if (data.updated && PACKAGES_META.is_outdated(data.updated)) { |
| 221 | + logsAtTheEnd.push({ type: 'outdated', pkg: data.name, extra: `${data.updated}` }); |
| 222 | + } |
| 223 | + |
| 224 | + if (data.deprecated_reason) { |
| 225 | + logsAtTheEnd.push({ type: 'deprecated', pkg: data.name, extra: `${data.deprecated_reason}` }); |
| 226 | + } |
| 227 | + |
| 228 | + writeJsonData(fullPath, data); |
| 229 | +} |
| 230 | + |
| 231 | +function writeJsonData(path: string, data: any) { |
| 232 | + const keysOrder: (keyof PackageKey | keyof PackageNpm | keyof PackageGithub)[] = [ |
| 233 | + 'name', |
| 234 | + 'npm_description', |
| 235 | + 'repo_url', |
| 236 | + 'authors', |
| 237 | + 'homepage', |
| 238 | + 'version', |
| 239 | + 'deprecated_reason', |
| 240 | + 'downloads', |
| 241 | + 'github_stars', |
| 242 | + 'updated', |
| 243 | + 'svelte_range', |
| 244 | + 'kit_range', |
| 245 | + 'typescript', |
| 246 | + 'runes', |
| 247 | + 'last_rune_check_version' |
| 248 | + ]; |
| 249 | + |
| 250 | + const sortedData: Record<string, any> = {}; |
| 251 | + for (const key of keysOrder) { |
| 252 | + if (data[key] !== undefined) { |
| 253 | + sortedData[key] = data[key]; |
| 254 | + } |
| 255 | + } |
| 256 | + // all all the remaining keys |
| 257 | + for (const key in data) { |
| 258 | + if (!keysOrder.includes(key as keyof PackageKey | keyof PackageNpm | keyof PackageGithub)) { |
| 259 | + sortedData[key] = data[key]; |
| 260 | + } |
| 261 | + } |
| 262 | + |
| 263 | + fs.writeFileSync(path, JSON.stringify(sortedData, null, 2)); |
| 264 | + execSync(`prettier --write ${path}`); |
| 265 | +} |
| 266 | + |
| 267 | +async function fetchJson(url: string, options: RequestInit = {}): Promise<any> { |
| 268 | + const headers = new Headers({ ...options.headers, 'User-Agent': 'svelte.dev/packages_v0.0.1' }); |
| 269 | + |
| 270 | + for (let i = 0; i < 5; i++) { |
| 271 | + try { |
| 272 | + const res = await fetch(url, { ...options, headers }); |
| 273 | + return await res.json(); |
| 274 | + } catch (e) { |
| 275 | + console.error(`Failed to fetch ${url} after ${i + 1} retries`); |
| 276 | + } |
| 277 | + |
| 278 | + await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, i + 1))); |
| 279 | + } |
| 280 | +} |
0 commit comments