[Docs] Update API link crawler (#18794)

Author: pedrosousa

bin/crawl-api-links.js

@@ -21,6 +21,13 @@ async function checkLinks() {
 	});
 	const page = await browser.newPage();
 
+	// skip image requests
+	await page.setRequestInterception(true);
+	page.on("request", (request) => {
+		if (request.resourceType() === "image") request.abort();
+		else request.continue();
+	});
+
 	const sitemapUrl = "https://developers.cloudflare.com/sitemap.xml";
 	await page.goto(sitemapUrl, { timeout: navigationTimeout });
 
@@ -51,23 +58,22 @@ async function checkLinks() {
 			}
 
 			if (
-				pageLink.includes("developers.cloudflare.com/api/operations/") ||
-				pageLink.startsWith("/api/operations/")
+				pageLink.includes("developers.cloudflare.com/api/resources/") ||
+				pageLink.startsWith("/api/resources/")
 			) {
 				console.log(`Evaluating link: ${pageLink}`);
-				await page.goto(pageLink, {
+				const response = await page.goto(pageLink, {
 					waitUntil: "networkidle0",
 					timeout: navigationTimeout,
 				});
 				visitedLinks.push(pageLink);
 
-				const statusCode = await page.evaluate(() => {
-					return {
-						url: window.location.href,
-					};
-				});
-				if (statusCode.url === "https://developers.cloudflare.com/api/") {
-					brokenLinks.push(pageLink);
+				if (response) {
+					if (response.status() === 404) {
+						brokenLinks.push(pageLink);
+					}
+				} else {
+					console.log("WARNING: Didn't receive a response... skipping.");
 				}
 			}
 		}

package.json

@@ -10,6 +10,7 @@
 		"check": "npm run check:astro && npm run check:worker",
 		"check:astro": "npm run sync && astro check",
 		"check:worker": "npm run build:worker && npx tsc --noEmit -p ./worker/tsconfig.json",
+		"crawl-api-links": "node bin/crawl-api-links.js",
 		"dev": "npx astro dev",
 		"format": "npm run format:core && npm run format:data",
 		"format:core": "npx prettier --write \"**/*.{js,jsx,ts,tsx,mjs,css}\"",