actually replace links

Author: cowboyd

staticalize.ts

@@ -1,9 +1,16 @@
-import { call, type Operation, resource, spawn, useAbortSignal } from "effection";
+import {
+  call,
+  type Operation,
+  resource,
+  spawn,
+  until,
+  useAbortSignal,
+} from "effection";
 import { dirname, join, normalize } from "@std/path";
 import { ensureDir } from "@std/fs/ensure-dir";
 import { stringify } from "@libs/xml/stringify";
 import { fromHtml } from "hast-util-from-html";
-//import { toHtml } from "hast-util-to-html";
+import { toHtml } from "hast-util-to-html";
 import { selectAll } from "hast-util-select";
 import { parse } from "@libs/xml/parse";
 import { useTaskBuffer } from "./task-buffer.ts";
@@ -43,7 +50,7 @@ export function* staticalize(options: StaticalizeOptions): Operation<void> {
     return Array.isArray(urls) ? urls : [urls];
   });
 
-  let downloader = yield* useDownloader({ host, outdir: dir });
+  let downloader = yield* useDownloader({ host, base, outdir: dir });
 
   yield* call(() => ensureDir(dir));
 
@@ -82,13 +89,14 @@ interface Downloader extends Operation<void> {
 
 interface DownloaderOptions {
   host: URL;
+  base: URL;
   outdir: string;
 }
 
 function useDownloader(opts: DownloaderOptions): Operation<Downloader> {
   let seen = new Map<string, boolean>();
   return resource(function* (provide) {
-    let { host, outdir } = opts;
+    let { host, base, outdir } = opts;
 
     let buffer = yield* useTaskBuffer(75);
 
@@ -116,27 +124,45 @@ function useDownloader(opts: DownloaderOptions): Operation<Downloader> {
           if (response.ok) {
             if (response.headers.get("Content-Type")?.includes("html")) {
               let destpath = join(path, "index.html");
-              let content = yield* call(() => response.text());
-	      let html = fromHtml(content);
+              let content = yield* until(response.text());
+              let html = fromHtml(content);
 
               let links = selectAll("link[href]", html);
 
               for (let link of links) {
-		let href = link.properties.href as string
+                let href = link.properties.href as string;
                 yield* downloader.download(href, source);
+
+                // replace self-referencing absolute urls with the destination site
+                if (href.startsWith(host.origin)) {
+                  let url = new URL(href);
+                  url.host = base.host;
+                  url.port = base.port;
+                  url.protocol = base.protocol;
+                  link.properties.href = url.href;
+                }
               }
-	      
+
               let assets = selectAll("[src]", html);
 
               for (let element of assets) {
-		let src = element.properties.src as string;
+                let src = element.properties.src as string;
                 yield* downloader.download(src, source);
+
+                // replace self-referencing absolute urls with the destination sie
+                if (src.startsWith(host.origin)) {
+                  let url = new URL(src);
+                  url.host = base.host;
+                  url.port = base.port;
+                  url.protocol = base.protocol;
+                  element.properties.src = url.href;
+                }
               }
 
               yield* call(async () => {
                 let destdir = dirname(destpath);
                 await ensureDir(destdir);
-                await Deno.writeTextFile(destpath, content);
+                await Deno.writeTextFile(destpath, toHtml(html));
               });
             } else {
               yield* call(async () => {

task-buffer.ts

@@ -2,13 +2,13 @@ import {
   createChannel,
   Err,
   Ok,
-  Operation,
-  Resolve,
+  type Operation,
+  type Resolve,
   resource,
-  Result,
+  type Result,
   spawn,
-  Stream,
-  Task,
+  type Stream,
+  type Task,
   useScope,
   withResolvers,
 } from "effection";

test/staticalize.test.ts

@@ -6,7 +6,7 @@ import { emptyDir, exists } from "@std/fs";
 
 import { Hono } from "jsr:@hono/hono";
 import { parse } from "@libs/xml";
-import { run, Task } from "effection";
+import { run, type Task } from "effection";
 
 describe("staticalize", () => {
   let server: ReturnType<typeof Deno.serve>;
@@ -58,13 +58,13 @@ describe("staticalize", () => {
     });
 
     await expect(content("test/dist/index.html")).resolves.toEqual(
-      "<h1>Index</h1>",
+      "<html><head></head><body><h1>Index</h1></body></html>",
     );
     await expect(content("test/dist/about/index.html")).resolves.toEqual(
-      "<h1>About</h1>",
+      "<html><head></head><body><h1>About</h1></body></html>",
     );
     await expect(content("test/dist/contact/index.html")).resolves.toEqual(
-      "<h1>Contact</h1>",
+      "<html><head></head><body><h1>Contact</h1></body></html>",
     );
 
     let xml = parse(await Deno.readTextFile("test/dist/sitemap.xml"));
@@ -88,7 +88,7 @@ describe("staticalize", () => {
       dir: "test/dist",
     });
     expect(content("test/dist/deeply/nested/page/index.html")).resolves.toEqual(
-      "<h1>Nested</h1>",
+      "<html><head></head><body><h1>Nested</h1></body></html>",
     );
   });
 
@@ -188,18 +188,19 @@ describe("staticalize", () => {
     await expect(exists("test/dist/assets/styles.css")).resolves.toEqual(true);
   });
 
-  it.skip("replaces references to absolute assets that have the sae host that we're scraping", async () => {
+  it("replaces references to absolute assets that have the sae host that we're scraping", async () => {
     const html = `
 <html>
   <head>
     <link rel="canonical"  href="${host}"/>
-    <link rel="alternate"  href="${host}alt"/>
+    <script src="${host}main.js"/>
   </head>
+  <body></body>
 </html>
-`
-    app.get("/", (c) =>
-      c.html(html))
+`;
+    app.get("/", (c) => c.html(html))
       .get("/alt", (c) => c.html(html))
+      .get("/main.js", (c) => c.text("console.log('hi')"))
       .get(...sitemap(["/"]));
 
     await staticalize({
@@ -208,14 +209,12 @@ describe("staticalize", () => {
       dir: "test/dist",
     });
 
-    await expect(content("test/dist/index.html")).resolves.toEqual(`
-<html>
-  <head>
-    <link rel="canonical"  href="https://fs.com/"/>
-    <link rel="alternate"  href="https://fs.com/alt"/>
-  </head>
-</html>
-`);
+    await expect(content("test/dist/index.html")).resolves.toMatch(
+      `<link rel="canonical" href="https://fs.com/">`,
+    );
+    await expect(content("test/dist/index.html")).resolves.toMatch(
+      `<script src="https://fs.com/main.js">`,
+    );
   });
 });