first draft to measure compressed and uncompressed size

Author: krausest

server/app.js

@@ -5,6 +5,12 @@ import staticRouter from "./src/static/staticRouter.js";
 import * as ejs from "ejs";
 import * as fastifyView from "@fastify/view";
 import minifier from "html-minifier";
+import { Stream } from "stream";
+import toArray from "stream-to-array";
+import zlib from "node:zlib";
+import { pipeline } from "node:stream";
+import { promisify } from "node:util";
+import { getSizeRouter } from "./src/responseSize/responseSizeRouter.js";
 
 /**
  * Builds the server but does not start it. Need it for testing API
@@ -23,8 +29,92 @@ function buildServer(options = {}) {
     },
   });
 
+  fastify.decorate("responseSize", {
+    use_compression: false,
+    size_uncompressed: 0,
+    size_compressed: 0,
+    get: function() {
+      return {
+        use_compression: this.use_compression,
+        size_uncompressed: this.size_uncompressed,
+        size_compressed: this.size_compressed,
+      };
+    },
+    reset: function() {
+      this.size_uncompressed = 0;
+      this.size_compressed = 0;
+    },
+    enableCompression: function(val) {
+      this.use_compression = val;
+    },
+    add: function(uncompressed, compressed) {
+      this.size_uncompressed += uncompressed;
+      this.size_compressed += compressed;
+    },
+  });
+
+  fastify.register(getSizeRouter);
+
+  fastify.addHook("onSend", async (request, reply, payload) => {
+    // const MISSING_HEADERS_AND_HTTP = 99;
+    let getSizeInfo = (original, compressed) => {
+      if (!compressed) {
+        compressed = original;
+      } else {
+        reply.header("Content-Length", compressed.length);
+        reply.header("Content-Encoding", "br");
+      }
+      let headers = Object.entries(reply.getHeaders()).reduce((p, [e, v]) => p + `${e}: ${v} \n`, "");
+      // console.log(request.url, reply.statusCode, "\n", headers);
+      return {
+        compressed: compressed.length, // + headers.length + MISSING_HEADERS_AND_HTTP,
+        uncompressed: original.length, // + headers.length + MISSING_HEADERS_AND_HTTP
+      };
+    };
+
+    if (request.url.startsWith("/css") || reply.statusCode != 200 || !fastify.responseSize.use_compression) {
+      return payload;
+    } else {
+      if (typeof payload == "string") {
+        let { uncompressed, compressed } = getSizeInfo(payload);
+        fastify.responseSize.add(uncompressed, compressed);
+        console.log(
+          `onSend: ${request.url} as string with uncompressed size ${uncompressed} sum uncompressed ${fastify.responseSize.size_uncompressed}, compressed ${fastify.responseSize.size_compressed}`
+        );
+      } else if (payload instanceof Stream) {
+        return toArray(payload)
+          .then((chunks) => {
+            const buffer = Buffer.concat(chunks);
+            if (buffer.length >= 1024) {
+              let out = zlib.brotliCompressSync(buffer);
+              let { uncompressed, compressed } = getSizeInfo(buffer, out);
+              fastify.responseSize.add(uncompressed, compressed);
+              console.log(
+                `onSend: ${request.url} as stream with uncompressed size ${uncompressed} compressed ${compressed} sum uncompressed ${fastify.responseSize.size_uncompressed}, compressed ${fastify.responseSize.size_compressed}`
+              );
+              return out;
+            } else {
+              let { uncompressed, compressed } = getSizeInfo(buffer);
+              fastify.responseSize.add(uncompressed, compressed);
+              console.log(
+                `onSend: ${request.url} as stream with uncompressed size ${uncompressed} (not compressed since below threshold) sum uncompressed ${fastify.responseSize.size_uncompressed}, compressed ${fastify.responseSize.size_compressed}`
+              );
+              return buffer;
+            }
+          })
+          .catch((err) => {
+            console.log("onSend: Error", err);
+          });
+      } else {
+        console.log("onSend: Unknown payload type", typeof payload, payload);
+      }
+      return payload;
+    }
+  });
+
   fastify.addHook("onRequest", (request, reply, done) => {
     if (request.url.endsWith("index.html")) {
+      fastify.responseSize.reset();
       reply.header("Cross-Origin-Embedder-Policy", "require-corp");
       reply.header("Cross-Origin-Opener-Policy", "same-origin");
     }

server/package-lock.json

@@ -18,7 +18,8 @@
     "@fastify/view": "^8.2.0",
     "ejs": "^3.1.9",
     "fastify": "^4.23.2",
-    "html-minifier": "^4.0.0"
+    "html-minifier": "^4.0.0",
+    "stream-to-array": "^2.3.0"
   },
   "devDependencies": {
     "@types/ejs": "^3.1.2",

server/package.json

@@ -0,0 +1,19 @@
+export function getSize(request, reply) {
+  console.log("/getSize");
+  reply.send({
+    size_uncompressed: request.server.responseSize.size_uncompressed,
+    size_compressed: request.server.responseSize.size_compressed,
+  });
+}
+
+export function enableCompression(request, reply) {
+  console.log("/enableCompression");
+  request.server.responseSize.enableCompression(true);
+  reply.send("OK");
+}
+
+export function disableCompression(request, reply) {
+  console.log("/disableCompression");
+  request.server.responseSize.enableCompression(false);
+  reply.send("OK");
+}

server/src/responseSize/responseSizeController.js

@@ -0,0 +1,7 @@
+import { disableCompression, enableCompression, getSize } from "./responseSizeController.js";
+
+export async function getSizeRouter(fastify) {
+  fastify.get("/enableCompression", enableCompression);
+  fastify.get("/disableCompression", disableCompression);
+  fastify.get("/getSize", getSize);
+}