feat: add ability to automatically decompress gzip responses returned from fetch (#497)

JakeChampion · web-flow · commit e08d06053516 · 2023-08-10T10:14:33.000Z
diff --git a/documentation/docs/globals/Request/Request.mdx b/documentation/docs/globals/Request/Request.mdx
@@ -40,4 +40,7 @@ new Request(input, options)
       - : Any body that you want to add to your request: this can be an `ArrayBuffer`, a `TypedArray`, a `DataView`, a `URLSearchParams`, string object or literal, or a `ReadableStream` object.
     - `backend` _**Fastly-specific**_
     - `cacheOverride` _**Fastly-specific**_
-    - `cacheKey` _**Fastly-specific**_
+    - `cacheKey` _**Fastly-specific**_
+    - `fastly` _**Fastly-specific**_
+      - `decompressGzip`_: boolean_ _**optional**_
+        - Whether to automatically gzip decompress the Response or not.
diff --git a/documentation/docs/globals/fetch.mdx b/documentation/docs/globals/fetch.mdx
@@ -56,6 +56,9 @@ fetch(resource, options)
       - *Fastly-specific*
     - `cacheOverride` _**Fastly-specific**_
     - `cacheKey` _**Fastly-specific**_
+    - `fastly` _**Fastly-specific**_
+      - `decompressGzip`_: boolean_ _**optional**_
+        - Whether to automatically gzip decompress the Response or not.
 
 ### Return value
 
diff --git a/integration-tests/js-compute/fixtures/request-auto-decompress/bin/index.js b/integration-tests/js-compute/fixtures/request-auto-decompress/bin/index.js
@@ -0,0 +1,105 @@
+/* eslint-env serviceworker */
+import { env } from 'fastly:env';
+import { pass, fail, assert, assertRejects } from "../../../assertions.js";
+
+addEventListener("fetch", event => {
+    event.respondWith(app(event))
+})
+/**
+ * @param {FetchEvent} event
+ * @returns {Response}
+ */
+function app(event) {
+    try {
+        const path = (new URL(event.request.url)).pathname;
+        console.log(`path: ${path}`)
+        console.log(`FASTLY_SERVICE_VERSION: ${env('FASTLY_SERVICE_VERSION')}`)
+        if (routes.has(path)) {
+            const routeHandler = routes.get(path);
+            return routeHandler()
+        }
+        return fail(`${path} endpoint does not exist`)
+    } catch (error) {
+        return fail(`The routeHandler threw an error: ${error.message}` + '\n' + error.stack)
+    }
+}
+
+const routes = new Map();
+routes.set('/', () => {
+    routes.delete('/');
+    let test_routes = Array.from(routes.keys())
+    return new Response(JSON.stringify(test_routes), { 'headers': { 'content-type': 'application/json' } });
+});
+
+// Request.fastly.decompressGzip option -- automatic gzip decompression of responses
+{
+    routes.set("/request/constructor/fastly/decompressGzip/true", async () => {
+        const request = new Request('https://httpbin.org/gzip', {
+            headers: {
+                accept: 'application/json'
+            },
+            backend: "httpbin",
+            fastly: {
+                decompressGzip: true
+            }
+        });
+        const response = await fetch(request);
+        // This should work because the response will be decompressed and valid json.
+        const body = await response.json();
+        let error = assert(body.gzipped, true, `body.gzipped`)
+        if (error) { return error }
+        return pass()
+    });
+    routes.set("/request/constructor/fastly/decompressGzip/false", async () => {
+        const request = new Request('https://httpbin.org/gzip', {
+            headers: {
+                accept: 'application/json'
+            },
+            backend: "httpbin",
+            fastly: {
+                decompressGzip: false
+            }
+        });
+        const response = await fetch(request);
+        let error = await assertRejects(async function() {
+            // This should throw because the response will be gzipped compressed, which we can not parse as json.
+            await response.json();
+        });
+        if (error) { return error }
+        return pass()
+    });
+
+    routes.set("/fetch/requestinit/fastly/decompressGzip/true", async () => {
+        const response = await fetch('https://httpbin.org/gzip', {
+            headers: {
+                accept: 'application/json'
+            },
+            backend: "httpbin",
+            fastly: {
+                decompressGzip: true
+            }
+        });
+        // This should work because the response will be decompressed and valid json.
+        const body = await response.json();
+        let error = assert(body.gzipped, true, `body.gzipped`)
+        if (error) { return error }
+        return pass()
+    });
+    routes.set("/fetch/requestinit/fastly/decompressGzip/false", async () => {
+        const response = await fetch('https://httpbin.org/gzip', {
+            headers: {
+                accept: 'application/json'
+            },
+            backend: "httpbin",
+            fastly: {
+                decompressGzip: false
+            }
+        });
+        let error = await assertRejects(async function() {
+            // This should throw because the response will be gzipped compressed, which we can not parse as json.
+            await response.json();
+        });
+        if (error) { return error }
+        return pass()
+    });
+}
diff --git a/integration-tests/js-compute/fixtures/request-auto-decompress/fastly.toml.in b/integration-tests/js-compute/fixtures/request-auto-decompress/fastly.toml.in
@@ -0,0 +1,23 @@
+# This file describes a Fastly Compute@Edge package. To learn more visit:
+# https://developer.fastly.com/reference/fastly-toml/
+
+authors = ["me@jakechampion.name"]
+description = ""
+language = "other"
+manifest_version = 2
+name = "request-auto-decompress"
+service_id = ""
+
+[scripts]
+  build = "node ../../../../js-compute-runtime-cli.js"
+
+[local_server]
+  [local_server.backends]
+    [local_server.backends.httpbin]
+      url = "https://httpbin.org/"
+
+[setup]
+  [setup.backends]
+    [setup.backends.httpbin]
+      address = "httpbin.org"
+      port = 443
diff --git a/integration-tests/js-compute/fixtures/request-auto-decompress/tests.json b/integration-tests/js-compute/fixtures/request-auto-decompress/tests.json
@@ -0,0 +1,42 @@
+{
+  "GET /request/constructor/fastly/decompressGzip/true": {
+    "environments": ["viceroy", "c@e"],
+    "downstream_request": {
+      "method": "GET",
+      "pathname": "/request/constructor/fastly/decompressGzip/true"
+    },
+    "downstream_response": {
+      "status": 200
+    }
+  },
+  "GET /request/constructor/fastly/decompressGzip/false": {
+    "environments": ["viceroy", "c@e"],
+    "downstream_request": {
+      "method": "GET",
+      "pathname": "/request/constructor/fastly/decompressGzip/false"
+    },
+    "downstream_response": {
+      "status": 200
+    }
+  },
+  "GET /fetch/requestinit/fastly/decompressGzip/true": {
+    "environments": ["viceroy", "c@e"],
+    "downstream_request": {
+      "method": "GET",
+      "pathname": "/fetch/requestinit/fastly/decompressGzip/true"
+    },
+    "downstream_response": {
+      "status": 200
+    }
+  },
+  "GET /fetch/requestinit/fastly/decompressGzip/false": {
+    "environments": ["viceroy", "c@e"],
+    "downstream_request": {
+      "method": "GET",
+      "pathname": "/fetch/requestinit/fastly/decompressGzip/false"
+    },
+    "downstream_response": {
+      "status": 200
+    }
+  }
+}
diff --git a/runtime/js-compute-runtime/builtins/request-response.cpp b/runtime/js-compute-runtime/builtins/request-response.cpp
@@ -1143,6 +1143,25 @@ bool Request::set_cache_override(JSContext *cx, JS::HandleObject self,
   return true;
 }
 
+bool Request::apply_auto_decompress_gzip(JSContext *cx, JS::HandleObject self) {
+  MOZ_ASSERT(cx);
+  MOZ_ASSERT(is_instance(self));
+  auto decompress =
+      JS::GetReservedSlot(self, static_cast<uint32_t>(Request::Slots::AutoDecompressGzip))
+          .toBoolean();
+  if (!decompress) {
+    return true;
+  }
+
+  auto res = Request::request_handle(self).auto_decompress_gzip();
+  if (auto *err = res.to_err()) {
+    HANDLE_ERROR(cx, *err);
+    return false;
+  }
+
+  return true;
+}
+
 /**
  * Apply the CacheOverride to a host-side request handle.
  */
@@ -1632,13 +1651,15 @@ JSObject *Request::create(JSContext *cx, JS::HandleObject requestInstance, JS::H
   JS::RootedValue body_val(cx);
   JS::RootedValue backend_val(cx);
   JS::RootedValue cache_override(cx);
+  JS::RootedValue fastly_val(cx);
   if (init_val.isObject()) {
     JS::RootedObject init(cx, init_val.toObjectOrNull());
     if (!JS_GetProperty(cx, init, "method", &method_val) ||
         !JS_GetProperty(cx, init, "headers", &headers_val) ||
         !JS_GetProperty(cx, init, "body", &body_val) ||
         !JS_GetProperty(cx, init, "backend", &backend_val) ||
-        !JS_GetProperty(cx, init, "cacheOverride", &cache_override)) {
+        !JS_GetProperty(cx, init, "cacheOverride", &cache_override) ||
+        !JS_GetProperty(cx, init, "fastly", &fastly_val)) {
       return nullptr;
     }
   } else if (!init_val.isNullOrUndefined()) {
@@ -1931,6 +1952,24 @@ JSObject *Request::create(JSContext *cx, JS::HandleObject requestInstance, JS::H
         JS::GetReservedSlot(input_request, static_cast<uint32_t>(Slots::CacheOverride)));
   }
 
+  if (fastly_val.isObject()) {
+    JS::RootedValue decompress_response_val(cx);
+    JS::RootedObject fastly(cx, fastly_val.toObjectOrNull());
+    if (!JS_GetProperty(cx, fastly, "decompressGzip", &decompress_response_val)) {
+      return nullptr;
+    }
+    auto value = JS::ToBoolean(decompress_response_val);
+    JS::SetReservedSlot(request, static_cast<uint32_t>(Slots::AutoDecompressGzip),
+                        JS::BooleanValue(value));
+  } else if (input_request) {
+    JS::SetReservedSlot(
+        request, static_cast<uint32_t>(Slots::AutoDecompressGzip),
+        JS::GetReservedSlot(input_request, static_cast<uint32_t>(Slots::AutoDecompressGzip)));
+  } else {
+    JS::SetReservedSlot(request, static_cast<uint32_t>(Slots::AutoDecompressGzip),
+                        JS::BooleanValue(false));
+  }
+
   return request;
 }
 
diff --git a/runtime/js-compute-runtime/builtins/request-response.h b/runtime/js-compute-runtime/builtins/request-response.h
@@ -130,6 +130,7 @@ class Request final : public BuiltinImpl<Request> {
     PendingRequest,
     ResponsePromise,
     IsDownstream,
+    AutoDecompressGzip,
     Count,
   };
 
@@ -139,6 +140,7 @@ class Request final : public BuiltinImpl<Request> {
   static bool set_cache_override(JSContext *cx, JS::HandleObject self,
                                  JS::HandleValue cache_override_val);
   static bool apply_cache_override(JSContext *cx, JS::HandleObject self);
+  static bool apply_auto_decompress_gzip(JSContext *cx, JS::HandleObject self);
 
   static host_api::HttpReq request_handle(JSObject *obj);
   static host_api::HttpPendingReq pending_handle(JSObject *obj);
diff --git a/runtime/js-compute-runtime/host_interface/component/fastly_world_adapter.cpp b/runtime/js-compute-runtime/host_interface/component/fastly_world_adapter.cpp
@@ -191,6 +191,13 @@ bool fastly_compute_at_edge_http_req_cache_override_set(
       err);
 }
 
+bool fastly_compute_at_edge_http_req_auto_decompress_response_set(
+    fastly_compute_at_edge_http_types_request_handle_t h,
+    fastly_compute_at_edge_http_types_content_encodings_t encodings,
+    fastly_compute_at_edge_types_error_t *err) {
+  return convert_result(fastly::req_auto_decompress_response_set(h, encodings), err);
+}
+
 bool fastly_compute_at_edge_http_req_downstream_client_ip_addr(
     fastly_world_list_u8_t *ret, fastly_compute_at_edge_types_error_t *err) {
   ret->ptr = static_cast<uint8_t *>(cabi_malloc(16, 1));
diff --git a/runtime/js-compute-runtime/host_interface/fastly.h b/runtime/js-compute-runtime/host_interface/fastly.h
@@ -145,6 +145,10 @@ int req_cache_override_v2_set(fastly_compute_at_edge_http_types_request_handle_t
                               int tag, uint32_t ttl, uint32_t stale_while_revalidate,
                               const char *surrogate_key, size_t surrogate_key_len);
 
+WASM_IMPORT("fastly_http_req", "auto_decompress_response_set")
+int req_auto_decompress_response_set(fastly_compute_at_edge_http_types_request_handle_t req_handle,
+                                     int tag);
+
 /**
  * `octets` must be a 16-byte array.
  * If, after a successful call, `nwritten` == 4, the value in `octets` is an IPv4 address.
diff --git a/runtime/js-compute-runtime/host_interface/host_api.cpp b/runtime/js-compute-runtime/host_interface/host_api.cpp
@@ -381,6 +381,22 @@ Result<Void> HttpReq::redirect_to_grip_proxy(std::string_view backend) {
   return res;
 }
 
+Result<Void> HttpReq::auto_decompress_gzip() {
+  Result<Void> res;
+
+  fastly_compute_at_edge_types_error_t err;
+  fastly_compute_at_edge_http_types_content_encodings_t encodings_to_decompress = 0;
+  encodings_to_decompress |= FASTLY_COMPUTE_AT_EDGE_HTTP_TYPES_CONTENT_ENCODINGS_GZIP;
+  if (!fastly_compute_at_edge_http_req_auto_decompress_response_set(
+          this->handle, encodings_to_decompress, &err)) {
+    res.emplace_err(err);
+  } else {
+    res.emplace();
+  }
+
+  return res;
+}
+
 Result<Void> HttpReq::register_dynamic_backend(std::string_view name, std::string_view target,
                                                const BackendConfig &config) {
   Result<Void> res;
diff --git a/runtime/js-compute-runtime/host_interface/host_api.h b/runtime/js-compute-runtime/host_interface/host_api.h
@@ -366,6 +366,8 @@ class HttpReq final : public HttpBase {
 
   static Result<HostBytes> http_req_downstream_tls_ja3_md5();
 
+  Result<Void> auto_decompress_gzip();
+
   /// Send this request synchronously, and wait for the response.
   Result<Response> send(HttpBody body, std::string_view backend);
 
diff --git a/runtime/js-compute-runtime/js-compute-builtins.cpp b/runtime/js-compute-runtime/js-compute-builtins.cpp
@@ -680,6 +680,10 @@ bool fetch(JSContext *cx, unsigned argc, Value *vp) {
     return false;
   }
 
+  if (!builtins::Request::apply_auto_decompress_gzip(cx, request)) {
+    return false;
+  }
+
   bool streaming = false;
   if (!builtins::RequestOrResponse::maybe_stream_body(cx, request, &streaming)) {
     return false;
diff --git a/types/globals.d.ts b/types/globals.d.ts
@@ -1033,6 +1033,9 @@ declare interface RequestInit {
   backend?: string;
   cacheOverride?: import('fastly:cache-override').CacheOverride;
   cacheKey?: string;
+  fastly?: {
+    decompressGzip?: boolean
+  }
 }
 
 /**
