fix: handle gzip encoding in API transformations

README.md

@@ -127,24 +127,41 @@ cd obn-gateway
 
 ### 3. Configure Environment Variables
 
-Same as step 3 for Docker installation
+Same as [step 3 for Docker installation](#3-configure-environment-variables)
 
 ### 4. Email service  
 
-Same as step 4 for Docker installation
+Same as [step 5 for Docker installation](#5-email-service)
 
 ### 5. Local Setup
 
 If you prefer to set up the project locally, install and run the project dependencies locally using pnpm:  
 
 ```bash  
 pnpm install  
+```
+
+### 6. Run Migrations and Setup
+
+Before starting the server, you need to run database migrations and the initial setup script. These steps are handled automatically in Docker but must be run manually for local installation:
+
+```bash
+pnpm --filter server migration:run
+pnpm --filter server setup
+```
+
+- `migration:run` applies database schema migrations
+- `setup` initializes the application with default data and configurations
+
+### 7. Start the Application
+
+```bash
 pnpm dev  
 ```  
 
-### 6. Accessing the Services  
+### 8. Accessing the Services  
 
-Same as step 7 for Docker installation
+Same as [step 7 for Docker installation](#7-accessing-the-services)
 
 ## Troubleshooting  
 

apps/server/package.json

@@ -13,7 +13,7 @@
     "dev": "NODE_EXTRA_CA_CERTS=./certs/ca/ca.crt nest start --watch",
     "start:debug": "NODE_EXTRA_CA_CERTS=./certs/ca/ca.crt nest start --debug --watch",
     "start:prod": "NODE_EXTRA_CA_CERTS=./certs/ca/ca.crt node dist/src/main",
-    "start:setup": "NODE_EXTRA_CA_CERTS=./certs/ca/ca.crt node dist/src/setup",
+    "start:setup": "TS_NODE_TRANSPILE_ONLY=true NODE_EXTRA_CA_CERTS=./certs/ca/ca.crt node -r ts-node/register -r tsconfig-paths/register ./src/setup.ts",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "jest --passWithNoTests --coverage=false",
     "test:ci": "jest --ci --coverage --maxWorkers=4 --config=jest.ci.config.js",

apps/server/src/apis/apis.service.ts

@@ -2055,22 +2055,81 @@ export class APIService {
       });
     }
 
+    // Kong plugin phases for request/response transformation with gzip handling:
+    //
+    // The transformation flow uses Kong's shared context (kong.ctx.shared) to pass
+    // transformed bodies between phases. User-defined transformations store their
+    // results in `upstream_request` (for requests) and `downstream_response` (for responses).
+    //
+    // GZIP HANDLING:
+    // Kong does NOT automatically decompress responses - the user's downstream transformation
+    // must manually decompress gzipped data using gzip.inflate_gzip() before manipulating it
+    // (see setup.ts for example). After transformation, we re-compress for clients expecting gzip.
+    //
+    // The workaround here ensures:
+    // 1. We normalize Accept-Encoding so upstream only sends gzip (which we handle) or uncompressed
+    // 2. We clear Content-Length since transformed body size differs from original
+    // 3. We re-compress the transformed response if client expects gzip
+    //
+    // Phase execution order:
+    // 1. access: Transform request → normalize Accept-Encoding → apply transformed body
+    // 2. header_filter: Transform response (includes manual gzip decompression) → clear stale headers
+    // 3. body_filter: Re-encode and re-compress the transformed response if needed
     const plugin = await this.kongRouteService.updateOrCreatePlugin(
       environment,
       route.routeId!,
       {
         config: {
           access: [
+            // User's upstream transformation runs first, stores result in kong.ctx.shared.upstream_request
             data.upstream,
+            // Normalize Accept-Encoding: We only support gzip decompression, so we either
+            // pass gzip, clear the header (for identity/*), or reject unsupported encodings
+            `
+            encoding = kong.request.get_header("Accept-Encoding")
+            if not encoding then
+              return
+            end
+            if encoding:lower():find("gzip") then
+              return kong.service.request.set_header("Accept-Encoding", "gzip")
+            end
+            if encoding:lower() == "identity" or encoding:lower() == "*" then
+              return kong.service.request.clear_header("Accept-Encoding")
+            end
+            return kong.response.error(400, "Unsupported encoding. Only gzip encoding is supported.")
+            `,
+            // Apply the transformed request body if the upstream transformation produced one
             `if kong.ctx.shared.upstream_request ~= nil then kong.service.request.set_body(kong.ctx.shared.upstream_request) end`,
           ],
-          // this is required to ensure that we dont send an invalid content length to the downstream/client
           header_filter: [
+            // User's downstream transformation runs here - it must manually decompress gzipped
+            // responses using gzip.inflate_gzip() before manipulation, then store the result
+            // in kong.ctx.shared.downstream_response
             data.downstream,
-            'kong.response.clear_header("Content-Length")',
+            // Clear Content-Length since transformed body size differs from original.
+            // Also clear Transfer-Encoding/Content-Encoding for non-gzip responses to
+            // prevent clients from attempting to decompress plain JSON.
+            `
+            kong.response.clear_header("Content-Length")
+            if kong.response.get_header("Content-Encoding") ~= "gzip" then
+              kong.response.clear_header("Transfer-Encoding")
+              kong.response.clear_header("Content-Encoding")
+            end
+            `,
           ],
           body_filter: [
-            `local cjson = require 'cjson.safe'\nif kong.ctx.shared.downstream_response ~= nil then kong.response.set_raw_body(cjson.encode(kong.ctx.shared.downstream_response)) end`,
+            // Re-encode the transformed response as JSON, with gzip compression if the
+            // client expects it (based on Content-Encoding header we preserved above)
+            `
+            local cjson = require 'cjson.safe'
+            local gzip = require 'kong.tools.gzip'
+            if kong.ctx.shared.downstream_response ~= nil then
+              if kong.response.get_header("Content-Encoding") == "gzip" then
+                kong.response.set_raw_body(gzip.deflate_gzip(cjson.encode(kong.ctx.shared.downstream_response)))
+              else
+                kong.response.set_raw_body(cjson.encode(kong.ctx.shared.downstream_response))
+              end
+            end`,
           ],
         },
         name: KONG_PLUGINS.POST_FUNCTION,

apps/server/src/setup.ts

@@ -229,6 +229,7 @@ async function performSetupTasks(): Promise<void> {
                 throw e;
               });
             if (api) continue;
+            if (!request.urlObject.path) continue;
 
             const regexPath =
               '~' +
@@ -293,7 +294,7 @@ async function performSetupTasks(): Promise<void> {
             );
             const transformationData = {
               upstream: `
-local function transform_upstream_request()
+local function transform_request_to_upstream()
   	local ok, err = pcall(function()
 		-- Read the request body
 		kong.service.request.enable_buffering()  -- Enable buffering to read body
@@ -318,26 +319,36 @@ local function transform_upstream_request()
 		return kong.response.error(500, "An unexpected error occurred")
 	end
 end
-return transform_upstream_request`,
+return transform_request_to_upstream`,
               downstream: `
 local cjson = require 'cjson.safe'
+local gzip = require 'kong.tools.gzip'
 
-local function transform_downstream_response()
+local function transform_response_to_downstream()
   	local ok, err = pcall(function()
 		${
       (response?.[0]?.body &&
         `
 		if kong.service.response.get_status() == nil then
 			return
 		end
-		local data = kong.service.response.get_body()
-		data = cjson.decode(data)
+		local data = kong.service.response.get_raw_body()
+    if kong.service.response.get_header("Content-Encoding") == "gzip" then
+        data = cjson.decode(gzip.inflate_gzip(data))
+    else
+        data = cjson.decode(data)
+    end
+    if data == nil then
+      return kong.response.error(500, "failed parsing json body")
+    end
 		if kong.service.response.get_status() >= 400 then
+      kong.ctx.shared.downstream_response = {
+        ["status"] = tostring(data.status),
+        ["message"] = tostring(data.message),
+      }
 			return
 		end
-		if data then
-			kong.ctx.shared.downstream_response = ${jsonToLua(response?.[0]?.body)}
-		end`) ||
+		kong.ctx.shared.downstream_response = ${jsonToLua(response?.[0]?.body)}`) ||
       ''
     }
 	end)
@@ -346,7 +357,7 @@ local function transform_downstream_response()
 		return kong.response.error(500, "An unexpected error occurred")
 	end
 end
-return transform_downstream_response`,
+return transform_response_to_downstream`,
             };
             await apiService.setTransformation(
               ctx,

docker-compose-kong.yml

@@ -10,7 +10,7 @@ x-kong-config: &kong-env
   KONG_PROXY_ACCESS_LOG: /dev/stdout
   KONG_PROXY_ERROR_LOG: /dev/stderr
   KONG_PREFIX: /var/run/kong
-  KONG_UNTRUSTED_LUA_SANDBOX_REQUIRES: cjson.safe
+  KONG_UNTRUSTED_LUA_SANDBOX_REQUIRES: cjson.safe, kong.tools.gzip
   KONG_ADMIN_LISTEN: "0.0.0.0:8001 reuseport backlog=16384, 0.0.0.0:8444 http2 ssl reuseport backlog=16384"
   KONG_PLUGINS: "bundled,obn-authorization,obn-request-validator"
   KONG_SSL_CERT: ${KONG_SSL_CERT}
@@ -27,7 +27,9 @@ services:
       POSTGRES_USER: ${KONG_PG_USER:-kong}
       POSTGRES_PASSWORD_FILE: /run/secrets/kong_postgres_password
     secrets:
-      - kong_postgres_password
+      - source: kong_postgres_password
+        target: kong_postgres_password
+        mode: 0444
     healthcheck:
       test: [ "CMD", "pg_isready", "-U", "${KONG_PG_USER:-kong}" ]
       interval: 1s
@@ -40,7 +42,7 @@ services:
       - kong-net
 
   kong-migrations:
-    image: "${KONG_DOCKER_TAG:-kong:3.5.0}"
+    image: "${KONG_DOCKER_TAG:-kong:3.9}"
     command:
       [
         "/bin/sh",
@@ -50,7 +52,9 @@ services:
     environment:
       <<: *kong-env
     secrets:
-      - kong_postgres_password
+      - source: kong_postgres_password
+        target: kong_postgres_password
+        mode: 0444
     networks:
       - kong-net
     restart: on-failure
@@ -60,7 +64,7 @@ services:
       - ./config/certs/kong:/opt/kong
 
   kong-migrations-up:
-    image: "${KONG_DOCKER_TAG:-kong:3.5.0}"
+    image: "${KONG_DOCKER_TAG:-kong:3.9}"
     command:
       [
         "/bin/sh",
@@ -70,7 +74,9 @@ services:
     environment:
       <<: *kong-env
     secrets:
-      - kong_postgres_password
+      - source: kong_postgres_password
+        target: kong_postgres_password
+        mode: 0444
     networks:
       - kong-net
     restart: on-failure
@@ -80,12 +86,14 @@ services:
       - ./config/certs/kong:/opt/kong
 
   kong:
-    image: "${KONG_DOCKER_TAG:-kong:3.5.0}"
+    image: "${KONG_DOCKER_TAG:-kong:3.9}"
     user: "${KONG_USER:-kong}"
     environment:
       <<: *kong-env
     secrets:
-      - kong_postgres_password
+      - source: kong_postgres_password
+        target: kong_postgres_password
+        mode: 0444
     command:
       [
         "/bin/sh",

docker-compose.yml

@@ -137,6 +137,8 @@ services:
     depends_on:
       kong-dev-db:
         condition: service_healthy
+      kong-dev-migrations:
+        condition: service_completed_successfully
     profiles:
       - dev
       - kong
@@ -190,6 +192,8 @@ services:
     depends_on:
       kong-prod-db:
         condition: service_healthy
+      kong-prod-migrations:
+        condition: service_completed_successfully
     profiles:
       - prod
       - kong