fix: certain redirect responses change request methods (#3296)

Aligns the `@crawlee/impit-client` implementation with the respective
RFC and browsers' behaviour.

Closes #2586

Author: barjin

packages/impit-client/package.json

@@ -60,7 +60,7 @@
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
-        "impit": "^0.7.0",
+        "impit": "^0.7.3",
         "tough-cookie": "^6.0.0"
     }
 }

packages/impit-client/src/index.ts

@@ -98,6 +98,17 @@ export class ImpitHttpClient implements BaseHttpClient {
         return body as any;
     }
 
+    private shouldRewriteRedirectToGet(httpStatus: number, method: HttpRequest<any>['method']): boolean {
+        // See https://github.com/mozilla-firefox/firefox/blob/911b3eec6c5e58a9a49e23aa105e49aa76e00f9c/netwerk/protocol/http/HttpBaseChannel.cpp#L4801
+        if ([301, 302].includes(httpStatus)) {
+            return method === 'POST';
+        }
+
+        if (httpStatus === 303) return method !== 'HEAD';
+
+        return false;
+    }
+
     /**
      * Common implementation for `sendRequest` and `stream` methods.
      * @param request `HttpRequest` object
@@ -141,6 +152,7 @@ export class ImpitHttpClient implements BaseHttpClient {
             return this.getResponse(
                 {
                     ...request,
+                    method: this.shouldRewriteRedirectToGet(response.status, request.method) ? 'GET' : request.method,
                     url: redirectUrl.href,
                 },
                 {

yarn.lock

@@ -618,7 +618,7 @@ __metadata:
   dependencies:
     "@apify/datastructures": "npm:^2.0.3"
     "@crawlee/core": "npm:^3.15.3"
-    impit: "npm:^0.7.0"
+    impit: "npm:^0.7.3"
     tough-cookie: "npm:^6.0.0"
   peerDependencies:
     "@crawlee/core": ^3.12.1
@@ -7487,9 +7487,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-darwin-arm64@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-darwin-arm64@npm:0.7.0"
+"impit-darwin-arm64@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-darwin-arm64@npm:0.7.3"
   conditions: os=darwin & cpu=arm64
   languageName: node
   linkType: hard
@@ -7501,9 +7501,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-darwin-x64@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-darwin-x64@npm:0.7.0"
+"impit-darwin-x64@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-darwin-x64@npm:0.7.3"
   conditions: os=darwin & cpu=x64
   languageName: node
   linkType: hard
@@ -7515,9 +7515,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-linux-arm64-gnu@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-linux-arm64-gnu@npm:0.7.0"
+"impit-linux-arm64-gnu@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-linux-arm64-gnu@npm:0.7.3"
   conditions: os=linux & cpu=arm64 & libc=glibc
   languageName: node
   linkType: hard
@@ -7529,9 +7529,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-linux-arm64-musl@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-linux-arm64-musl@npm:0.7.0"
+"impit-linux-arm64-musl@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-linux-arm64-musl@npm:0.7.3"
   conditions: os=linux & cpu=arm64 & libc=musl
   languageName: node
   linkType: hard
@@ -7543,9 +7543,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-linux-x64-gnu@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-linux-x64-gnu@npm:0.7.0"
+"impit-linux-x64-gnu@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-linux-x64-gnu@npm:0.7.3"
   conditions: os=linux & cpu=x64 & libc=glibc
   languageName: node
   linkType: hard
@@ -7557,9 +7557,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-linux-x64-musl@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-linux-x64-musl@npm:0.7.0"
+"impit-linux-x64-musl@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-linux-x64-musl@npm:0.7.3"
   conditions: os=linux & cpu=x64 & libc=musl
   languageName: node
   linkType: hard
@@ -7571,9 +7571,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-win32-arm64-msvc@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-win32-arm64-msvc@npm:0.7.0"
+"impit-win32-arm64-msvc@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-win32-arm64-msvc@npm:0.7.3"
   conditions: os=win32 & cpu=arm64
   languageName: node
   linkType: hard
@@ -7585,9 +7585,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit-win32-x64-msvc@npm:0.7.0":
-  version: 0.7.0
-  resolution: "impit-win32-x64-msvc@npm:0.7.0"
+"impit-win32-x64-msvc@npm:0.7.3":
+  version: 0.7.3
+  resolution: "impit-win32-x64-msvc@npm:0.7.3"
   conditions: os=win32 & cpu=x64
   languageName: node
   linkType: hard
@@ -7625,18 +7625,18 @@ __metadata:
   languageName: node
   linkType: hard
 
-"impit@npm:^0.7.0":
-  version: 0.7.0
-  resolution: "impit@npm:0.7.0"
-  dependencies:
-    impit-darwin-arm64: "npm:0.7.0"
-    impit-darwin-x64: "npm:0.7.0"
-    impit-linux-arm64-gnu: "npm:0.7.0"
-    impit-linux-arm64-musl: "npm:0.7.0"
-    impit-linux-x64-gnu: "npm:0.7.0"
-    impit-linux-x64-musl: "npm:0.7.0"
-    impit-win32-arm64-msvc: "npm:0.7.0"
-    impit-win32-x64-msvc: "npm:0.7.0"
+"impit@npm:^0.7.3":
+  version: 0.7.3
+  resolution: "impit@npm:0.7.3"
+  dependencies:
+    impit-darwin-arm64: "npm:0.7.3"
+    impit-darwin-x64: "npm:0.7.3"
+    impit-linux-arm64-gnu: "npm:0.7.3"
+    impit-linux-arm64-musl: "npm:0.7.3"
+    impit-linux-x64-gnu: "npm:0.7.3"
+    impit-linux-x64-musl: "npm:0.7.3"
+    impit-win32-arm64-msvc: "npm:0.7.3"
+    impit-win32-x64-msvc: "npm:0.7.3"
   dependenciesMeta:
     impit-darwin-arm64:
       optional: true
@@ -7654,7 +7654,7 @@ __metadata:
       optional: true
     impit-win32-x64-msvc:
       optional: true
-  checksum: 10c0/bd5887db006b68913de1480da47379f4de11cbc7ba9725fd0106faca993eaa16a7a13e9869cc641790880dd044a07f7ce6312848d05555a98b266487e0740f57
+  checksum: 10c0/8a7138303b5ea01d59e35c678e8c4a1483db1af722e1f747bf7c3635f86edcc51d49d8949820fd74993a041903d9fd29a57547cfe904a0bb5e9295e9c7269bf3
   languageName: node
   linkType: hard