diff --git a/doc/admin-guide/plugins/escalate.en.rst b/doc/admin-guide/plugins/escalate.en.rst index 945083ceefa..6a967b362b0 100644 --- a/doc/admin-guide/plugins/escalate.en.rst +++ b/doc/admin-guide/plugins/escalate.en.rst @@ -59,6 +59,14 @@ when the origin server in the remap rule returns a 401, option overrides the default behavior and enables escalation for non-GET requests in addition to GET. + .. note:: + + For POST body buffering to work with escalation, + :ts:cv:`proxy.config.http.post_copy_size` must be set large enough to buffer + the expected POST body sizes (e.g., 2048 bytes or larger). This enables + Traffic Server to buffer POST bodies before sending to the origin, so they + can be replayed if escalation is needed. + Installation ------------ diff --git a/plugins/escalate/escalate.cc b/plugins/escalate/escalate.cc index 67b6d775185..8b80ddd7ded 100644 --- a/plugins/escalate/escalate.cc +++ b/plugins/escalate/escalate.cc @@ -209,6 +209,7 @@ EscalateResponse(TSCont cont, TSEvent event, void *edata) // Now update the Redirect URL, if set if (url_str) { + Dbg(dbg_ctl, "Calling TSHttpTxnRedirectUrlSet for status %d with URL: %.*s", status, url_len, url_str); TSHttpTxnRedirectUrlSet(txn, url_str, url_len); // Transfers ownership // Add our x-escalate-redirect header marker if it doesn't already exist and the option is enabled. @@ -320,6 +321,11 @@ TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri */) { EscalationState *es = static_cast(instance); + // Note: For POST body buffering to work with escalation, + // proxy.config.http.post_copy_size should be set large enough to buffer the + // expected POST body sizes. This enables ATS to buffer POST bodies before + // sending to the origin so they can be replayed if escalation is needed. + TSHttpTxnHookAdd(txn, TS_HTTP_READ_RESPONSE_HDR_HOOK, es->cont); TSHttpTxnHookAdd(txn, TS_HTTP_SEND_RESPONSE_HDR_HOOK, es->cont); return TSREMAP_NO_REMAP; diff --git a/src/proxy/http/HttpTunnel.cc b/src/proxy/http/HttpTunnel.cc index feb863cf13a..6788ecb5abb 100644 --- a/src/proxy/http/HttpTunnel.cc +++ b/src/proxy/http/HttpTunnel.cc @@ -1028,7 +1028,18 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) return; } } else { - body_bytes_copied += sm->postbuf_copy_partial_data(body_bytes_to_copy); + int64_t bytes_to_copy_now = body_bytes_to_copy; + + // For POST redirect buffering with HTTP_CLIENT producer, we need special handling. + // When POST data is pre-buffered (common for small POSTs), producer_n (ntodo) is 0, + // so body_bytes_to_copy calculated earlier would be 0. Instead, copy what's available, + // but limit it to the actual content length to avoid copying pipelined requests. + if (p->vc_type == HttpTunnelType_t::HTTP_CLIENT && sm->enable_redirection && body_bytes_to_copy == 0) { + int64_t bytes_available = p->buffer_start->read_avail(); + bytes_to_copy_now = (p->total_bytes >= 0) ? std::min(bytes_available, p->total_bytes) : bytes_available; + } + + body_bytes_copied += sm->postbuf_copy_partial_data(bytes_to_copy_now); body_bytes_to_copy = 0; } } // end of added logic for partial POST diff --git a/tests/gold_tests/pluginTest/escalate/escalate.test.py b/tests/gold_tests/pluginTest/escalate/escalate.test.py index a13f977c4bf..39e6b4cdc33 100644 --- a/tests/gold_tests/pluginTest/escalate/escalate.test.py +++ b/tests/gold_tests/pluginTest/escalate/escalate.test.py @@ -205,6 +205,10 @@ def _setup_servers(self, tr: 'TestRun') -> None: 'uuid: GET_failed', "Verify the origin server received the failed GET request.") self._server_origin.Streams.All += Testers.ContainsExpression( 'uuid: POST_success', "Verify the origin server received the successful POST request.") + self._server_origin.Streams.All += Testers.ContainsExpression( + 'uuid: POST_fail_escalated', "Verify the origin server received the POST request that will be escalated.") + self._server_origin.Streams.All += Testers.ContainsExpression( + 'uuid: POST_small_fail', "Verify the origin server received the small POST request that will be escalated.") self._server_origin.Streams.All += Testers.ContainsExpression( 'uuid: HEAD_fail_escalated', "Verify the origin server received the HEAD request that will be escalated.") @@ -217,6 +221,11 @@ def _setup_servers(self, tr: 'TestRun') -> None: 'uuid: GET_failed', "Verify the failover server received the failed GET request.") self._server_failover.Streams.All += Testers.ContainsExpression( 'uuid: GET_down_origin', "Verify the failover server received the down origin GET request.") + # With --escalate-non-get-methods, the POST request should now be escalated + self._server_failover.Streams.All += Testers.ContainsExpression( + 'uuid: POST_fail_escalated', "Verify the failover server received the POST that is now escalated.") + self._server_failover.Streams.All += Testers.ContainsExpression( + 'uuid: POST_small_fail', "Verify the failover server received the small POST that is now escalated.") # With --escalate-non-get-methods, the HEAD request should now be escalated self._server_failover.Streams.All += Testers.ContainsExpression( 'uuid: HEAD_fail_escalated', "Verify the failover server received the HEAD that is now escalated.") @@ -231,11 +240,12 @@ def _setup_ts(self, tr: 'TestRun') -> None: self._ts.Disk.records_config.update( { 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|escalate', + 'proxy.config.diags.debug.tags': 'http|escalate|http_redirect', 'proxy.config.dns.nameservers': f'127.0.0.1:{self._dns.Variables.Port}', 'proxy.config.dns.resolv_conf': 'NULL', 'proxy.config.http.redirect.actions': 'self:follow', 'proxy.config.http.number_of_redirections': 4, + 'proxy.config.http.post_copy_size': 400000, }) # Set up a dead port for the down origin scenario @@ -266,6 +276,10 @@ def _setup_client(self, tr: 'TestRun') -> None: client.Streams.All += Testers.ContainsExpression('x-response: third', 'Verify third GET response received (escalated).') client.Streams.All += Testers.ContainsExpression('x-response: fourth', 'Verify fourth GET response received (escalated).') client.Streams.All += Testers.ContainsExpression('x-response: post_success', 'Verify successful POST response received.') + client.Streams.All += Testers.ContainsExpression( + 'x-response: post_fail_escalated', 'Verify escalated POST response received.') + client.Streams.All += Testers.ContainsExpression( + 'x-response: post_small_fail', 'Verify escalated small POST response received.') client.Streams.All += Testers.ContainsExpression( 'x-response: head_fail_escalated', 'Verify escalated HEAD response received.') diff --git a/tests/gold_tests/pluginTest/escalate/escalate_failover.replay.yaml b/tests/gold_tests/pluginTest/escalate/escalate_failover.replay.yaml index 04fabbb7a7f..2a910804114 100644 --- a/tests/gold_tests/pluginTest/escalate/escalate_failover.replay.yaml +++ b/tests/gold_tests/pluginTest/escalate/escalate_failover.replay.yaml @@ -125,22 +125,87 @@ sessions: content: size: 320000 + # POST request response for escalated requests (with --escalate-non-get-methods) + - client-request: + method: "POST" + version: "1.1" + url: /api/post/data + headers: + fields: + - [ Host, origin.server.com ] + - [ Content-Type, "application/json" ] + - [ Content-Length, 320000 ] + - [ X-Request, post_fail_escalated ] + - [ uuid, POST_fail_escalated ] + + proxy-request: + method: "POST" + headers: + fields: + - [ X-Request, { value: post_fail_escalated, as: equal } ] + - [ Content-Length, { value: 320000, as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 320000 ] + - [ X-Response, post_fail_escalated ] + + # Small POST request response (32 bytes) for escalated requests + - client-request: + method: "POST" + version: "1.1" + url: /api/post/small + headers: + fields: + - [ Host, origin.server.com ] + - [ Content-Type, "application/json" ] + - [ Content-Length, 32 ] + - [ X-Request, post_small_fail ] + - [ uuid, POST_small_fail ] + content: + encoding: plain + data: '{"status":"active","id":123}' + + proxy-request: + method: "POST" + headers: + fields: + - [ X-Request, { value: post_small_fail, as: equal } ] + - [ Content-Length, { value: 32, as: equal } ] + content: + encoding: plain + data: '{"status":"active","id":123}' + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 32 ] + - [ X-Response, post_small_fail ] + content: + encoding: plain + data: '{"status":"active","id":123}' + # HEAD request response for escalated requests (with --escalate-non-get-methods) - client-request: method: "HEAD" version: "1.1" - url: /api/head/data + url: /path/head_fail headers: fields: - [ Host, origin.server.com ] - - [ X-Request, head_fail_escalated ] + - [ X-Request, head_fail ] - [ uuid, HEAD_fail_escalated ] proxy-request: method: "HEAD" headers: fields: - - [ X-Request, { value: head_fail_escalated, as: equal } ] + - [ X-Request, { value: head_fail, as: equal } ] server-response: status: 200 diff --git a/tests/gold_tests/pluginTest/escalate/escalate_non_get_methods.replay.yaml b/tests/gold_tests/pluginTest/escalate/escalate_non_get_methods.replay.yaml index 638a8c7b04a..35904e07f2b 100644 --- a/tests/gold_tests/pluginTest/escalate/escalate_non_get_methods.replay.yaml +++ b/tests/gold_tests/pluginTest/escalate/escalate_non_get_methods.replay.yaml @@ -19,7 +19,7 @@ meta: sessions: - transactions: - # Original GET transactions from escalate_original.replay.yaml + # Non-escalated GET request. - client-request: method: "GET" version: "1.1" @@ -51,6 +51,7 @@ sessions: fields: - [ X-Response, { value: first, as: equal } ] + # Non-escalated GET request with a chunked response. - client-request: method: "GET" version: "1.1" @@ -84,6 +85,7 @@ sessions: fields: - [ X-Response, { value: second, as: equal } ] + # GET request that will fail and be escalated - client-request: method: "GET" version: "1.1" @@ -107,13 +109,16 @@ sessions: headers: fields: - [ Content-Length, 0 ] + - [ X-Response, third ] + # With escalation, the failover responds with 200 proxy-response: - # The failover server should reply with a 200 OK (GET requests are escalated with --escalate-non-get-methods). status: 200 headers: fields: - [ X-Response, { value: third, as: equal } ] + content: + size: 320000 # This will not make it to the origin server since the Host is set to a # non-responsive server. But the failover server should reply with a 200 OK. @@ -134,6 +139,7 @@ sessions: fields: - [ X-Request, { value: fourth, as: equal } ] + # This server-response is just for replay file validity - origin server won't receive this transaction server-response: status: 200 reason: OK @@ -147,34 +153,32 @@ sessions: fields: - [ X-Response, { value: fourth, as: equal } ] - # POST request with sizable body that gets proxied normally. + # POST request that succeeds without escalation - client-request: method: "POST" version: "1.1" - url: /api/upload/data + url: /api/post/success headers: fields: - [ Host, origin.server.com ] - [ Content-Type, "application/json" ] - - [ Content-Length, 32 ] + - [ Content-Length, 320000 ] - [ X-Request, post_success ] - [ uuid, POST_success ] - content: - encoding: plain - size: 32 proxy-request: method: "POST" headers: fields: - [ X-Request, { value: post_success, as: equal } ] + - [ Content-Length, { value: 320000, as: equal } ] server-response: status: 200 reason: OK headers: fields: - - [ Content-Length, 32 ] + - [ Content-Length, 0 ] - [ X-Response, post_success ] proxy-response: @@ -183,22 +187,96 @@ sessions: fields: - [ X-Response, { value: post_success, as: equal } ] - # A HEAD request that will be escalated with --escalate-non-get-methods + # POST request with body that will be escalated with --escalate-non-get-methods + - client-request: + method: "POST" + version: "1.1" + url: /api/post/data + headers: + fields: + - [ Host, origin.server.com ] + - [ Content-Type, "application/json" ] + - [ Content-Length, 320000 ] + - [ X-Request, post_fail_escalated ] + - [ uuid, POST_fail_escalated ] + + proxy-request: + method: "POST" + headers: + fields: + - [ X-Request, { value: post_fail_escalated, as: equal } ] + - [ Content-Length, { value: 320000, as: equal } ] + + server-response: + status: 502 + reason: Bad Gateway + headers: + fields: + - [ Content-Length, 0 ] + - [ X-Response, post_fail_escalated ] + + # With --escalate-non-get-methods, POST request is escalated to failover. + proxy-response: + status: 200 + headers: + fields: + - [ X-Response, { value: post_fail_escalated, as: equal } ] + + # Small POST request (32 bytes) that will be escalated + - client-request: + method: "POST" + version: "1.1" + url: /api/post/small + headers: + fields: + - [ Host, origin.server.com ] + - [ Content-Type, "application/json" ] + - [ Content-Length, 32 ] + - [ X-Request, post_small_fail ] + - [ uuid, POST_small_fail ] + content: + encoding: plain + data: '{"status":"active","id":123}' + + proxy-request: + method: "POST" + headers: + fields: + - [ X-Request, { value: post_small_fail, as: equal } ] + - [ Content-Length, { value: 32, as: equal } ] + + server-response: + status: 502 + reason: Bad Gateway + headers: + fields: + - [ Content-Length, 0 ] + - [ X-Response, post_small_fail ] + + # With --escalate-non-get-methods, POST request is escalated to failover. + proxy-response: + status: 200 + headers: + fields: + - [ X-Response, { value: post_small_fail, as: equal } ] + + # HEAD request that will be escalated with --escalate-non-get-methods - client-request: method: "HEAD" version: "1.1" - url: /api/head/data + url: /path/head_fail headers: fields: - [ Host, origin.server.com ] - - [ X-Request, head_fail_escalated ] + - [ Content-Length, 0 ] + - [ X-Request, head_fail ] - [ uuid, HEAD_fail_escalated ] proxy-request: method: "HEAD" headers: fields: - - [ X-Request, { value: head_fail_escalated, as: equal } ] + - [ X-Request, { value: head_fail, as: equal } ] server-response: status: 502 @@ -206,7 +284,7 @@ sessions: headers: fields: - [ Content-Length, 0 ] - - [ X-Response, head_fail_escalated ] + - [ X-Response, head_fail ] # With --escalate-non-get-methods, HEAD request is escalated to failover. proxy-response: