Skip to content

Commit 3e25e57

Browse files
tkan145tkan145
committed
Fix 'Transfer-Encoding: chunked' issue when sending request via proxy
1 parent 1ae6223 commit 3e25e57

File tree

5 files changed

+936
-0
lines changed

5 files changed

+936
-0
lines changed

‎gateway/src/apicast/http_proxy.lua‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
local format = string.format
2+
local string_lower = string.lower
23

34
local resty_url = require "resty.url"
45
local resty_resolver = require 'resty.resolver'
56
local round_robin = require 'resty.balancer.round_robin'
67
local http_proxy = require 'resty.http.proxy'
78
local file_reader = require("resty.file").file_reader
9+
local file_size = require("resty.file").file_size
810

911
local _M = { }
1012

@@ -81,6 +83,27 @@ local function absolute_url(uri)
8183
)
8284
end
8385

86+
-- Sets a header with the given value. Any existing header with the same name will be overidden
87+
-- This also taking care of lowercase header when running on Openshift
88+
local function set_header(headers, header, value)
89+
local lowercase_header = string_lower(header)
90+
if headers[header] then
91+
headers[header] = value
92+
else
93+
headers[lowercase_header] = value
94+
end
95+
end
96+
97+
local function modify_chunked_request_headers(req, file)
98+
local contentLength, err = file_size(file)
99+
if err then
100+
ngx.log(ngx.ERR, "HTTPS proxy: Failed to set content length, err: ", err)
101+
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
102+
end
103+
set_header(req.headers, "Content-Length", contentLength)
104+
end
105+
106+
84107
local function forward_https_request(proxy_uri, uri, skip_https_connect)
85108
-- This is needed to call ngx.req.get_body_data() below.
86109
ngx.req.read_body()
@@ -114,10 +137,22 @@ local function forward_https_request(proxy_uri, uri, skip_https_connect)
114137
ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err)
115138
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
116139
end
140+
if ngx.var.http_transfer_encoding == "chunked" then
141+
-- If the body is smaller than "client_boby_buffer_size" the Content-Length header is
142+
-- set based on the size of the buffer. However, when the body is rendered to a file,
143+
-- we will need to calculate and manually set the Content-Length header based on the
144+
-- file size
145+
modify_chunked_request_headers(request, temp_file_path)
146+
end
117147
request.body = body
118148
end
119149
end
120150

151+
-- The whole request is buffered in the memory so remove the Transfer-Encoding: chunked
152+
if ngx.var.http_transfer_encoding == "chunked" then
153+
set_header(request.headers, "Transfer-Encoding", nil)
154+
end
155+
121156
local httpc, err = http_proxy.new(request, skip_https_connect)
122157

123158
if not httpc then

‎gateway/src/resty/file.lua‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,16 @@ function _M.file_reader(filename)
2828
end)
2929
end
3030

31+
function _M.file_size(filename)
32+
local handle, err = open(filename)
33+
if err then
34+
return nil, err
35+
end
36+
local current = handle:seek()
37+
local size = handle:seek("end")
38+
handle:seek("set", current)
39+
handle:close()
40+
return size
41+
end
42+
3143
return _M

‎t/apicast-policy-camel.t‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,3 +315,223 @@ ETag: foobar
315315
<<EOF
316316
using proxy: http://127.0.0.1:$Test::Nginx::Util::PROXY_SSL_PORT,
317317
EOF
318+
319+
320+
=== TEST 5: API backend connection uses http proxy with chunked request
321+
--- configuration
322+
{
323+
"services": [
324+
{
325+
"id": 42,
326+
"backend_version": 1,
327+
"backend_authentication_type": "service_token",
328+
"backend_authentication_value": "token-value",
329+
"proxy": {
330+
"api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/",
331+
"proxy_rules": [
332+
{ "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 }
333+
],
334+
"policy_chain": [
335+
{
336+
"name": "apicast.policy.apicast"
337+
},
338+
{
339+
"name": "apicast.policy.camel",
340+
"configuration": {
341+
"http_proxy": "$TEST_NGINX_HTTP_PROXY"
342+
}
343+
}
344+
]
345+
}
346+
}
347+
]
348+
}
349+
--- backend
350+
location /transactions/authrep.xml {
351+
content_by_lua_block {
352+
ngx.exit(ngx.OK)
353+
}
354+
}
355+
--- upstream
356+
server_name test-upstream.lvh.me;
357+
location / {
358+
access_by_lua_block {
359+
assert = require('luassert')
360+
local content_length = ngx.req.get_headers()["Content-Length"]
361+
local encoding = ngx.req.get_headers()["Transfer-Encoding"]
362+
assert.equal('12', content_length)
363+
assert.falsy(encoding)
364+
ngx.say("yay, api backend")
365+
}
366+
}
367+
--- more_headers
368+
Transfer-Encoding: chunked
369+
--- request eval
370+
"POST /?user_key=value
371+
7\r
372+
hello, \r
373+
5\r
374+
world\r
375+
0\r
376+
\r
377+
"
378+
--- error_code: 200
379+
--- error_log env
380+
using proxy: $TEST_NGINX_HTTP_PROXY
381+
382+
383+
=== TEST 6: API backend using all_proxy with chunked request
384+
--- configuration
385+
{
386+
"services": [
387+
{
388+
"id": 42,
389+
"backend_version": 1,
390+
"backend_authentication_type": "service_token",
391+
"backend_authentication_value": "token-value",
392+
"proxy": {
393+
"api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/",
394+
"proxy_rules": [
395+
{ "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 }
396+
],
397+
"policy_chain": [
398+
{
399+
"name": "apicast.policy.apicast"
400+
},
401+
{
402+
"name": "apicast.policy.http_proxy",
403+
"configuration": {
404+
"all_proxy": "$TEST_NGINX_HTTP_PROXY"
405+
}
406+
}
407+
]
408+
}
409+
}
410+
]
411+
}
412+
--- backend
413+
location /transactions/authrep.xml {
414+
content_by_lua_block {
415+
local expected = "service_token=token-value&service_id=42&usage%5Bhits%5D=2&user_key=value"
416+
require('luassert').same(ngx.decode_args(expected), ngx.req.get_uri_args(0))
417+
}
418+
}
419+
--- upstream
420+
server_name test-upstream.lvh.me;
421+
location / {
422+
access_by_lua_block {
423+
assert = require('luassert')
424+
local content_length = ngx.req.get_headers()["Content-Length"]
425+
local encoding = ngx.req.get_headers()["Transfer-Encoding"]
426+
assert.equal('12', content_length)
427+
assert.falsy(encoding)
428+
ngx.say("yay, api backend")
429+
}
430+
}
431+
--- more_headers
432+
Transfer-Encoding: chunked
433+
--- request eval
434+
"POST /?user_key=value
435+
7\r
436+
hello, \r
437+
5\r
438+
world\r
439+
0\r
440+
\r
441+
"
442+
--- error_code: 200
443+
--- error_log env
444+
using proxy: $TEST_NGINX_HTTP_PROXY
445+
446+
447+
=== TEST 7: using HTTPS proxy for backend with chunked request
448+
--- ONLY
449+
--- init eval
450+
$Test::Nginx::Util::PROXY_SSL_PORT = Test::APIcast::get_random_port();
451+
$Test::Nginx::Util::ENDPOINT_SSL_PORT = Test::APIcast::get_random_port();
452+
--- configuration random_port env eval
453+
<<EOF
454+
{
455+
"services": [
456+
{
457+
"backend_version": 1,
458+
"proxy": {
459+
"api_backend": "https://localhost:$Test::Nginx::Util::ENDPOINT_SSL_PORT",
460+
"proxy_rules": [
461+
{ "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 }
462+
],
463+
"policy_chain": [
464+
{
465+
"name": "apicast.policy.apicast"
466+
},
467+
{
468+
"name": "apicast.policy.camel",
469+
"configuration": {
470+
"https_proxy": "http://127.0.0.1:$Test::Nginx::Util::PROXY_SSL_PORT"
471+
}
472+
}
473+
]
474+
}
475+
}
476+
]
477+
}
478+
EOF
479+
--- backend
480+
location /transactions/authrep.xml {
481+
content_by_lua_block {
482+
ngx.exit(ngx.OK)
483+
}
484+
}
485+
--- upstream eval
486+
<<EOF
487+
# Endpoint config
488+
listen $Test::Nginx::Util::ENDPOINT_SSL_PORT ssl;
489+
490+
ssl_certificate $Test::Nginx::Util::ServRoot/html/server.crt;
491+
ssl_certificate_key $Test::Nginx::Util::ServRoot/html/server.key;
492+
493+
server_name _ default_server;
494+
495+
location / {
496+
access_by_lua_block {
497+
assert = require('luassert')
498+
local content_length = ngx.req.get_headers()["Content-Length"]
499+
local encoding = ngx.req.get_headers()["Transfer-Encoding"]
500+
assert.equal('12', content_length)
501+
assert.falsy(encoding)
502+
ngx.say("yay, api backend")
503+
}
504+
}
505+
}
506+
server {
507+
# Proxy config
508+
listen $Test::Nginx::Util::PROXY_SSL_PORT ssl;
509+
510+
ssl_certificate $Test::Nginx::Util::ServRoot/html/server.crt;
511+
ssl_certificate_key $Test::Nginx::Util::ServRoot/html/server.key;
512+
513+
514+
server_name _ default_server;
515+
516+
location ~ /.* {
517+
proxy_http_version 1.1;
518+
proxy_pass https://\$http_host;
519+
}
520+
EOF
521+
--- more_headers
522+
Transfer-Encoding: chunked
523+
--- request eval
524+
"POST /?user_key=value
525+
7\r
526+
hello, \r
527+
5\r
528+
world\r
529+
0\r
530+
\r
531+
"
532+
--- error_code: 200
533+
--- user_files fixture=tls.pl eval
534+
--- error_log eval
535+
<<EOF
536+
using proxy: http://127.0.0.1:$Test::Nginx::Util::PROXY_SSL_PORT,
537+
EOF

0 commit comments

Comments
 (0)