Skip to content

Commit f58329b

Browse files
tkan145tkan145
committed
Initial support to proxy request with Transfer-Encoding: chunked
1 parent fdf5b2e commit f58329b

File tree

3 files changed

+258
-24
lines changed

3 files changed

+258
-24
lines changed

‎gateway/src/apicast/configuration/service.lua‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,8 @@ local function get_request_params(method)
176176

177177
if method == "GET" then
178178
return params
179+
elseif ngx.req.get_headers()["Transfer-Encoding"] == "chunked" then
180+
return params
179181
else
180182
ngx.req.read_body()
181183
local body_params, err = ngx.req.get_post_args()

‎gateway/src/apicast/http_proxy.lua‎

Lines changed: 90 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ local resty_resolver = require 'resty.resolver'
55
local round_robin = require 'resty.balancer.round_robin'
66
local http_proxy = require 'resty.http.proxy'
77
local file_reader = require("resty.file").file_reader
8+
local chunked_reader = require('resty.http.chunked').chunked_reader
9+
local chunked_writer = require('resty.http.chunked').chunked_writer
10+
local req_headers = ngx.req.get_headers
11+
local http_ver = ngx.req.http_version
12+
local ngx_send_headers = ngx.send_headers
13+
local ngx_flush = ngx.flush
814

915
local _M = { }
1016

@@ -82,14 +88,58 @@ local function absolute_url(uri)
8288
end
8389

8490
local function forward_https_request(proxy_uri, uri, skip_https_connect)
85-
-- This is needed to call ngx.req.get_body_data() below.
86-
ngx.req.read_body()
91+
local sock, ok, err
92+
local chunksize = 32 * 1024
93+
local body
94+
local headers = req_headers()
95+
96+
local encoding = headers.transfer_encoding
97+
if type(encoding) == "table" then
98+
encoding = encoding[1]
99+
end
87100

88-
local request = {
89-
uri = uri,
90-
method = ngx.req.get_method(),
91-
headers = ngx.req.get_headers(0, true),
92-
path = format('%s%s%s', ngx.var.uri, ngx.var.is_args, ngx.var.query_string or ''),
101+
if encoding and encoding:lower() == "chunked" then
102+
if http_ver() ~= 1.1 then
103+
ngx.log(ngx.ERR, "bad http version")
104+
ngx.exit(ngx.HTTP_BAD_REQUEST)
105+
end
106+
local expect = headers["Expect"]
107+
108+
if type(expect) == "table" then
109+
expect = expect[1]
110+
end
111+
112+
if expect and expect:lower() == "100-continue" then
113+
ngx.status = 100
114+
ok, err = ngx_send_headers()
115+
116+
if not ok then
117+
ngx.log(ngx.ERR, "failed to send response header: " .. (err or "unknown"))
118+
end
119+
120+
ok, err = ngx_flush(true)
121+
if not ok then
122+
ngx.log(ngx.ERR, "failed to flush response header: " .. (err or "unknown"))
123+
end
124+
end
125+
126+
-- The default ngx reader does not support chunked request
127+
-- so we will need to get the raw request socket and manually
128+
-- decode the chunked request
129+
sock, err = ngx.req.socket(true)
130+
131+
if not sock then
132+
if err == "no body" then
133+
body = nil
134+
else
135+
return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)
136+
end
137+
else
138+
body = chunked_reader(sock, chunksize)
139+
end
140+
else
141+
-- This is needed to call ngx.req.get_body_data() below.
142+
ngx.req.read_body()
93143

94144
-- We cannot use resty.http's .get_client_body_reader().
95145
-- In POST requests with HTTPS, the result of that call is nil, and it
@@ -100,24 +150,31 @@ local function forward_https_request(proxy_uri, uri, skip_https_connect)
100150
-- read and need to be cached in a local file. This request will return
101151
-- nil, so after this we need to read the temp file.
102152
-- https://github.com/openresty/lua-nginx-module#ngxreqget_body_data
103-
body = ngx.req.get_body_data(),
104-
proxy_uri = proxy_uri
105-
}
106-
107-
if not request.body then
108-
local temp_file_path = ngx.req.get_body_file()
109-
ngx.log(ngx.INFO, "HTTPS Proxy: Request body is bigger than client_body_buffer_size, read the content from path='", temp_file_path, "'")
110-
111-
if temp_file_path then
112-
local body, err = file_reader(temp_file_path)
113-
if err then
114-
ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err)
115-
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
116-
end
117-
request.body = body
153+
body = ngx.req.get_body_data()
154+
155+
if not body then
156+
local temp_file_path = ngx.req.get_body_file()
157+
ngx.log(ngx.INFO, "HTTPS Proxy: Request body is bigger than client_body_buffer_size, read the content from path='", temp_file_path, "'")
158+
159+
if temp_file_path then
160+
body, err = file_reader(temp_file_path)
161+
if err then
162+
ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err)
163+
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
164+
end
165+
end
118166
end
119167
end
120168

169+
local request = {
170+
uri = uri,
171+
method = ngx.req.get_method(),
172+
headers = ngx.req.get_headers(0, true),
173+
path = format('%s%s%s', ngx.var.uri, ngx.var.is_args, ngx.var.query_string or ''),
174+
body = body,
175+
proxy_uri = proxy_uri
176+
}
177+
121178
local httpc, err = http_proxy.new(request, skip_https_connect)
122179

123180
if not httpc then
@@ -130,11 +187,20 @@ local function forward_https_request(proxy_uri, uri, skip_https_connect)
130187
res, err = httpc:request(request)
131188

132189
if res then
133-
httpc:proxy_response(res)
190+
-- if we are using raw socket we will need to send the response back with sock:send
191+
if sock then
192+
chunked_writer(sock, res ,chunksize)
193+
else
194+
httpc:proxy_response(res)
195+
end
134196
httpc:set_keepalive()
135197
else
136198
ngx.log(ngx.ERR, 'failed to proxy request to: ', proxy_uri, ' err : ', err)
137-
return ngx.exit(ngx.HTTP_BAD_GATEWAY)
199+
if sock then
200+
sock:send("HTTP/1.1 502 Bad Gateway\\r\\n")
201+
else
202+
return ngx.exit(ngx.HTTP_BAD_GATEWAY)
203+
end
138204
end
139205
end
140206

‎gateway/src/resty/http/chunked.lua‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
local co_wrap_iter = require("resty.coroutines").co_wrap_iter
2+
local co_yield = coroutine._yield
3+
4+
local _M = {
5+
}
6+
7+
local cr_lf = "\r\n"
8+
local default_max_chunk_size = 32 * 1024 -- 32K
9+
10+
local function send(socket, data)
11+
if not data or data == '' then
12+
ngx.log(ngx.DEBUG, 'skipping sending nil')
13+
return
14+
end
15+
16+
return socket:send(data)
17+
end
18+
19+
local function print_err(error_code, ...)
20+
ngx.log(ngx.ERR, ...)
21+
return ngx.exit(error_code)
22+
end
23+
24+
-- This is a copy of lua-resty-http _chunked_body_reader function but with
25+
-- extra bits to make chunked encoding work with raw socket
26+
-- https://github.com/ledgetech/lua-resty-http/blob/v0.16.1/lib/resty/http.lua#L418
27+
28+
-- chunked_reader return a body reader that translates the data read from sock
29+
-- out of HTTP "chunked" format before returning it
30+
--
31+
-- The chunked reader return nil when the final 0-length chunk is read
32+
function _M.chunked_reader(sock, max_chunk_size)
33+
max_chunk_size = max_chunk_size or default_max_chunk_size
34+
35+
if not sock then
36+
return nil, "chunked_reader: invalid sock"
37+
end
38+
39+
return co_wrap_iter(function()
40+
local eof = false
41+
local remaining = 0
42+
local size = 0
43+
repeat
44+
-- If we still have data on this chunk
45+
if max_chunk_size and remaining > 0 then
46+
if remaining > max_chunk_size then
47+
-- Consume up to max_chunk_size
48+
size = max_chunk_size
49+
remaining = remaining - max_chunk_size
50+
else
51+
-- Consume all remaining
52+
size = remaining
53+
remaining = 0
54+
end
55+
else
56+
-- read a line from socket
57+
-- chunk-size CRLF
58+
local line, err = sock:receive()
59+
if not line then
60+
co_yield(nil, "chunked_reader: failed to receive chunk size, err: " .. (err or "unknown"))
61+
end
62+
63+
size = tonumber(line, 16)
64+
if not size then
65+
co_yield(nil, "chunked_reader: unable to read chunksize")
66+
end
67+
68+
if max_chunk_size and size > max_chunk_size then
69+
-- Consume up to max_chunk_size
70+
remaining = size - max_chunk_size
71+
size = max_chunk_size
72+
end
73+
end
74+
75+
76+
if size > 0 then
77+
-- Receive the chunk
78+
local chunk, err = sock:receive(size)
79+
if not chunk then
80+
co_yield(nil, "chunked_reader: failed to receive chunk of size " .. size .. " err: " .. (err or "unknown"))
81+
end
82+
83+
if remaining == 0 then
84+
-- We're at the end of a chunk, read the next two bytes
85+
-- and verify they are "\r\n"
86+
local data, err = sock:receive(2)
87+
if not data then
88+
co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. (err or "unknown"))
89+
end
90+
end
91+
92+
chunk = string.format("%x\r\n", size) .. chunk .. cr_lf
93+
94+
co_yield(chunk)
95+
else
96+
-- we're at the end of a chunk, read the next two
97+
-- bytes to verify they are "\r\n".
98+
local chunk, err = sock:receive(2)
99+
if not chunk then
100+
co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. (err or "unknown"))
101+
end
102+
103+
if chunk ~= "\r\n" then
104+
co_yield(nil, "chunked_reader: bad chunk terminator")
105+
end
106+
107+
eof = true
108+
co_yield("0\r\n\r\n")
109+
break
110+
end
111+
until eof
112+
end)
113+
end
114+
115+
-- chunked_writer writes response body reader to sock in the HTTP/1.x server response format,
116+
-- including the status line, headers, body, and optional trailer.
117+
function _M.chunked_writer(sock, res, chunksize)
118+
local bytes, err
119+
chunksize = chunksize or 65536
120+
121+
-- Status line
122+
-- FIXME: should get protocol version from res?
123+
local status = "HTTP/1.1 " .. res.status .. " " .. res.reason .. cr_lf
124+
bytes, err = send(sock, status)
125+
if not bytes then
126+
print_err(503, "chunked_writer: failed to send status line, err: " .. (err or "unknown"))
127+
end
128+
129+
-- Rest of header
130+
for k, v in pairs(res.headers) do
131+
local header = k .. ": " .. v .. cr_lf
132+
bytes, err = send(sock, header)
133+
if not bytes then
134+
print_err(503, "chunked_writer: failed to send header, err: " .. (err or "unknown"))
135+
end
136+
end
137+
138+
-- End-of-header
139+
bytes, err = send(sock, cr_lf)
140+
if not bytes then
141+
print_err(503, "chunked_writer: failed to send end of header, err: " .. (err or "unknown"))
142+
end
143+
144+
-- Write body and trailer
145+
-- TODO: handle trailer
146+
if res.has_body then
147+
local reader = res.body_reader
148+
repeat
149+
local chunk, read_err
150+
151+
chunk, read_err = reader(chunksize)
152+
if read_err then
153+
print_err(503, "chunked_writer: failed to read body, err: " .. (err or "unknown"))
154+
end
155+
156+
if chunk then
157+
bytes, err = send(sock, chunk)
158+
if not bytes then
159+
print_err(503, "chunked_writer: failed to send body, err: " .. (err or "unknown"))
160+
end
161+
end
162+
until not chunk
163+
end
164+
end
165+
166+
return _M

0 commit comments

Comments
 (0)