Skip to content

Commit f456904

Browse files
GUIGUI
committed
Begin work on trying to test edge-cases around upload handling.
1 parent 8a049f1 commit f456904

File tree

2 files changed

+199
-2
lines changed

2 files changed

+199
-2
lines changed

‎templates/etc/test-env/nginx/apis.conf.etlua‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,12 @@ server {
150150

151151
location = /upload {
152152
content_by_lua_block {
153-
local cjson = require "cjson";
154-
local upload = require "resty.upload";
153+
local cjson = require "cjson"
154+
local resty_sha256 = require "resty.sha256"
155+
local str = require "resty.string"
156+
local upload = require "resty.upload"
157+
158+
local sha256 = resty_sha256:new()
155159

156160
local upload_size = 0
157161
local chunk_size = 4096
@@ -160,18 +164,61 @@ server {
160164
local typ, res, err = form:read()
161165
if typ == "body" then
162166
upload_size = upload_size + #res
167+
sha256:update(res)
163168
elseif typ == "eof" then
164169
break
165170
end
166171
end
167172

173+
local digest = sha256:final()
174+
168175
ngx.header["Content-Type"] = "application/json"
169176
ngx.print(cjson.encode({
177+
method = ngx.var.request_method,
178+
headers = ngx.req.get_headers(500),
170179
upload_size = upload_size,
180+
upload_checksum = str.to_hex(digest),
171181
}))
172182
}
173183
}
174184

185+
location = /read-body {
186+
content_by_lua_block {
187+
local cjson = require "cjson";
188+
local resty_sha256 = require "resty.sha256"
189+
local str = require "resty.string"
190+
191+
local sha256 = resty_sha256:new()
192+
193+
ngx.req.read_body()
194+
local data = ngx.req.get_body_data()
195+
if not data then
196+
local file = ngx.req.get_body_file()
197+
if file then
198+
local f = io.open(file, "r")
199+
data = f:read("*a")
200+
f:close()
201+
end
202+
end
203+
204+
sha256:update(data)
205+
local digest = sha256:final()
206+
207+
ngx.header["Content-Type"] = "application/json"
208+
local response = cjson.encode({
209+
method = ngx.var.request_method,
210+
headers = ngx.req.get_headers(500),
211+
body_size = #data,
212+
body_checksum = str.to_hex(digest),
213+
})
214+
ngx.say(response)
215+
}
216+
}
217+
218+
location = /unread-body {
219+
return 200 "{\"method\":\"$request_method\",\"http_content_length\":\"$http_content_length\"}";
220+
}
221+
175222
location = /chunked {
176223
echo -n "hello";
177224
echo_flush;

‎test/proxy/test_uploads.rb‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,154 @@ def test_large_uploads
2828
file.close
2929
file.unlink
3030
end
31+
32+
def test_mixed_uploads_stress_test
33+
requests = []
34+
35+
info_get_requests = Array.new(200) do
36+
Typhoeus::Request.new("http://127.0.0.1:9080/api/info/", http_options.deep_merge({
37+
headers: {
38+
"X-Unique" => SecureRandom.hex(40),
39+
},
40+
}))
41+
end
42+
requests += info_get_requests
43+
44+
read_body_requests = {}
45+
unread_body_requests = {}
46+
multipart_body_requests = {}
47+
above_max_body_size_requests = {}
48+
49+
[:post, :put, :patch].each do |method|
50+
read_body_requests[method] = Array.new(5) do
51+
body = SecureRandom.random_bytes(5 * 1024 * 1024 + rand(-200..200)).freeze # 5MB
52+
Typhoeus::Request.new("http://127.0.0.1:9080/api/read-body", http_options.deep_merge({
53+
method: method,
54+
body: body,
55+
headers: {
56+
"Content-Type" => "text/plain",
57+
"X-Expected-Body-Size" => body.bytesize,
58+
"X-Expected-Body-Checksum" => Digest::SHA256.hexdigest(body),
59+
},
60+
}))
61+
end
62+
requests += read_body_requests[method]
63+
64+
unread_body_requests[method] = Array.new(5) do
65+
body = SecureRandom.random_bytes(5 * 1024 * 1024 + rand(-200..200)).freeze # 5MB
66+
Typhoeus::Request.new("http://127.0.0.1:9080/api/unread-body", http_options.deep_merge({
67+
method: method,
68+
body: body,
69+
headers: {
70+
"Content-Type" => "text/plain",
71+
"X-Expected-Body-Size" => body.bytesize,
72+
},
73+
}))
74+
end
75+
requests += unread_body_requests[method]
76+
77+
multipart_body_requests[method] = Array.new(5) do
78+
file = Tempfile.new("large")
79+
body = SecureRandom.random_bytes(5 * 1024 * 1024 + rand(-200..200)).freeze # 5MB
80+
file.write(body)
81+
Typhoeus::Request.new("http://127.0.0.1:9080/api/upload", http_options.deep_merge({
82+
# Workaround for PUT not working with Typhoeus and multipart uploads
83+
# currently:
84+
# https://github.com/typhoeus/typhoeus/issues/389#issuecomment-3186406150
85+
method: :post,
86+
customrequest: method.to_s.upcase,
87+
body: { upload: file },
88+
headers: {
89+
"X-Expected-Body-Size" => body.bytesize,
90+
"X-Expected-Body-Checksum" => Digest::SHA256.hexdigest(body),
91+
},
92+
}))
93+
end
94+
requests += multipart_body_requests[method]
95+
96+
above_max_body_size_requests[method] = Array.new(5) do
97+
body = SecureRandom.random_bytes(5 * 1024 * 1024 + rand(-200..200)).freeze # 5MB
98+
Typhoeus::Request.new("https://127.0.0.1:9081/api-umbrella/v1/users", http_options.deep_merge({
99+
method: method,
100+
body: body,
101+
headers: {
102+
"Content-Type" => "text/plain",
103+
},
104+
}))
105+
end
106+
requests += above_max_body_size_requests[method]
107+
end
108+
109+
hydra = Typhoeus::Hydra.new(max_concurrency: 10)
110+
requests.shuffle
111+
requests.each do |request|
112+
hydra.queue(request)
113+
end
114+
hydra.run
115+
116+
requests.each do |request|
117+
puts request.response.code
118+
end
119+
120+
assert_equal(200, info_get_requests.length)
121+
info_get_requests.each do |request|
122+
assert_response_code(200, request.response)
123+
request_headers = request.original_options.fetch(:headers)
124+
data = MultiJson.load(request.response.body)
125+
assert_equal(request_headers.fetch("X-Unique"), data.fetch("headers").fetch("x-unique"))
126+
end
127+
128+
assert_equal(3, read_body_requests.length)
129+
read_body_requests.each do |method, method_requests|
130+
assert_equal(5, method_requests.length)
131+
method_requests.each do |request|
132+
assert_response_code(200, request.response)
133+
request_headers = request.original_options.fetch(:headers)
134+
data = MultiJson.load(request.response.body)
135+
assert_equal(request.original_options.fetch(:method).to_s.upcase, data.fetch("method"))
136+
assert_equal("text/plain", data.fetch("headers").fetch("content-type"))
137+
assert_equal(request_headers.fetch("X-Expected-Body-Size").to_s, data.fetch("headers").fetch("content-length"))
138+
assert_equal(request_headers.fetch("X-Expected-Body-Size"), data.fetch("body_size"))
139+
assert_equal(request_headers.fetch("X-Expected-Body-Checksum"), data.fetch("body_checksum"))
140+
end
141+
end
142+
143+
assert_equal(3, unread_body_requests.length)
144+
unread_body_requests.each do |method, method_requests|
145+
assert_equal(5, method_requests.length)
146+
method_requests.each do |request|
147+
assert_response_code(200, request.response)
148+
request_headers = request.original_options.fetch(:headers)
149+
data = MultiJson.load(request.response.body)
150+
assert_equal(request.original_options.fetch(:method).to_s.upcase, data.fetch("method"))
151+
assert_equal(request_headers.fetch("X-Expected-Body-Size").to_s, data.fetch("http_content_length"))
152+
refute(data.key?("body_size"))
153+
refute(data.key?("body_checksum"))
154+
end
155+
end
156+
157+
assert_equal(3, multipart_body_requests.length)
158+
multipart_body_requests.each do |method, method_requests|
159+
assert_equal(5, method_requests.length)
160+
method_requests.each do |request|
161+
assert_response_code(200, request.response)
162+
request_headers = request.original_options.fetch(:headers)
163+
data = MultiJson.load(request.response.body)
164+
assert_equal(request.original_options.fetch(:customrequest).to_s.upcase, data.fetch("method"))
165+
assert_match("multipart/form-data; boundary=", data.fetch("headers").fetch("content-type"))
166+
assert_in_delta(request_headers.fetch("X-Expected-Body-Size"), data.fetch("headers").fetch("content-length").to_i, 400)
167+
assert_equal(request_headers.fetch("X-Expected-Body-Size"), data.fetch("upload_size"))
168+
assert_equal(request_headers.fetch("X-Expected-Body-Checksum"), data.fetch("upload_checksum"))
169+
end
170+
end
171+
172+
assert_equal(3, above_max_body_size_requests.length)
173+
above_max_body_size_requests.each do |method, method_requests|
174+
assert_equal(5, method_requests.length)
175+
method_requests.each do |request|
176+
assert_response_code((request.response.code == 502) ? 502 : 413, request.response)
177+
assert_equal("text/html", request.response.headers["content-type"])
178+
end
179+
end
180+
end
31181
end

0 commit comments

Comments
 (0)