Skip to content

Commit c9522df

Browse files
authored
chunk uploads for large sast scan uploads (#64)
* chunk uploadds for large sast scan upload * change limi to 50 mb for chunk upload * bump version
1 parent be6b931 commit c9522df

File tree

3 files changed

+39
-7
lines changed

3 files changed

+39
-7
lines changed

Cargo.lock

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "corgea"
3-
version = "1.7.1"
3+
version = "1.7.2"
44
edition = "2021"
55

66
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

src/scan.rs

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -223,11 +223,43 @@ pub fn upload_scan(config: &Config, paths: Vec<String>, scanner: String, input:
223223
println!("Uploading the scan...");
224224

225225
// main scan upload
226-
debug(&format!("POST: {}", scan_upload_url));
227-
let res = client.post(scan_upload_url)
228-
.header(header::CONTENT_TYPE, "application/json")
229-
.body(input.clone())
230-
.send();
226+
let input_bytes = input.as_bytes();
227+
let input_size = input_bytes.len();
228+
let max_upload_size = 50 * 1024 * 1024; // 50mb
229+
let chunk_size = 1024 * 1024; // 1mb
230+
let res = if input_size > max_upload_size {
231+
let total_chunks = (input_size + chunk_size - 1) / chunk_size;
232+
debug(&format!("Uploading scan in {} chunks", total_chunks));
233+
let mut offset = 0usize;
234+
let mut last_response = None;
235+
236+
for (index, chunk) in input_bytes.chunks(chunk_size).enumerate() {
237+
debug(&format!("POST: {} (chunk {}/{})", scan_upload_url, index + 1, total_chunks));
238+
let response = client.post(&scan_upload_url)
239+
.header(header::CONTENT_TYPE, "application/json")
240+
.header("Upload-Offset", offset.to_string())
241+
.header("Upload-Length", input_size.to_string())
242+
.body(chunk.to_vec())
243+
.send();
244+
let should_break = match &response {
245+
Ok(res) => !res.status().is_success(),
246+
Err(_) => true,
247+
};
248+
last_response = Some(response);
249+
if should_break {
250+
break;
251+
}
252+
offset += chunk.len();
253+
}
254+
255+
last_response.expect("Failed to upload scan.")
256+
} else {
257+
debug(&format!("POST: {}", scan_upload_url));
258+
client.post(&scan_upload_url)
259+
.header(header::CONTENT_TYPE, "application/json")
260+
.body(input.clone())
261+
.send()
262+
};
231263

232264
let mut sast_scan_id: Option<String> = None;
233265

0 commit comments

Comments
 (0)