Skip to content

feat: use resuming downloads for file downloader #4415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NikolaMilosa merged 31 commits into from
Mar 26, 2025
Merged

feat: use resuming downloads for file downloader #4415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
f1f8ca9
refactor: use resuming downloads for file downloader
NikolaMilosa Mar 18, 2025
a05dac5
repin
NikolaMilosa Mar 18, 2025
7f7e05d
buildifier and gazelle
NikolaMilosa Mar 18, 2025
7ae412b
moving tests to use async mockito because of tokio
NikolaMilosa Mar 18, 2025
9face5d
Merge branch 'master' into nm-cont-downloads
NikolaMilosa Mar 18, 2025
25d6cc4
suggestions and fixes
NikolaMilosa Mar 18, 2025
24cf509
removing file if the file exists and no hash checking expected
NikolaMilosa Mar 19, 2025
6765b70
moving resuming download to a systest
NikolaMilosa Mar 19, 2025
9476319
Merge branch 'master' into nm-cont-downloads
NikolaMilosa Mar 19, 2025
4083247
clippy
NikolaMilosa Mar 19, 2025
d011d0b
remove extra deps from system test
NikolaMilosa Mar 19, 2025
96cba7c
adding better logging
NikolaMilosa Mar 19, 2025
3a2fc8f
adding clear comments
NikolaMilosa Mar 20, 2025
4d6451d
clippy
NikolaMilosa Mar 20, 2025
25b368b
clippy
NikolaMilosa Mar 20, 2025
617a6e5
buildifier and gazelle
NikolaMilosa Mar 20, 2025
ee21a2a
moving test to rust_test
NikolaMilosa Mar 20, 2025
ecce574
extracting a long test
NikolaMilosa Mar 20, 2025
ed805e6
suggestions
NikolaMilosa Mar 20, 2025
f5e1fab
renaming the test
NikolaMilosa Mar 20, 2025
ffe91b0
Merge branch 'master' into nm-cont-downloads
NikolaMilosa Mar 20, 2025
def4d3b
removing request timeout in favour of chunks timeout
NikolaMilosa Mar 21, 2025
c623a4d
First check then proceed
NikolaMilosa Mar 21, 2025
38268f6
reverting test for already existing file
NikolaMilosa Mar 22, 2025
8dc3d78
Merge branch 'master' into nm-cont-downloads
NikolaMilosa Mar 22, 2025
5e6d437
addressing suggestions
NikolaMilosa Mar 25, 2025
0b2d694
adding the distinction between an overall timeout and chunk timeout
NikolaMilosa Mar 26, 2025
c661930
better error handling
NikolaMilosa Mar 26, 2025
4e72339
better test name
NikolaMilosa Mar 26, 2025
2119069
Merge branch 'master' into nm-cont-downloads
NikolaMilosa Mar 26, 2025
0c84ff2
Merge branch 'master' into nm-cont-downloads
NikolaMilosa Mar 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 34 additions & 14 deletions Cargo.Bazel.Fuzzing.json.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"checksum": "bff29c17220de7fac8fd4a275c96c1b88ff7b1ecf5f0ef987bacf362e5044cff",
"checksum": "68aa001ce9abfda2b276ee96f8a584a5fe878c7eef2f8817c1a4f6301cf09995",
"crates": {
"abnf 0.12.0": {
"name": "abnf",
Expand Down Expand Up @@ -21093,7 +21093,7 @@
"target": "mockall"
},
{
"id": "mockito 1.2.0",
"id": "mockito 1.7.0",
"target": "mockito"
},
{
Expand Down Expand Up @@ -31723,7 +31723,6 @@
"common": [
"client",
"default",
"full",
"h2",
"http1",
"http2",
Expand Down Expand Up @@ -46390,14 +46389,14 @@
],
"license_file": "LICENSE-APACHE"
},
"mockito 1.2.0": {
"mockito 1.7.0": {
"name": "mockito",
"version": "1.2.0",
"version": "1.7.0",
"package_url": "https://github.com/lipanski/mockito",
"repository": {
"Http": {
"url": "https://static.crates.io/crates/mockito/1.2.0/download",
"sha256": "f8d3038e23466858569c2d30a537f691fa0d53b51626630ae08262943e3bbb8b"
"url": "https://static.crates.io/crates/mockito/1.7.0/download",
"sha256": "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48"
}
},
"targets": [
Expand All @@ -46423,7 +46422,8 @@
"common": [
"color",
"colored",
"default"
"default",
"parking_lot"
],
"selects": {}
},
Expand All @@ -46433,24 +46433,44 @@
"id": "assert-json-diff 2.0.2",
"target": "assert_json_diff"
},
{
"id": "bytes 1.9.0",
"target": "bytes"
},
{
"id": "colored 2.0.4",
"target": "colored"
},
{
"id": "futures 0.3.31",
"target": "futures"
"id": "futures-util 0.3.31",
"target": "futures_util"
},
{
"id": "hyper 0.14.27",
"id": "http 1.2.0",
"target": "http"
},
{
"id": "http-body 1.0.1",
"target": "http_body"
},
{
"id": "http-body-util 0.1.2",
"target": "http_body_util"
},
{
"id": "hyper 1.5.1",
"target": "hyper"
},
{
"id": "hyper-util 0.1.10",
"target": "hyper_util"
},
{
"id": "log 0.4.20",
"target": "log"
},
{
"id": "rand 0.8.5",
"id": "rand 0.9.0",
"target": "rand"
},
{
Expand All @@ -46477,7 +46497,7 @@
"selects": {}
},
"edition": "2021",
"version": "1.2.0"
"version": "1.7.0"
},
"license": "MIT",
"license_ids": [
Expand Down Expand Up @@ -94417,7 +94437,7 @@
"minicbor 0.19.1",
"minicbor-derive 0.13.0",
"mockall 0.13.0",
"mockito 1.2.0",
"mockito 1.7.0",
"moka 0.12.8",
"more-asserts 0.3.1",
"nftables 0.4.1",
Expand Down
15 changes: 10 additions & 5 deletions Cargo.Bazel.Fuzzing.toml.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7726,16 +7726,21 @@ dependencies = [

[[package]]
name = "mockito"
version = "1.2.0"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8d3038e23466858569c2d30a537f691fa0d53b51626630ae08262943e3bbb8b"
checksum = "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48"
dependencies = [
"assert-json-diff",
"bytes",
"colored",
"futures",
"hyper 0.14.27",
"futures-util",
"http 1.2.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.5.1",
"hyper-util",
"log",
"rand 0.8.5",
"rand 0.9.0",
"regex",
"serde_json",
"serde_urlencoded",
Expand Down
48 changes: 34 additions & 14 deletions Cargo.Bazel.json.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"checksum": "6b5cac5f03c3f73653b9fcc13d0d44e0d958cc37cd309d006ec71b5733d99f13",
"checksum": "0263b9a223c42fc2ecc48a3b8183818e33d79e8eb22e2726049e9d5495906a8c",
"crates": {
"abnf 0.12.0": {
"name": "abnf",
Expand Down Expand Up @@ -20921,7 +20921,7 @@
"target": "mockall"
},
{
"id": "mockito 1.2.0",
"id": "mockito 1.7.0",
"target": "mockito"
},
{
Expand Down Expand Up @@ -31578,7 +31578,6 @@
"common": [
"client",
"default",
"full",
"h2",
"http1",
"http2",
Expand Down Expand Up @@ -46230,14 +46229,14 @@
],
"license_file": "LICENSE-APACHE"
},
"mockito 1.2.0": {
"mockito 1.7.0": {
"name": "mockito",
"version": "1.2.0",
"version": "1.7.0",
"package_url": "https://github.com/lipanski/mockito",
"repository": {
"Http": {
"url": "https://static.crates.io/crates/mockito/1.2.0/download",
"sha256": "f8d3038e23466858569c2d30a537f691fa0d53b51626630ae08262943e3bbb8b"
"url": "https://static.crates.io/crates/mockito/1.7.0/download",
"sha256": "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48"
}
},
"targets": [
Expand All @@ -46263,7 +46262,8 @@
"common": [
"color",
"colored",
"default"
"default",
"parking_lot"
],
"selects": {}
},
Expand All @@ -46273,24 +46273,44 @@
"id": "assert-json-diff 2.0.2",
"target": "assert_json_diff"
},
{
"id": "bytes 1.9.0",
"target": "bytes"
},
{
"id": "colored 2.0.4",
"target": "colored"
},
{
"id": "futures 0.3.31",
"target": "futures"
"id": "futures-util 0.3.31",
"target": "futures_util"
},
{
"id": "hyper 0.14.27",
"id": "http 1.2.0",
"target": "http"
},
{
"id": "http-body 1.0.1",
"target": "http_body"
},
{
"id": "http-body-util 0.1.2",
"target": "http_body_util"
},
{
"id": "hyper 1.5.1",
"target": "hyper"
},
{
"id": "hyper-util 0.1.10",
"target": "hyper_util"
},
{
"id": "log 0.4.20",
"target": "log"
},
{
"id": "rand 0.8.5",
"id": "rand 0.9.0",
"target": "rand"
},
{
Expand All @@ -46317,7 +46337,7 @@
"selects": {}
},
"edition": "2021",
"version": "1.2.0"
"version": "1.7.0"
},
"license": "MIT",
"license_ids": [
Expand Down Expand Up @@ -94330,7 +94350,7 @@
"minicbor 0.19.1",
"minicbor-derive 0.13.0",
"mockall 0.13.0",
"mockito 1.2.0",
"mockito 1.7.0",
"moka 0.12.8",
"more-asserts 0.3.1",
"nftables 0.4.1",
Expand Down
15 changes: 10 additions & 5 deletions Cargo.Bazel.toml.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7717,16 +7717,21 @@ dependencies = [

[[package]]
name = "mockito"
version = "1.2.0"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8d3038e23466858569c2d30a537f691fa0d53b51626630ae08262943e3bbb8b"
checksum = "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48"
dependencies = [
"assert-json-diff",
"bytes",
"colored",
"futures",
"hyper 0.14.27",
"futures-util",
"http 1.2.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.5.1",
"hyper-util",
"log",
"rand 0.8.5",
"rand 0.9.0",
"regex",
"serde_json",
"serde_urlencoded",
Expand Down
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,7 @@ memmap2 = "0.9.5"
minicbor = { version = "0.19.1", features = ["alloc", "derive"] }
minicbor-derive = "0.13.0"
mockall = "0.13.0"
mockito = "1.2.0"
mockito = "1.6.1"
nftables = "0.4"
nix = { version = "0.24.3", features = ["ptrace"] }
num_cpus = "1.16.0"
Expand Down
2 changes: 1 addition & 1 deletion bazel/external_crates.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,7 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable
version = "^0.13.0",
),
"mockito": crate.spec(
version = "^1.2.0",
version = "^1.6.1",
),
"moka": crate.spec(
version = "^0.12.8",
Expand Down
12 changes: 12 additions & 0 deletions rs/http_utils/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
load("//rs/tests:common.bzl", "MAINNET_ENV")

package(default_visibility = ["//visibility:public"])

Expand Down Expand Up @@ -37,3 +38,14 @@ rust_test(
crate = ":http_utils",
deps = DEPENDENCIES + DEV_DEPENDENCIES,
)

rust_test(
name = "resuming_download_test",
srcs = glob(["tests/**/*.rs"]),
env = MAINNET_ENV,
tags = [
"long_test", # this test doesn't necessarily take long but it downloads a big image from download proxy so, to save bandwidth, we don't want to do that for every update to every PR and only run this on pushes to master.
"requires-network",
],
deps = DEPENDENCIES + DEV_DEPENDENCIES + [":http_utils"],
)
Loading
Loading