Skip to content

Commit f5255ac

Browse files
AlenarAlenar
committed
refactor(client): ensure unknown locations are skipped in cardano db v2 client
And fails if they're all Unknown.
1 parent 67484e5 commit f5255ac

File tree

2 files changed

+148
-136
lines changed

2 files changed

+148
-136
lines changed

‎mithril-client/src/cardano_database_client/download_unpack.rs‎

Lines changed: 94 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use anyhow::anyhow;
1111
use mithril_common::{
1212
digesters::{IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR},
1313
entities::{AncillaryLocation, CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation},
14-
messages::CardanoDatabaseSnapshotMessage,
14+
messages::{AncillaryMessagePart, CardanoDatabaseSnapshotMessage, ImmutablesMessagePart},
1515
};
1616

1717
use crate::feedback::{FeedbackSender, MithrilEvent, MithrilEventCardanoDatabase};
@@ -121,24 +121,17 @@ impl InternalArtifactDownloader {
121121
last_immutable_file_number,
122122
)?;
123123
Self::verify_can_write_to_target_directory(target_dir, &download_unpack_options)?;
124-
let immutable_locations = &cardano_database_snapshot.immutables.locations;
125-
let average_size_uncompressed = cardano_database_snapshot
126-
.immutables
127-
.average_size_uncompressed;
128124

129125
let mut tasks = VecDeque::from(self.build_download_tasks_for_immutables(
130-
immutable_locations,
126+
&cardano_database_snapshot.immutables,
131127
immutable_file_number_range,
132128
target_dir,
133-
average_size_uncompressed,
134129
&download_id,
135130
)?);
136131
if download_unpack_options.include_ancillary {
137-
let ancillary = &cardano_database_snapshot.ancillary;
138132
tasks.push_back(self.new_ancillary_download_task(
139-
&ancillary.locations,
133+
&cardano_database_snapshot.ancillary,
140134
target_dir,
141-
ancillary.size_uncompressed,
142135
&download_id,
143136
)?);
144137
}
@@ -218,10 +211,9 @@ impl InternalArtifactDownloader {
218211

219212
fn build_download_tasks_for_immutables(
220213
&self,
221-
immutable_locations: &[ImmutablesLocation],
214+
immutable_locations: &ImmutablesMessagePart,
222215
immutable_file_number_range: RangeInclusive<ImmutableFileNumber>,
223216
target_dir: &Path,
224-
average_size_uncompressed: u64,
225217
download_id: &str,
226218
) -> MithrilResult<Vec<DownloadTask>> {
227219
let immutable_file_numbers_to_download = immutable_file_number_range
@@ -234,7 +226,6 @@ impl InternalArtifactDownloader {
234226
immutable_locations,
235227
immutable_file_number,
236228
target_dir,
237-
average_size_uncompressed,
238229
download_id,
239230
)?);
240231
}
@@ -243,14 +234,13 @@ impl InternalArtifactDownloader {
243234

244235
fn new_immutable_download_task(
245236
&self,
246-
locations: &[ImmutablesLocation],
237+
immutable_locations: &ImmutablesMessagePart,
247238
immutable_file_number: ImmutableFileNumber,
248239
immutable_files_target_dir: &Path,
249-
average_size_uncompressed: u64,
250240
download_id: &str,
251241
) -> MithrilResult<DownloadTask> {
252242
let mut locations_to_try = vec![];
253-
let mut locations_sorted = locations.to_owned();
243+
let mut locations_sorted = immutable_locations.sanitized_locations()?;
254244
locations_sorted.sort();
255245
for location in locations_sorted {
256246
let location_to_try = match location {
@@ -269,9 +259,8 @@ impl InternalArtifactDownloader {
269259
compression_algorithm: compression_algorithm.to_owned(),
270260
}
271261
}
272-
ImmutablesLocation::Unknown => {
273-
return Err(anyhow!("Unknown location type to download immutable"));
274-
}
262+
// Note: unknown locations should have been filtered out by `sanitized_locations`
263+
ImmutablesLocation::Unknown => unreachable!(),
275264
};
276265

277266
locations_to_try.push(location_to_try);
@@ -281,7 +270,7 @@ impl InternalArtifactDownloader {
281270
name: format!("immutable_file_{:05}", immutable_file_number),
282271
locations_to_try,
283272
target_dir: immutable_files_target_dir.to_path_buf(),
284-
size_uncompressed: average_size_uncompressed,
273+
size_uncompressed: immutable_locations.average_size_uncompressed,
285274
download_event: DownloadEvent::Immutable {
286275
download_id: download_id.to_string(),
287276
immutable_file_number,
@@ -291,13 +280,12 @@ impl InternalArtifactDownloader {
291280

292281
fn new_ancillary_download_task(
293282
&self,
294-
locations: &[AncillaryLocation],
283+
locations: &AncillaryMessagePart,
295284
ancillary_file_target_dir: &Path,
296-
size_uncompressed: u64,
297285
download_id: &str,
298286
) -> MithrilResult<DownloadTask> {
299287
let mut locations_to_try = vec![];
300-
let mut locations_sorted = locations.to_owned();
288+
let mut locations_sorted = locations.sanitized_locations()?;
301289
locations_sorted.sort();
302290
for location in locations_sorted {
303291
let location_to_try = match location {
@@ -314,9 +302,8 @@ impl InternalArtifactDownloader {
314302
compression_algorithm: compression_algorithm.to_owned(),
315303
}
316304
}
317-
AncillaryLocation::Unknown => {
318-
return Err(anyhow!("Unknown location type to download immutable"));
319-
}
305+
// Note: unknown locations should have been filtered out by `sanitized_locations`
306+
AncillaryLocation::Unknown => unreachable!(),
320307
};
321308

322309
locations_to_try.push(location_to_try);
@@ -326,7 +313,7 @@ impl InternalArtifactDownloader {
326313
name: "ancillary".to_string(),
327314
locations_to_try,
328315
target_dir: ancillary_file_target_dir.to_path_buf(),
329-
size_uncompressed,
316+
size_uncompressed: locations.size_uncompressed,
330317
download_event: DownloadEvent::Ancillary {
331318
download_id: download_id.to_string(),
332319
},
@@ -820,17 +807,19 @@ mod tests {
820807

821808
let tasks = artifact_downloader
822809
.build_download_tasks_for_immutables(
823-
&[ImmutablesLocation::CloudStorage {
824-
uri: MultiFilesUri::Template(TemplateUri(
825-
"http://whatever/{immutable_file_number}.tar.gz".to_string(),
826-
)),
827-
compression_algorithm: Some(CompressionAlgorithm::default()),
828-
}],
810+
&ImmutablesMessagePart {
811+
locations: vec![ImmutablesLocation::CloudStorage {
812+
uri: MultiFilesUri::Template(TemplateUri(
813+
"http://whatever/{immutable_file_number}.tar.gz".to_string(),
814+
)),
815+
compression_algorithm: Some(CompressionAlgorithm::default()),
816+
}],
817+
average_size_uncompressed: 0,
818+
},
829819
immutable_file_range
830820
.to_range_inclusive(total_immutable_files)
831821
.unwrap(),
832822
&target_dir,
833-
0,
834823
"download_id",
835824
)
836825
.unwrap();
@@ -842,7 +831,7 @@ mod tests {
842831
}
843832

844833
#[tokio::test]
845-
async fn building_immutables_download_tasks_fails_if_location_is_unknown() {
834+
async fn building_immutables_download_tasks_fails_if_all_locations_are_unknown() {
846835
let total_immutable_files = 2;
847836
let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files);
848837
let target_dir = TempDir::new("cardano_database_client", "download_unpack").build();
@@ -853,18 +842,20 @@ mod tests {
853842
);
854843

855844
let build_tasks_result = artifact_downloader.build_download_tasks_for_immutables(
856-
&[ImmutablesLocation::Unknown {}],
845+
&ImmutablesMessagePart {
846+
locations: vec![ImmutablesLocation::Unknown {}],
847+
average_size_uncompressed: 0,
848+
},
857849
immutable_file_range
858850
.to_range_inclusive(total_immutable_files)
859851
.unwrap(),
860852
&target_dir,
861-
0,
862853
"download_id",
863854
);
864855

865856
assert!(
866857
build_tasks_result.is_err(),
867-
"building tasks should fail if a location is unknown"
858+
"building tasks should fail if all location are unknown"
868859
);
869860
}
870861

@@ -891,17 +882,19 @@ mod tests {
891882

892883
let tasks = artifact_downloader
893884
.build_download_tasks_for_immutables(
894-
&[ImmutablesLocation::CloudStorage {
895-
uri: MultiFilesUri::Template(TemplateUri(
896-
"http://whatever-1/{immutable_file_number}.tar.gz".to_string(),
897-
)),
898-
compression_algorithm: Some(CompressionAlgorithm::default()),
899-
}],
885+
&ImmutablesMessagePart {
886+
locations: vec![ImmutablesLocation::CloudStorage {
887+
uri: MultiFilesUri::Template(TemplateUri(
888+
"http://whatever-1/{immutable_file_number}.tar.gz".to_string(),
889+
)),
890+
compression_algorithm: Some(CompressionAlgorithm::default()),
891+
}],
892+
average_size_uncompressed: 0,
893+
},
900894
immutable_file_range
901895
.to_range_inclusive(total_immutable_files)
902896
.unwrap(),
903897
&target_dir,
904-
0,
905898
"download_id",
906899
)
907900
.unwrap();
@@ -944,25 +937,27 @@ mod tests {
944937

945938
let tasks = artifact_downloader
946939
.build_download_tasks_for_immutables(
947-
&[
948-
ImmutablesLocation::CloudStorage {
949-
uri: MultiFilesUri::Template(TemplateUri(
950-
"http://whatever-1/{immutable_file_number}.tar.gz".to_string(),
951-
)),
952-
compression_algorithm: Some(CompressionAlgorithm::default()),
953-
},
954-
ImmutablesLocation::CloudStorage {
955-
uri: MultiFilesUri::Template(TemplateUri(
956-
"http://whatever-2/{immutable_file_number}.tar.gz".to_string(),
957-
)),
958-
compression_algorithm: Some(CompressionAlgorithm::default()),
959-
},
960-
],
940+
&ImmutablesMessagePart {
941+
locations: vec![
942+
ImmutablesLocation::CloudStorage {
943+
uri: MultiFilesUri::Template(TemplateUri(
944+
"http://whatever-1/{immutable_file_number}.tar.gz".to_string(),
945+
)),
946+
compression_algorithm: Some(CompressionAlgorithm::default()),
947+
},
948+
ImmutablesLocation::CloudStorage {
949+
uri: MultiFilesUri::Template(TemplateUri(
950+
"http://whatever-2/{immutable_file_number}.tar.gz".to_string(),
951+
)),
952+
compression_algorithm: Some(CompressionAlgorithm::default()),
953+
},
954+
],
955+
average_size_uncompressed: 0,
956+
},
961957
immutable_file_range
962958
.to_range_inclusive(total_immutable_files)
963959
.unwrap(),
964960
&target_dir,
965-
0,
966961
"download_id",
967962
)
968963
.unwrap();
@@ -989,12 +984,14 @@ mod tests {
989984

990985
let task = artifact_downloader
991986
.new_ancillary_download_task(
992-
&[AncillaryLocation::CloudStorage {
993-
uri: "http://whatever-1/ancillary.tar.gz".to_string(),
994-
compression_algorithm: Some(CompressionAlgorithm::default()),
995-
}],
987+
&AncillaryMessagePart {
988+
locations: vec![AncillaryLocation::CloudStorage {
989+
uri: "http://whatever-1/ancillary.tar.gz".to_string(),
990+
compression_algorithm: Some(CompressionAlgorithm::default()),
991+
}],
992+
size_uncompressed: 0,
993+
},
996994
target_dir,
997-
0,
998995
"download_id",
999996
)
1000997
.unwrap();
@@ -1006,7 +1003,7 @@ mod tests {
10061003
}
10071004

10081005
#[tokio::test]
1009-
async fn building_ancillary_download_task_fails_if_location_is_unknown() {
1006+
async fn building_ancillary_download_tasks_fails_if_all_locations_are_unknown() {
10101007
let target_dir = Path::new(".");
10111008
let artifact_downloader = InternalArtifactDownloader::new(
10121009
Arc::new(MockFileDownloader::new()),
@@ -1015,15 +1012,17 @@ mod tests {
10151012
);
10161013

10171014
let build_tasks_result = artifact_downloader.new_ancillary_download_task(
1018-
&[AncillaryLocation::Unknown {}],
1015+
&AncillaryMessagePart {
1016+
locations: vec![AncillaryLocation::Unknown {}],
1017+
size_uncompressed: 0,
1018+
},
10191019
target_dir,
1020-
0,
10211020
"download_id",
10221021
);
10231022

10241023
assert!(
10251024
build_tasks_result.is_err(),
1026-
"building tasks should fail if a location is unknown"
1025+
"building tasks should fail if all location are unknown"
10271026
);
10281027
}
10291028

@@ -1048,18 +1047,20 @@ mod tests {
10481047

10491048
let task = artifact_downloader
10501049
.new_ancillary_download_task(
1051-
&[
1052-
AncillaryLocation::CloudStorage {
1053-
uri: "http://whatever-1/ancillary.tar.gz".to_string(),
1054-
compression_algorithm: Some(CompressionAlgorithm::default()),
1055-
},
1056-
AncillaryLocation::CloudStorage {
1057-
uri: "http://whatever-2/ancillary.tar.gz".to_string(),
1058-
compression_algorithm: Some(CompressionAlgorithm::default()),
1059-
},
1060-
],
1050+
&AncillaryMessagePart {
1051+
locations: vec![
1052+
AncillaryLocation::CloudStorage {
1053+
uri: "http://whatever-1/ancillary.tar.gz".to_string(),
1054+
compression_algorithm: Some(CompressionAlgorithm::default()),
1055+
},
1056+
AncillaryLocation::CloudStorage {
1057+
uri: "http://whatever-2/ancillary.tar.gz".to_string(),
1058+
compression_algorithm: Some(CompressionAlgorithm::default()),
1059+
},
1060+
],
1061+
size_uncompressed: 0,
1062+
},
10611063
target_dir,
1062-
0,
10631064
"download_id",
10641065
)
10651066
.unwrap();
@@ -1087,18 +1088,20 @@ mod tests {
10871088

10881089
let task = artifact_downloader
10891090
.new_ancillary_download_task(
1090-
&[
1091-
AncillaryLocation::CloudStorage {
1092-
uri: "http://whatever-1/ancillary.tar.gz".to_string(),
1093-
compression_algorithm: Some(CompressionAlgorithm::default()),
1094-
},
1095-
AncillaryLocation::CloudStorage {
1096-
uri: "http://whatever-2/ancillary.tar.gz".to_string(),
1097-
compression_algorithm: Some(CompressionAlgorithm::default()),
1098-
},
1099-
],
1091+
&AncillaryMessagePart {
1092+
locations: vec![
1093+
AncillaryLocation::CloudStorage {
1094+
uri: "http://whatever-1/ancillary.tar.gz".to_string(),
1095+
compression_algorithm: Some(CompressionAlgorithm::default()),
1096+
},
1097+
AncillaryLocation::CloudStorage {
1098+
uri: "http://whatever-2/ancillary.tar.gz".to_string(),
1099+
compression_algorithm: Some(CompressionAlgorithm::default()),
1100+
},
1101+
],
1102+
size_uncompressed: 0,
1103+
},
11001104
target_dir,
1101-
0,
11021105
"download_id",
11031106
)
11041107
.unwrap();

0 commit comments

Comments
 (0)