Skip to content

Commit 79b1478

Browse files
ghubertpaloDamien LACHAUME / PALO-IT
ghubertpalo
authored and
Damien LACHAUME / PALO-IT
committed
import complete certificate chain
1 parent 43a7d24 commit 79b1478

File tree

2 files changed

+137
-69
lines changed

2 files changed

+137
-69
lines changed

‎mithril-aggregator-fake/scripts/import.sh‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ download_artifacts() {
2626
local -r url=${1:?"No URL given to download_list function."};
2727
local -r artifact=${2:?"No artifact type given to download_list function."};
2828
local -r json_field=${3:?"No JSON field given to read from artifact list."};
29-
local -i nb=1
29+
local -i nb=0
3030

3131
echo -n "Downloading ${artifact} data: "
3232
tput sc;
@@ -38,25 +38,27 @@ download_artifacts() {
3838
let "nb=nb+1"
3939
echo -n "$nb "
4040
done
41-
echo
41+
echo " ✅";
4242
}
4343

4444
download_certificate_chain() {
45-
echo -n "Downloading certificate chain: "
46-
tput sc;
4745
local parent_hash=$(jq -r .[0].hash $DATA_DIR/certificates.json);
4846
local certificate_hash;
47+
local -i nb=0
48+
49+
echo -n "Downloading certificate chain: "
50+
tput sc;
4951

5052
until [ -z "$parent_hash" ];
5153
do
54+
tput rc;
5255
certificate_hash=$parent_hash;
5356
wget -O $DATA_DIR/certificate-${certificate_hash}.json --quiet "${BASE_URL}/certificate/${certificate_hash}";
5457
parent_hash=$(jq -r .previous_hash $DATA_DIR/certificate-${certificate_hash}.json);
55-
echo -n "${parent_hash} ";
56-
tput rc;
58+
let "nb=nb+1"
59+
echo -n "$nb "
5760
done
58-
59-
echo "ok ";
61+
echo " ✅";
6062
}
6163

6264
# MAIN execution
@@ -71,12 +73,13 @@ export DATA_DIR URL;
7173

7274
clean_directory;
7375

74-
download_list $BASE_URL/certificates certificates
76+
download_list "$BASE_URL/certificates" "certificates"
77+
download_artifacts "$BASE_URL/certificate" "certificate" "hash"
7578
download_certificate_chain
7679

77-
download_list $BASE_URL/artifact/snapshots snapshots
78-
download_artifacts $BASE_URL/artifact/snapshot snapshot digest
80+
download_list "$BASE_URL/artifact/snapshots" "snapshots"
81+
download_artifacts "$BASE_URL/artifact/snapshot" "snapshot" "digest"
7982

80-
download_list $BASE_URL/artifact/mithril-stake-distributions mithril-stake-distributions
81-
download_artifacts $BASE_URL/artifact/mithril-stake-distribution mithril-stake-distribution "hash"
83+
download_list "$BASE_URL/artifact/mithril-stake-distributions" "mithril-stake-distributions"
84+
download_artifacts "$BASE_URL/artifact/mithril-stake-distribution" "mithril-stake-distribution" "hash"
8285

‎mithril-aggregator-fake/src/shared_state.rs‎

Lines changed: 121 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
#[path = "default_values.rs"]
22
mod default_values;
33

4-
use std::{collections::BTreeMap, path::Path, sync::Arc};
4+
use std::{
5+
collections::BTreeMap,
6+
path::{Path, PathBuf},
7+
sync::Arc,
8+
};
59

610
use anyhow::{anyhow, Context};
711
use serde_json::Value;
@@ -48,10 +52,12 @@ impl AppState {
4852
/// Construct the Application state by reading data from the given directory.
4953
/// This will fail if some files are missing or are inconsistent.
5054
pub fn from_directory(data_dir: &Path) -> StdResult<Self> {
55+
let reader = DataDir::new(data_dir)?;
5156
let epoch_settings = default_values::epoch_settings().to_owned();
52-
let (certificate_list, certificates) = read_files(data_dir, "certificate", "hash")?;
53-
let (snapshot_list, snapshots) = read_files(data_dir, "snapshot", "digest")?;
54-
let (msd_list, msds) = read_files(data_dir, "mithril-stake-distribution", "hash")?;
57+
let (certificate_list, mut certificates) = reader.read_files("certificate", "hash")?;
58+
reader.read_certificate_chain(&certificate_list, &mut certificates)?;
59+
let (snapshot_list, snapshots) = reader.read_files("snapshot", "digest")?;
60+
let (msd_list, msds) = reader.read_files("mithril-stake-distribution", "hash")?;
5561

5662
let instance = Self {
5763
epoch_settings,
@@ -102,71 +108,130 @@ impl AppState {
102108
}
103109
}
104110

105-
/// Read related entity JSON files in the given directory.
106-
fn read_files(
107-
data_dir: &Path,
108-
entity: &str,
109-
field_id: &str,
110-
) -> StdResult<(String, BTreeMap<String, String>)> {
111-
debug!("Read data files, entity='{entity}', field='{field_id}'.");
112-
113-
if !data_dir.exists() {
114-
return Err(anyhow!(format!(
115-
"Path '{}' does not exist.",
116-
data_dir.display()
117-
)));
118-
}
111+
struct DataDir {
112+
data_dir: PathBuf,
113+
}
114+
115+
impl DataDir {
116+
/// public constructor
117+
pub fn new(data_dir: &Path) -> StdResult<Self> {
118+
if !data_dir.exists() {
119+
return Err(anyhow!(format!(
120+
"Path '{}' does not exist.",
121+
data_dir.display()
122+
)));
123+
}
119124

120-
if !data_dir.is_dir() {
121-
return Err(anyhow!(format!(
122-
"Path '{}' is not a directory!",
123-
data_dir.display()
124-
)));
125+
if !data_dir.is_dir() {
126+
return Err(anyhow!(format!(
127+
"Path '{}' is not a directory!",
128+
data_dir.display()
129+
)));
130+
}
131+
132+
let instance = Self {
133+
data_dir: data_dir.to_owned(),
134+
};
135+
136+
Ok(instance)
125137
}
126-
let list_file = {
127-
let list_file_name = format!("{entity}s.json");
128-
129-
data_dir.to_owned().join(list_file_name)
130-
};
131-
trace!("Reading JSON list file '{}'.", list_file.display());
132-
let list = std::fs::read_to_string(&list_file)
133-
.with_context(|| format!("Error while reading file '{}'.", list_file.display()))?;
134-
let list_json: Value = serde_json::from_str(&list)
135-
.with_context(|| format!("Could not parse JSON in file '{}'.", list_file.display()))?;
136-
let ids: Vec<String> = list_json
137-
.as_array()
138-
.ok_or_else(|| {
139-
anyhow!(format!(
140-
"List file for entity {entity} is not a JSON array."
141-
))
142-
})?
143-
.iter()
144-
.map(|v| {
145-
v[field_id].as_str().map(|s| s.to_owned()).ok_or_else(|| {
146-
anyhow!(format!(
147-
"Field '{field_id}' for type '{entity}' did not return a string (value: '{}').",
148-
v.to_string()
149-
))
150-
})
151-
})
152-
.collect::<StdResult<Vec<String>>>()?;
153138

154-
let mut collection: BTreeMap<String, String> = BTreeMap::new();
139+
fn read_list_file(&self, entity: &str) -> StdResult<(String, Value)> {
140+
let list_file = {
141+
let list_file_name = format!("{entity}s.json");
142+
143+
self.data_dir.to_owned().join(list_file_name)
144+
};
145+
trace!("Reading JSON list file '{}'.", list_file.display());
146+
let list = std::fs::read_to_string(&list_file)
147+
.with_context(|| format!("Error while reading file '{}'.", list_file.display()))?;
148+
let list_json: Value = serde_json::from_str(&list)
149+
.with_context(|| format!("Could not parse JSON in file '{}'.", list_file.display()))?;
150+
151+
Ok((list, list_json))
152+
}
155153

156-
for id in ids {
154+
fn read_entity_file(&self, entity: &str, id: &str) -> StdResult<(String, Value)> {
157155
let filename = format!("{entity}-{id}.json");
158-
let path = data_dir.to_owned().join(&filename);
156+
let path = self.data_dir.to_owned().join(&filename);
159157
trace!("Reading {entity} JSON file '{}'.", path.display());
160158
let content = std::fs::read_to_string(&path)
161159
.with_context(|| format!("Could not read entity file '{}'.", path.display()))?;
162-
let _value: Value = serde_json::from_str(&content).with_context(|| {
160+
let value: Value = serde_json::from_str(&content).with_context(|| {
163161
format!(
164162
"Entity file '{}' does not seem to hold valid JSON content.",
165163
path.display()
166164
)
167165
})?;
168-
collection.insert(id, content);
166+
167+
Ok((content, value))
169168
}
170169

171-
Ok((list, collection))
170+
/// Read related entity JSON files in the given directory.
171+
pub fn read_files(
172+
&self,
173+
entity: &str,
174+
field_id: &str,
175+
) -> StdResult<(String, BTreeMap<String, String>)> {
176+
debug!("Read data files, entity='{entity}', field='{field_id}'.");
177+
178+
let (list, list_json) = self.read_list_file(entity)?;
179+
let ids: Vec<String> = list_json
180+
.as_array()
181+
.ok_or_else(|| {
182+
anyhow!(format!(
183+
"List file for entity {entity} is not a JSON array."
184+
))
185+
})?
186+
.iter()
187+
.map(|v| {
188+
v[field_id].as_str().map(|s| s.to_owned()).ok_or_else(|| {
189+
anyhow!(format!(
190+
"Field '{field_id}' for type '{entity}' did not return a string (value: '{}').",
191+
v.to_string()
192+
))
193+
})
194+
})
195+
.collect::<StdResult<Vec<String>>>()?;
196+
197+
let mut collection: BTreeMap<String, String> = BTreeMap::new();
198+
199+
for id in &ids {
200+
let (content, _value) = self.read_entity_file(entity, id)?;
201+
collection.insert(id.to_owned(), content);
202+
}
203+
204+
Ok((list, collection))
205+
}
206+
207+
pub fn read_certificate_chain(
208+
&self,
209+
certificate_list: &str,
210+
certificates: &mut BTreeMap<String, String>,
211+
) -> StdResult<()> {
212+
trace!("fetching certificate chain");
213+
let list = serde_json::from_str::<Value>(certificate_list)?
214+
.as_array()
215+
.map(|v| v.to_owned())
216+
.ok_or_else(|| anyhow!("Could not cast certificates.json as JSON array."))?;
217+
let mut previous_hash = list[0]["previous_hash"]
218+
.as_str()
219+
.map(|v| v.to_owned())
220+
.ok_or_else(|| anyhow!("Field 'previous_hash' does not exist in the first certificate of the certificatd list."))?;
221+
222+
while previous_hash.len() > 0 {
223+
let (certificate, value) = self.read_entity_file("certificate", &previous_hash)?;
224+
let _ = certificates.insert(previous_hash.clone(), certificate);
225+
previous_hash = value["previous_hash"]
226+
.as_str()
227+
.map(|v| v.to_owned())
228+
.ok_or_else(|| {
229+
anyhow!(
230+
"field 'previous_hash' does not exist in certificate id='{previous_hash}'."
231+
)
232+
})?;
233+
}
234+
235+
Ok(())
236+
}
172237
}

0 commit comments

Comments
 (0)