add import script

Author: ghubertpalo

Cargo.lock

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set +a -eu -o pipefail
+#set -v
+
+error() {
+    echo $1;
+    exit 1;
+}
+
+clean_directory() {
+    echo "Cleaning data directory…"
+    rm $DATA_DIR/*.json || true
+}
+
+# $1=URL $2=artifact_name
+download_list() {
+    local -r url=${1:?"No URL given to download_list function."};
+    local -r artifact=${2:?"No artifact type given to download_list function."};
+
+    echo "Downloading ${artifact} list."
+    wget -O $DATA_DIR/${artifact}.json --quiet "${url}"
+}
+
+# $1=URL $2=artifact_name $3=JSON field
+download_artifacts() {
+    local -r url=${1:?"No URL given to download_list function."};
+    local -r artifact=${2:?"No artifact type given to download_list function."};
+    local -r json_field=${3:?"No JSON field given to read from artifact list."};
+    local -i nb=1
+
+    echo -n "Downloading ${artifact} data: "
+    tput sc;
+
+    for digest in $(jq -r .[].${json_field} < $DATA_DIR/${artifact}s.json);
+    do
+        tput rc;
+        wget -O $DATA_DIR/${artifact}-${digest}.json --quiet "${url}/${digest}"
+        let "nb=nb+1"
+        echo -n "$nb   "
+    done
+    echo
+}
+
+# MAIN execution
+
+declare -r DATA_DIR=${1:?"No data directory given to download JSON files."};
+declare -r BASE_URL=${2:?"No Mithril Aggregator URL given."};
+
+if [ ! -d "$DATA_DIR" ]; then error "Specified directory '${DATA_DIR}' is not a directory."; fi
+wget --quiet --server-response --spider $BASE_URL || error "Could not reach URL '${BASE_URL}'.";
+
+export DATA_DIR URL;
+
+clean_directory;
+
+download_list $BASE_URL/certificates certificates
+download_artifacts $BASE_URL/certificate certificate "hash"
+
+download_list $BASE_URL/artifact/snapshots snapshots
+download_artifacts $BASE_URL/artifact/snapshot snapshot digest
+
+download_list $BASE_URL/artifact/mithril-stake-distributions  mithril-stake-distributions
+download_artifacts $BASE_URL/artifact/mithril-stake-distribution mithril-stake-distribution "hash"
+

mithril-fake-aggregator/scripts/import.sh

@@ -12,7 +12,7 @@ pub async fn epoch_settings(State(state): State<SharedState>) -> Result<String,
     let app_state = state.read().await;
     let epoch_settings = app_state.get_epoch_settings().await?;
 
-    Ok(epoch_settings.into())
+    Ok(epoch_settings)
 }
 
 pub async fn snapshot(

mithril-fake-aggregator/src/handlers.rs

@@ -21,7 +21,7 @@ use tower_http::{
     trace::{DefaultMakeSpan, DefaultOnRequest, DefaultOnResponse, TraceLayer},
     LatencyUnit,
 };
-use tracing::{debug, error, info, trace, Level};
+use tracing::{debug, error, info, trace, warn, Level};
 
 use crate::shared_state::{AppState, SharedState};
 
@@ -105,10 +105,13 @@ impl OsSignalHandler {
         while let Some(signal) = signals.next().await {
             match signal {
                 SIGTERM | SIGINT | SIGQUIT => {
-                    tracing::warn!("Signal caught: {signal}");
+                    warn!("Signal caught: {signal}, exiting.");
 
                     break;
                 }
+                SIGHUP => {
+                    info!("SIGHUP signal caught.");
+                }
                 _ => unreachable!(),
             }
         }
@@ -126,7 +129,7 @@ async fn main() -> StdResult<()> {
 
     match &result {
         Err(e) => error!("{e}"),
-        Ok(_) => trace!("terminated!"),
+        Ok(_) => debug!("soft terminated"),
     };
 
     result
@@ -142,15 +145,21 @@ impl Application {
 
         trace!("setting up signal hook…");
         // supported signals
-        let signals = Signals::new(&[SIGTERM, SIGINT, SIGQUIT])?;
+        let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT, SIGHUP])?;
 
         // launch signal detector
         let signal_handler = signals.handle();
 
         trace!("setting up shared state…");
         let shared_state: SharedState = match params.data_directory {
-            Some(directory) => AppState::from_directory(&directory)?.into(),
-            None => AppState::default().into(),
+            Some(directory) => {
+                info!("Read data files from directory '{}'.", directory.display());
+                AppState::from_directory(&directory)?.into()
+            }
+            None => {
+                debug!("Using default data set.");
+                AppState::default().into()
+            }
         };
 
         trace!("configuring router…");
@@ -190,7 +199,7 @@ impl Application {
             );
         let listener = {
             let connection_string = format!("{}:{}", params.ip_address, params.tcp_port);
-            debug!("binding on {connection_string}");
+            info!("binding on {connection_string}");
             tokio::net::TcpListener::bind(&connection_string)
                 .await
                 .with_context(|| format!("Could not listen on '{}'.", connection_string))?

mithril-fake-aggregator/src/main.rs

@@ -3,6 +3,7 @@ use std::{collections::BTreeMap, path::Path, sync::Arc};
 use anyhow::{anyhow, Context};
 use serde_json::Value;
 use tokio::sync::RwLock;
+use tracing::{debug, trace};
 
 use crate::StdResult;
 
@@ -43,7 +44,7 @@ impl AppState {
         let epoch_settings = default_values::epoch_settings().to_owned();
         let (certificate_list, certificates) = read_files(data_dir, "certificate", "hash")?;
         let (snapshot_list, snapshots) = read_files(data_dir, "snapshot", "digest")?;
-        let (msd_list, msds) = read_files(data_dir, "msd", "hash")?;
+        let (msd_list, msds) = read_files(data_dir, "mithril-stake-distribution", "hash")?;
 
         let instance = Self {
             epoch_settings,
@@ -92,6 +93,8 @@ fn read_files(
     entity: &str,
     field_id: &str,
 ) -> StdResult<(String, BTreeMap<String, String>)> {
+    debug!("Read data files, entity='{entity}', field='{field_id}'.");
+
     if !data_dir.exists() {
         return Err(anyhow!(format!(
             "Path '{}' does not exist.",
@@ -108,8 +111,9 @@ fn read_files(
     let list_file = {
         let list_file_name = format!("{entity}s.json");
 
-        data_dir.to_owned().join(&list_file_name)
+        data_dir.to_owned().join(list_file_name)
     };
+    trace!("Reading JSON list file '{}'.", list_file.display());
     let list = std::fs::read_to_string(&list_file)
         .with_context(|| format!("Error while reading file '{}'.", list_file.display()))?;
     let list_json: Value = serde_json::from_str(&list)
@@ -137,6 +141,7 @@ fn read_files(
     for id in ids {
         let filename = format!("{entity}-{id}.json");
         let path = data_dir.to_owned().join(&filename);
+        trace!("Reading {entity} JSON file '{}'.", path.display());
         let content = std::fs::read_to_string(&path)
             .with_context(|| format!("Could not read entity file '{}'.", path.display()))?;
         let _value: Value = serde_json::from_str(&content).with_context(|| {