Skip to content

Commit 22ee912

Browse files
tizoctizoc
committed
tmp
1 parent 3a2c907 commit 22ee912

File tree

9 files changed

+200
-25
lines changed

9 files changed

+200
-25
lines changed

‎node/common/src/service/block_producer/mod.rs

Lines changed: 119 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
mod vrf_evaluator;
22

3-
use std::sync::Arc;
3+
use std::{io::Write, sync::Arc};
44

55
use ledger::proofs::{
66
block::BlockParams, generate_block_proof, provers::BlockProver,
@@ -107,11 +107,27 @@ fn prover_loop(
107107
let res = prove(provers, &mut input, &keypair, false);
108108
if let Err(error) = &res {
109109
openmina_core::error!(message = "Block proof failed", error = format!("{error:?}"));
110-
if let Err(error) = dump_failed_block_proof_input(block_hash.clone(), input, error) {
111-
openmina_core::error!(
112-
message = "Failure when dumping failed block proof inputs",
113-
error = format!("{error}")
114-
);
110+
let submission_url = std::env::var("OPENMINA_ERROR_SINK_SERVICE_URL").ok();
111+
if let Some(submission_url) = submission_url {
112+
if let Err(error) = submit_failed_block_proof_input(
113+
block_hash.clone(),
114+
input,
115+
error,
116+
&submission_url,
117+
) {
118+
openmina_core::error!(
119+
message = "Failed to submit failed block proof",
120+
error = format!("{error}")
121+
);
122+
}
123+
} else {
124+
if let Err(error) = dump_failed_block_proof_input(block_hash.clone(), input, error)
125+
{
126+
openmina_core::error!(
127+
message = "Failure when dumping failed block proof inputs",
128+
error = format!("{error}")
129+
);
130+
}
115131
}
116132
}
117133
let res = res.map_err(|err| err.to_string());
@@ -181,11 +197,20 @@ impl node::service::BlockProducerService for crate::NodeService {
181197
}
182198
}
183199

184-
fn dump_failed_block_proof_input(
200+
/// Represents the destination for failed block proof data
201+
pub enum BlockProofOutputDestination {
202+
/// Save the proof data to a file in the debug directory
203+
FilesystemDump,
204+
/// Submit the proof data to an external service via HTTP
205+
ErrorService(String),
206+
}
207+
208+
fn handle_failed_block_proof_input(
185209
block_hash: StateHash,
186210
mut input: Box<ProverExtendBlockchainInputStableV2>,
187211
error: &anyhow::Error,
188-
) -> std::io::Result<()> {
212+
destination: BlockProofOutputDestination,
213+
) -> anyhow::Result<()> {
189214
use ledger::proofs::transaction::ProofError;
190215
use rsa::Pkcs1v15Encrypt;
191216

@@ -229,7 +254,7 @@ kGqG7QLzSPjAtP/YbUponwaD+t+A0kBg0hV4hhcJOkPeA2NOi04K93bz3HuYCVRe
229254

230255
let error_str = error.to_string();
231256

232-
let input = DumpBlockProof {
257+
let input_data = DumpBlockProof {
233258
input,
234259
key: encrypted_producer_private_key,
235260
error: error_str.as_bytes().to_vec(),
@@ -239,15 +264,89 @@ kGqG7QLzSPjAtP/YbUponwaD+t+A0kBg0hV4hhcJOkPeA2NOi04K93bz3HuYCVRe
239264
},
240265
};
241266

242-
let debug_dir = openmina_core::get_debug_dir();
243-
let filename = debug_dir
244-
.join(format!("failed_block_proof_input_{block_hash}.binprot"))
245-
.to_string_lossy()
246-
.to_string();
247-
openmina_core::warn!(message = "Dumping failed block proof.", filename = filename);
248-
std::fs::create_dir_all(&debug_dir)?;
249-
let mut file = std::fs::File::create(&filename)?;
250-
input.binprot_write(&mut file)?;
251-
file.sync_all()?;
252-
Ok(())
267+
// Serialize the data
268+
let mut buffer = Vec::new();
269+
input_data.binprot_write(&mut buffer)?;
270+
271+
// Handle the data according to the destination
272+
match destination {
273+
BlockProofOutputDestination::FilesystemDump => {
274+
let debug_dir = openmina_core::get_debug_dir();
275+
let filename = debug_dir
276+
.join(format!("failed_block_proof_input_{block_hash}.binprot"))
277+
.to_string_lossy()
278+
.to_string();
279+
openmina_core::warn!(message = "Dumping failed block proof.", filename = filename);
280+
std::fs::create_dir_all(&debug_dir)?;
281+
let mut file = std::fs::File::create(&filename)?;
282+
file.write_all(&buffer)?;
283+
file.sync_all()?;
284+
Ok(())
285+
}
286+
BlockProofOutputDestination::ErrorService(url) => {
287+
use reqwest::blocking::Client;
288+
289+
openmina_core::warn!(
290+
message = "Submitting failed block proof to external service.",
291+
block_hash = format!("{block_hash}"),
292+
url = url
293+
);
294+
295+
let client = Client::new();
296+
let response = client
297+
.post(&url)
298+
.header("Content-Type", "application/octet-stream")
299+
.body(buffer)
300+
.send()?;
301+
302+
// Check if the request was successful
303+
if response.status().is_success() {
304+
openmina_core::info!(
305+
message = "Successfully submitted failed block proof.",
306+
block_hash = format!("{block_hash}"),
307+
status = response.status().as_u16()
308+
);
309+
Ok(())
310+
} else {
311+
let error_message = format!(
312+
"Failed to submit block proof: HTTP error {}",
313+
response.status()
314+
);
315+
openmina_core::error!(
316+
message = "Failed to submit block proof",
317+
block_hash = format!("{block_hash}"),
318+
status = response.status().as_u16()
319+
);
320+
Err(anyhow::anyhow!(error_message))
321+
}
322+
}
323+
}
324+
}
325+
326+
fn dump_failed_block_proof_input(
327+
block_hash: StateHash,
328+
input: Box<ProverExtendBlockchainInputStableV2>,
329+
error: &anyhow::Error,
330+
) -> std::io::Result<()> {
331+
handle_failed_block_proof_input(
332+
block_hash,
333+
input,
334+
error,
335+
BlockProofOutputDestination::FilesystemDump,
336+
)
337+
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
338+
}
339+
340+
pub fn submit_failed_block_proof_input(
341+
block_hash: StateHash,
342+
input: Box<ProverExtendBlockchainInputStableV2>,
343+
error: &anyhow::Error,
344+
submission_url: &str,
345+
) -> anyhow::Result<()> {
346+
handle_failed_block_proof_input(
347+
block_hash,
348+
input,
349+
error,
350+
BlockProofOutputDestination::ErrorService(submission_url.to_string()),
351+
)
253352
}

‎node/src/action_kind.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ pub enum ActionKind {
112112
BlockProducerBlockInject,
113113
BlockProducerBlockInjected,
114114
BlockProducerBlockProduced,
115+
BlockProducerBlockProveError,
115116
BlockProducerBlockProveInit,
116117
BlockProducerBlockProvePending,
117118
BlockProducerBlockProveSuccess,
@@ -127,6 +128,7 @@ pub enum ActionKind {
127128
BlockProducerWonSlotTransactionsSuccess,
128129
BlockProducerWonSlotWait,
129130
BlockProducerEffectfulBlockProduced,
131+
BlockProducerEffectfulBlockProveError,
130132
BlockProducerEffectfulBlockProveInit,
131133
BlockProducerEffectfulBlockProveSuccess,
132134
BlockProducerEffectfulBlockUnprovenBuild,
@@ -738,7 +740,7 @@ pub enum ActionKind {
738740
}
739741

740742
impl ActionKind {
741-
pub const COUNT: u16 = 628;
743+
pub const COUNT: u16 = 630;
742744
}
743745

744746
impl std::fmt::Display for ActionKind {
@@ -1016,6 +1018,7 @@ impl ActionKindGet for BlockProducerAction {
10161018
Self::BlockProveInit => ActionKind::BlockProducerBlockProveInit,
10171019
Self::BlockProvePending => ActionKind::BlockProducerBlockProvePending,
10181020
Self::BlockProveSuccess { .. } => ActionKind::BlockProducerBlockProveSuccess,
1021+
Self::BlockProveError { .. } => ActionKind::BlockProducerBlockProveError,
10191022
Self::BlockProduced => ActionKind::BlockProducerBlockProduced,
10201023
Self::BlockInject => ActionKind::BlockProducerBlockInject,
10211024
Self::BlockInjected => ActionKind::BlockProducerBlockInjected,
@@ -1038,6 +1041,7 @@ impl ActionKindGet for BlockProducerEffectfulAction {
10381041
Self::BlockUnprovenBuild => ActionKind::BlockProducerEffectfulBlockUnprovenBuild,
10391042
Self::BlockProveInit => ActionKind::BlockProducerEffectfulBlockProveInit,
10401043
Self::BlockProveSuccess => ActionKind::BlockProducerEffectfulBlockProveSuccess,
1044+
Self::BlockProveError { .. } => ActionKind::BlockProducerEffectfulBlockProveError,
10411045
Self::BlockProduced { .. } => ActionKind::BlockProducerEffectfulBlockProduced,
10421046
}
10431047
}

‎node/src/block_producer/block_producer_actions.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ pub enum BlockProducerAction {
6161
BlockProveSuccess {
6262
proof: Arc<MinaBaseProofStableV2>,
6363
},
64+
BlockProveError {
65+
error: String,
66+
},
6467
BlockProduced,
6568
#[action_event(level = trace)]
6669
BlockInject,
@@ -183,7 +186,8 @@ impl redux::EnablingCondition<crate::State> for BlockProducerAction {
183186
BlockProducerCurrentState::BlockUnprovenBuilt { .. }
184187
)
185188
}),
186-
BlockProducerAction::BlockProveSuccess { .. } => {
189+
BlockProducerAction::BlockProveSuccess { .. }
190+
| BlockProducerAction::BlockProveError { .. } => {
187191
state.block_producer.with(false, |this| {
188192
matches!(
189193
this.current,

‎node/src/block_producer/block_producer_reducer.rs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,34 @@ impl BlockProducerEnabled {
283283
let dispatcher = state_context.into_dispatcher();
284284
dispatcher.push(BlockProducerEffectfulAction::BlockProveSuccess);
285285
}
286+
BlockProducerAction::BlockProveError { error } => {
287+
let current_state = std::mem::take(&mut state.current);
288+
289+
if let BlockProducerCurrentState::BlockProvePending {
290+
won_slot,
291+
chain,
292+
block,
293+
block_hash,
294+
..
295+
} = current_state
296+
{
297+
state.current = BlockProducerCurrentState::BlockProveError {
298+
time: meta.time(),
299+
won_slot,
300+
chain,
301+
block,
302+
block_hash,
303+
error: error.clone(),
304+
};
305+
} else {
306+
bug_condition!("Invalid state for `BlockProducerAction::BlockProveError` expected: `BlockProducerCurrentState::BlockProvePending`, found: {:?}", current_state);
307+
}
308+
309+
let dispatcher = state_context.into_dispatcher();
310+
dispatcher.push(BlockProducerEffectfulAction::BlockProveError {
311+
error: error.clone(),
312+
});
313+
}
286314
BlockProducerAction::BlockProduced => {
287315
let current_state = std::mem::take(&mut state.current);
288316

‎node/src/block_producer/block_producer_state.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,15 @@ pub enum BlockProducerCurrentState {
119119
block_hash: v2::StateHash,
120120
proof: Arc<v2::MinaBaseProofStableV2>,
121121
},
122+
BlockProveError {
123+
time: redux::Timestamp,
124+
won_slot: BlockProducerWonSlot,
125+
/// Chain that we are extending.
126+
chain: Vec<AppliedBlock>,
127+
block: BlockWithoutProof,
128+
block_hash: v2::StateHash,
129+
error: String,
130+
},
122131
Produced {
123132
time: redux::Timestamp,
124133
won_slot: BlockProducerWonSlot,
@@ -245,6 +254,7 @@ impl BlockProducerCurrentState {
245254
| Self::BlockUnprovenBuilt { .. }
246255
| Self::BlockProvePending { .. }
247256
| Self::BlockProveSuccess { .. }
257+
| Self::BlockProveError { .. }
248258
| Self::Produced { .. } => false,
249259
}
250260
}
@@ -263,6 +273,7 @@ impl BlockProducerCurrentState {
263273
| Self::BlockUnprovenBuilt { won_slot, .. }
264274
| Self::BlockProvePending { won_slot, .. }
265275
| Self::BlockProveSuccess { won_slot, .. }
276+
| Self::BlockProveError { won_slot, .. }
266277
| Self::Produced { won_slot, .. }
267278
| Self::Injected { won_slot, .. } => Some(won_slot),
268279
}
@@ -282,6 +293,7 @@ impl BlockProducerCurrentState {
282293
| Self::BlockUnprovenBuilt { chain, .. }
283294
| Self::BlockProvePending { chain, .. }
284295
| Self::BlockProveSuccess { chain, .. }
296+
| Self::BlockProveError { chain, .. }
285297
| Self::Produced { chain, .. }
286298
| Self::Injected { chain, .. } => Some(chain),
287299
}
@@ -352,6 +364,7 @@ impl BlockProducerCurrentState {
352364
| Self::WonSlotDiscarded { .. }
353365
| Self::WonSlot { .. }
354366
| Self::WonSlotWait { .. }
367+
| Self::BlockProveError { .. }
355368
| Self::Injected { .. } => false,
356369
Self::WonSlotProduceInit { .. }
357370
| Self::WonSlotTransactionsGet { .. }

‎node/src/block_producer_effectful/block_producer_effectful_actions.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ pub enum BlockProducerEffectfulAction {
1717
BlockUnprovenBuild,
1818
BlockProveInit,
1919
BlockProveSuccess,
20+
BlockProveError {
21+
error: String,
22+
},
2023
BlockProduced {
2124
block: ArcBlockWithHash,
2225
},

‎node/src/block_producer_effectful/block_producer_effectful_effects.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,14 @@ pub fn block_producer_effects<S: crate::Service>(
215215
}
216216
store.dispatch(BlockProducerAction::BlockProduced);
217217
}
218+
BlockProducerEffectfulAction::BlockProveError { error } => {
219+
if let Some(stats) = store.service.stats() {
220+
stats
221+
.block_producer()
222+
.proof_create_error(meta.time(), error.clone());
223+
}
224+
store.dispatch(BlockProducerAction::BlockProveError { error });
225+
}
218226
BlockProducerEffectfulAction::WonSlotDiscard { reason } => {
219227
if let Some(stats) = store.service.stats() {
220228
stats.block_producer().discarded(meta.time(), reason);

‎node/src/event_source/event_source_effects.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -458,9 +458,9 @@ pub fn event_source_effects<S: Service>(store: &mut Store<S>, action: EventSourc
458458
}
459459
},
460460
BlockProducerEvent::BlockProve(block_hash, res) => match res {
461-
Err(err) => todo!(
462-
"error while trying to produce block proof for block {block_hash} - {err}"
463-
),
461+
Err(error) => {
462+
store.dispatch(BlockProducerAction::BlockProveError { error });
463+
}
464464
Ok(proof) => {
465465
if store
466466
.state()

‎node/src/stats/stats_block_producer.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ pub struct BlockProductionTimes {
4545
pub produced: Option<redux::Timestamp>,
4646
pub proof_create_start: Option<redux::Timestamp>,
4747
pub proof_create_end: Option<redux::Timestamp>,
48+
pub proof_create_error: Option<redux::Timestamp>,
4849
pub block_apply_start: Option<redux::Timestamp>,
4950
pub block_apply_end: Option<redux::Timestamp>,
5051
pub committed: Option<redux::Timestamp>,
@@ -60,6 +61,9 @@ pub enum BlockProductionStatus {
6061
Produced,
6162
ProofCreatePending,
6263
ProofCreateSuccess,
64+
ProofCreateError {
65+
error: String,
66+
},
6367
BlockApplyPending,
6468
BlockApplySuccess,
6569
Committed,
@@ -201,6 +205,7 @@ impl BlockProducerStats {
201205
produced: None,
202206
proof_create_start: None,
203207
proof_create_end: None,
208+
proof_create_error: None,
204209
block_apply_start: None,
205210
block_apply_end: None,
206211
committed: None,
@@ -277,6 +282,17 @@ impl BlockProducerStats {
277282
});
278283
}
279284

285+
pub fn proof_create_error(&mut self, time: redux::Timestamp, error: String) {
286+
self.update("proof_create_error", move |attempt| match attempt.status {
287+
BlockProductionStatus::ProofCreatePending => {
288+
attempt.status = BlockProductionStatus::ProofCreateError { error };
289+
attempt.times.proof_create_error = Some(time);
290+
true
291+
}
292+
_ => false,
293+
});
294+
}
295+
280296
pub fn block_apply_start(&mut self, time: redux::Timestamp, hash: &BlockHash) {
281297
if !self.is_our_just_produced_block(hash) {
282298
return;

0 commit comments

Comments
 (0)