Skip to content

Commit 3f12f83

Browse files
authored
Use different log levels for local and remote errors. (#4785)
## Motivation Validators shouldn't produce ERROR-level logs just because a client sent them an invalid request. ## Proposal Distinguish between local and potentially remote errors, and log only the former at level ERROR. ## Test Plan CI Keep an eye on testnet logs: they should have fewer ERRORs now. ## Release Plan - These changes should be backported to `testnet_conway` and: - released in a validator hotfix. ## Links - [reviewer checklist](https://github.com/linera-io/linera-protocol/blob/main/CONTRIBUTING.md#reviewer-checklist)
1 parent 4ee4f79 commit 3f12f83

File tree

6 files changed

+187
-42
lines changed

6 files changed

+187
-42
lines changed

linera-chain/src/lib.rs

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ use linera_base::{
3030
};
3131
use linera_execution::ExecutionError;
3232
use linera_views::ViewError;
33-
use rand_distr::WeightedError;
3433
use thiserror::Error;
3534

3635
#[derive(Error, Debug)]
@@ -144,8 +143,6 @@ pub enum ChainError {
144143
BlockProposalTooLarge(usize),
145144
#[error(transparent)]
146145
BcsError(#[from] bcs::Error),
147-
#[error("Invalid owner weights: {0}")]
148-
OwnerWeightError(#[from] WeightedError),
149146
#[error("Closed chains cannot have operations, accepted messages or empty blocks")]
150147
ClosedChain,
151148
#[error("Empty blocks are not allowed")]
@@ -162,6 +159,51 @@ pub enum ChainError {
162159
NotTimedOutYet(Timestamp),
163160
}
164161

162+
impl ChainError {
163+
/// Returns whether this error is caused by an issue in the local node.
164+
///
165+
/// Returns `false` whenever the error could be caused by a bad message from a peer.
166+
pub fn is_local(&self) -> bool {
167+
match self {
168+
ChainError::CryptoError(_)
169+
| ChainError::ArithmeticError(_)
170+
| ChainError::ViewError(ViewError::NotFound(_))
171+
| ChainError::InactiveChain(_)
172+
| ChainError::IncorrectMessageOrder { .. }
173+
| ChainError::CannotRejectMessage { .. }
174+
| ChainError::CannotSkipMessage { .. }
175+
| ChainError::IncorrectBundleTimestamp { .. }
176+
| ChainError::InvalidSigner
177+
| ChainError::UnexpectedBlockHeight { .. }
178+
| ChainError::UnexpectedPreviousBlockHash
179+
| ChainError::BlockHeightOverflow
180+
| ChainError::InvalidBlockTimestamp { .. }
181+
| ChainError::InsufficientRound(_)
182+
| ChainError::InsufficientRoundStrict(_)
183+
| ChainError::WrongRound(_)
184+
| ChainError::HasIncompatibleConfirmedVote(..)
185+
| ChainError::MustBeNewerThanLockingBlock(..)
186+
| ChainError::MissingEarlierBlocks { .. }
187+
| ChainError::CertificateValidatorReuse
188+
| ChainError::CertificateRequiresQuorum
189+
| ChainError::BlockProposalTooLarge(_)
190+
| ChainError::ClosedChain
191+
| ChainError::EmptyBlock
192+
| ChainError::AuthorizedApplications(_)
193+
| ChainError::MissingMandatoryApplications(_)
194+
| ChainError::MissingOracleResponseList
195+
| ChainError::RoundDoesNotTimeOut
196+
| ChainError::NotTimedOutYet(_)
197+
| ChainError::MissingCrossChainUpdate { .. } => false,
198+
ChainError::ViewError(_)
199+
| ChainError::UnexpectedMessage { .. }
200+
| ChainError::InternalError(_)
201+
| ChainError::BcsError(_) => true,
202+
ChainError::ExecutionError(execution_error, _) => execution_error.is_local(),
203+
}
204+
}
205+
}
206+
165207
#[derive(Copy, Clone, Debug)]
166208
#[cfg_attr(with_testing, derive(Eq, PartialEq))]
167209
pub enum ChainExecutionContext {

linera-core/src/worker.rs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,44 @@ pub enum WorkerError {
250250
},
251251
}
252252

253+
impl WorkerError {
254+
/// Returns whether this error is caused by an issue in the local node.
255+
///
256+
/// Returns `false` whenever the error could be caused by a bad message from a peer.
257+
pub fn is_local(&self) -> bool {
258+
match self {
259+
WorkerError::CryptoError(_)
260+
| WorkerError::ArithmeticError(_)
261+
| WorkerError::InvalidOwner
262+
| WorkerError::InvalidSigner(_)
263+
| WorkerError::UnexpectedBlockHeight { .. }
264+
| WorkerError::InvalidEpoch { .. }
265+
| WorkerError::EventsNotFound(_)
266+
| WorkerError::InvalidBlockChaining
267+
| WorkerError::IncorrectOutcome { .. }
268+
| WorkerError::InvalidTimestamp
269+
| WorkerError::MissingCertificateValue
270+
| WorkerError::InvalidLiteCertificate
271+
| WorkerError::FastBlockUsingOracles
272+
| WorkerError::BlobsNotFound(_)
273+
| WorkerError::InvalidBlockProposal(_)
274+
| WorkerError::UnexpectedBlob
275+
| WorkerError::TooManyPublishedBlobs(_)
276+
| WorkerError::ViewError(ViewError::NotFound(_)) => false,
277+
WorkerError::InvalidCrossChainRequest
278+
| WorkerError::ViewError(_)
279+
| WorkerError::ConfirmedLogEntryNotFound { .. }
280+
| WorkerError::PreprocessedBlocksEntryNotFound { .. }
281+
| WorkerError::JoinError
282+
| WorkerError::MissingNetworkDescription
283+
| WorkerError::ChainActorSendError { .. }
284+
| WorkerError::ChainActorRecvError { .. }
285+
| WorkerError::ReadCertificatesError(_) => true,
286+
WorkerError::ChainError(chain_error) => chain_error.is_local(),
287+
}
288+
}
289+
}
290+
253291
impl From<ChainError> for WorkerError {
254292
#[instrument(level = "trace", skip(chain_error))]
255293
fn from(chain_error: ChainError) -> Self {

linera-execution/src/lib.rs

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,74 @@ pub enum ExecutionError {
322322
OutdatedUpdateStreams,
323323
}
324324

325+
impl ExecutionError {
326+
/// Returns whether this error is caused by an issue in the local node.
327+
///
328+
/// Returns `false` whenever the error could be caused by a bad message from a peer.
329+
pub fn is_local(&self) -> bool {
330+
match self {
331+
ExecutionError::ArithmeticError(_)
332+
| ExecutionError::UserError(_)
333+
| ExecutionError::DecompressionError(_)
334+
| ExecutionError::InvalidPromise
335+
| ExecutionError::CrossApplicationCallInFinalize { .. }
336+
| ExecutionError::ReentrantCall(_)
337+
| ExecutionError::ApplicationBytecodeNotFound(_)
338+
| ExecutionError::UnsupportedDynamicApplicationLoad(_)
339+
| ExecutionError::ExcessiveRead
340+
| ExecutionError::ExcessiveWrite
341+
| ExecutionError::MaximumFuelExceeded(_)
342+
| ExecutionError::MaximumServiceOracleExecutionTimeExceeded
343+
| ExecutionError::ServiceOracleResponseTooLarge
344+
| ExecutionError::BlockTooLarge
345+
| ExecutionError::HttpResponseSizeLimitExceeded { .. }
346+
| ExecutionError::UnauthorizedApplication(_)
347+
| ExecutionError::UnexpectedOracleResponse
348+
| ExecutionError::JsonError(_)
349+
| ExecutionError::BcsError(_)
350+
| ExecutionError::OracleResponseMismatch
351+
| ExecutionError::ServiceOracleQueryOperations(_)
352+
| ExecutionError::AssertBefore { .. }
353+
| ExecutionError::StreamNameTooLong
354+
| ExecutionError::BlobTooLarge
355+
| ExecutionError::BytecodeTooLarge
356+
| ExecutionError::UnauthorizedHttpRequest(_)
357+
| ExecutionError::InvalidUrlForHttpRequest(_)
358+
| ExecutionError::InactiveChain(_)
359+
| ExecutionError::BlobsNotFound(_)
360+
| ExecutionError::EventsNotFound(_)
361+
| ExecutionError::InvalidHeaderName(_)
362+
| ExecutionError::InvalidHeaderValue(_)
363+
| ExecutionError::InvalidEpoch { .. }
364+
| ExecutionError::IncorrectTransferAmount
365+
| ExecutionError::UnauthenticatedTransferOwner
366+
| ExecutionError::InsufficientBalance { .. }
367+
| ExecutionError::FeesExceedFunding { .. }
368+
| ExecutionError::IncorrectClaimAmount
369+
| ExecutionError::UnauthenticatedClaimOwner
370+
| ExecutionError::AdminOperationOnNonAdminChain
371+
| ExecutionError::InvalidCommitteeEpoch { .. }
372+
| ExecutionError::InvalidCommitteeRemoval
373+
| ExecutionError::MissingOracleResponse
374+
| ExecutionError::UnprocessedStreams
375+
| ExecutionError::OutdatedUpdateStreams
376+
| ExecutionError::ViewError(ViewError::NotFound(_)) => false,
377+
#[cfg(with_wasm_runtime)]
378+
ExecutionError::WasmError(_) => false,
379+
#[cfg(with_revm)]
380+
ExecutionError::EvmError(..) => false,
381+
ExecutionError::MissingRuntimeResponse
382+
| ExecutionError::ViewError(_)
383+
| ExecutionError::ReqwestError(_)
384+
| ExecutionError::ContractModuleSend(_)
385+
| ExecutionError::ServiceModuleSend(_)
386+
| ExecutionError::NoNetworkDescriptionFound
387+
| ExecutionError::InternalError(_)
388+
| ExecutionError::IoError(_) => true,
389+
}
390+
}
391+
}
392+
325393
/// The public entry points provided by the contract part of an application.
326394
pub trait UserContract {
327395
/// Instantiate the application state on the chain that owns the application.

linera-rpc/src/grpc/server.rs

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use linera_base::{
1616
use linera_core::{
1717
join_set_ext::JoinSet,
1818
node::NodeError,
19-
worker::{NetworkActions, Notification, Reason, WorkerError, WorkerState},
19+
worker::{NetworkActions, Notification, Reason, WorkerState},
2020
JoinSetExt as _, TaskHandle,
2121
};
2222
use linera_storage::Storage;
@@ -429,6 +429,15 @@ where
429429
}
430430
}
431431
}
432+
433+
fn log_error(&self, error: &linera_core::worker::WorkerError, context: &str) {
434+
let nickname = self.state.nickname();
435+
if error.is_local() {
436+
error!(nickname, %error, "{}", context);
437+
} else {
438+
debug!(nickname, %error, "{}", context);
439+
}
440+
}
432441
}
433442

434443
#[tonic::async_trait]
@@ -508,12 +517,7 @@ where
508517
}
509518
Err(error) => {
510519
Self::log_request_outcome_and_latency(start, false, "handle_lite_certificate");
511-
let nickname = self.state.nickname();
512-
if let WorkerError::MissingCertificateValue = &error {
513-
debug!(nickname, %error, "Failed to handle lite certificate");
514-
} else {
515-
error!(nickname, %error, "Failed to handle lite certificate");
516-
}
520+
self.log_error(&error, "Failed to handle lite certificate");
517521
Ok(Response::new(NodeError::from(error).try_into()?))
518522
}
519523
}
@@ -557,8 +561,7 @@ where
557561
}
558562
Err(error) => {
559563
Self::log_request_outcome_and_latency(start, false, "handle_confirmed_certificate");
560-
let nickname = self.state.nickname();
561-
error!(nickname, %error, "Failed to handle confirmed certificate");
564+
self.log_error(&error, "Failed to handle confirmed certificate");
562565
Ok(Response::new(NodeError::from(error).try_into()?))
563566
}
564567
}
@@ -593,8 +596,7 @@ where
593596
}
594597
Err(error) => {
595598
Self::log_request_outcome_and_latency(start, false, "handle_validated_certificate");
596-
let nickname = self.state.nickname();
597-
error!(nickname, %error, "Failed to handle validated certificate");
599+
self.log_error(&error, "Failed to handle validated certificate");
598600
Ok(Response::new(NodeError::from(error).try_into()?))
599601
}
600602
}
@@ -628,8 +630,7 @@ where
628630
}
629631
Err(error) => {
630632
Self::log_request_outcome_and_latency(start, false, "handle_timeout_certificate");
631-
let nickname = self.state.nickname();
632-
error!(nickname, %error, "Failed to handle timeout certificate");
633+
self.log_error(&error, "Failed to handle timeout certificate");
633634
Ok(Response::new(NodeError::from(error).try_into()?))
634635
}
635636
}
@@ -659,8 +660,7 @@ where
659660
}
660661
Err(error) => {
661662
Self::log_request_outcome_and_latency(start, false, "handle_chain_info_query");
662-
let nickname = self.state.nickname();
663-
error!(nickname, %error, "Failed to handle chain info query");
663+
self.log_error(&error, "Failed to handle chain info query");
664664
Ok(Response::new(NodeError::from(error).try_into()?))
665665
}
666666
}
@@ -694,8 +694,7 @@ where
694694
}
695695
Err(error) => {
696696
Self::log_request_outcome_and_latency(start, false, "download_pending_blob");
697-
let nickname = self.state.nickname();
698-
error!(nickname, %error, "Failed to download pending blob");
697+
self.log_error(&error, "Failed to download pending blob");
699698
Ok(Response::new(NodeError::from(error).try_into()?))
700699
}
701700
}
@@ -726,8 +725,7 @@ where
726725
}
727726
Err(error) => {
728727
Self::log_request_outcome_and_latency(start, false, "handle_pending_blob");
729-
let nickname = self.state.nickname();
730-
error!(nickname, %error, "Failed to handle pending blob");
728+
self.log_error(&error, "Failed to handle pending blob");
731729
Ok(Response::new(NodeError::from(error).try_into()?))
732730
}
733731
}

linera-rpc/src/simple/server.rs

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use linera_core::{
1515
use linera_storage::Storage;
1616
use tokio::{sync::oneshot, task::JoinSet};
1717
use tokio_util::sync::CancellationToken;
18-
use tracing::{debug, error, info, instrument, warn};
18+
use tracing::{debug, error, info, instrument};
1919

2020
use super::transport::{MessageHandler, ServerHandle, TransportProtocol};
2121
use crate::{
@@ -188,8 +188,7 @@ where
188188
Ok(Some(RpcMessage::ChainInfoResponse(Box::new(info))))
189189
}
190190
Err(error) => {
191-
let nickname = self.server.state.nickname();
192-
warn!(nickname, %error, "Failed to handle block proposal");
191+
self.log_error(&error, "Failed to handle block proposal");
193192
Err(error.into())
194193
}
195194
}
@@ -242,8 +241,7 @@ where
242241
Ok(Some(RpcMessage::ChainInfoResponse(Box::new(info))))
243242
}
244243
Err(error) => {
245-
let nickname = self.server.state.nickname();
246-
error!(nickname, %error, "Failed to handle timeout certificate");
244+
self.log_error(&error, "Failed to handle timeout certificate");
247245
Err(error.into())
248246
}
249247
}
@@ -262,10 +260,7 @@ where
262260
Ok(Some(RpcMessage::ChainInfoResponse(Box::new(info))))
263261
}
264262
Err(error) => {
265-
error!(
266-
nickname = self.server.state.nickname(), %error,
267-
"Failed to handle validated certificate"
268-
);
263+
self.log_error(&error, "Failed to handle validated certificate");
269264
Err(error.into())
270265
}
271266
}
@@ -293,8 +288,7 @@ where
293288
Ok(Some(RpcMessage::ChainInfoResponse(Box::new(info))))
294289
}
295290
Err(error) => {
296-
let nickname = self.server.state.nickname();
297-
error!(nickname, %error, "Failed to handle confirmed certificate");
291+
self.log_error(&error, "Failed to handle confirmed certificate");
298292
Err(error.into())
299293
}
300294
}
@@ -308,8 +302,7 @@ where
308302
Ok(Some(RpcMessage::ChainInfoResponse(Box::new(info))))
309303
}
310304
Err(error) => {
311-
let nickname = self.server.state.nickname();
312-
error!(nickname, %error, "Failed to handle chain info query");
305+
self.log_error(&error, "Failed to handle chain info query");
313306
Err(error.into())
314307
}
315308
}
@@ -320,8 +313,7 @@ where
320313
self.handle_network_actions(actions);
321314
}
322315
Err(error) => {
323-
let nickname = self.server.state.nickname();
324-
error!(nickname, %error, "Failed to handle cross-chain request");
316+
self.log_error(&error, "Failed to handle cross-chain request");
325317
}
326318
}
327319
// No user to respond to.
@@ -339,8 +331,7 @@ where
339331
blob.into(),
340332
)))),
341333
Err(error) => {
342-
let nickname = self.server.state.nickname();
343-
error!(nickname, %error, "Failed to handle pending blob request");
334+
self.log_error(&error, "Failed to handle pending blob request");
344335
Err(error.into())
345336
}
346337
}
@@ -355,8 +346,7 @@ where
355346
{
356347
Ok(info) => Ok(Some(RpcMessage::ChainInfoResponse(Box::new(info)))),
357348
Err(error) => {
358-
let nickname = self.server.state.nickname();
359-
error!(nickname, %error, "Failed to handle pending blob");
349+
self.log_error(&error, "Failed to handle pending blob");
360350
Err(error.into())
361351
}
362352
}
@@ -442,4 +432,13 @@ where
442432
}
443433
}
444434
}
435+
436+
fn log_error(&self, error: &WorkerError, context: &str) {
437+
let nickname = self.server.state.nickname();
438+
if error.is_local() {
439+
error!(nickname, %error, "{}", context);
440+
} else {
441+
debug!(nickname, %error, "{}", context);
442+
}
443+
}
445444
}

0 commit comments

Comments
 (0)