Skip to content

Commit 89c5fa7

Browse files
winters.zceryugey
authored andcommitted
fix: low probability IO hang problem
we found some applications may encounter the io hang: Call Trace: [<ffffffff8173ca3b>] ? __schedule+0x23b/0x780 [<ffffffff8173cfb6>] schedule+0x36/0x80 [<ffffffffa0377840>] request_wait_answer+0xc0/0x1f0 [fuse] [<ffffffff810d5350>] ? prepare_to_wait_event+0x100/0x100 [<ffffffffa03779f4>] __fuse_request_send+0x84/0x90 [fuse] [<ffffffffa0377a27>] fuse_request_send+0x27/0x30 [fuse] [<ffffffffa037adaf>] fuse_simple_request+0xcf/0x1a0 [fuse] [<ffffffffa037c80b>] fuse_dentry_revalidate+0x18b/0x300 [fuse] [<ffffffff8125b79b>] lookup_fast+0x2eb/0x310 [<ffffffff8125c527>] walk_component+0x47/0x310 [<ffffffff8125d7c7>] path_lookupat+0x67/0x120 [<ffffffff811b6e99>] ? generic_file_read_iter+0x699/0x960 [<ffffffff81260421>] filename_lookup+0xb1/0x180 [<ffffffff812235c6>] ? kmem_cache_alloc+0x146/0x1a0 [<ffffffff8126001f>] ? getname_flags+0x4f/0x1f0 [<ffffffff8126003f>] ? getname_flags+0x6f/0x1f0 [<ffffffff81254896>] vfs_fstatat+0x66/0xc0 [<ffffffff81254e51>] SYSC_newlstat+0x31/0x60 The root cause MAYBE that fuse daemon encounters errors (bytes_to_cstr() error) when processing requests, but does not reply to kernel. This patch fixes the issue by adding a reply. Because the exception does not produce any logs, we cannot be 100% sure that it is the cause. However, after applying this patch in our production environment, the issue no longer occurs. Signed-off-by: winters.zc <[email protected]>
1 parent d62776c commit 89c5fa7

File tree

2 files changed

+77
-18
lines changed

2 files changed

+77
-18
lines changed

src/api/server/async_io.rs

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,17 @@ impl<F: AsyncFileSystem + Sync> Server<F> {
212212

213213
async fn async_lookup<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
214214
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, 0)?;
215-
let name = bytes_to_cstr(buf.as_ref())?;
215+
let name = match bytes_to_cstr(buf.as_ref()) {
216+
Ok(name) => name,
217+
Err(e) => {
218+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
219+
let _ = ctx
220+
.async_reply_error(io::Error::from_raw_os_error(libc::EINVAL))
221+
.await;
222+
return Err(e);
223+
}
224+
};
225+
216226
let version = self.vers.load();
217227
let result = self
218228
.fs
@@ -443,7 +453,17 @@ impl<F: AsyncFileSystem + Sync> Server<F> {
443453
async fn async_create<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
444454
let args: CreateIn = ctx.r.read_obj().map_err(Error::DecodeMessage)?;
445455
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, size_of::<CreateIn>())?;
446-
let name = bytes_to_cstr(&buf)?;
456+
let name = match bytes_to_cstr(buf.as_ref()) {
457+
Ok(name) => name,
458+
Err(e) => {
459+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
460+
let _ = ctx
461+
.async_reply_error(io::Error::from_raw_os_error(libc::EINVAL))
462+
.await;
463+
return Err(e);
464+
}
465+
};
466+
447467
let result = self
448468
.fs
449469
.async_create(ctx.context(), ctx.nodeid(), name, args)

src/api/server/sync_io.rs

Lines changed: 55 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,12 @@ impl<F: FileSystem + Sync> Server<F> {
129129

130130
fn lookup<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
131131
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, 0)?;
132-
let name = bytes_to_cstr(buf.as_ref())?;
132+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
133+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
134+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
135+
e
136+
})?;
137+
133138
let version = self.vers.load();
134139
let result = self.fs.lookup(ctx.context(), ctx.nodeid(), name);
135140

@@ -213,7 +218,11 @@ impl<F: FileSystem + Sync> Server<F> {
213218
mode, rdev, umask, ..
214219
} = ctx.r.read_obj().map_err(Error::DecodeMessage)?;
215220
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, size_of::<MknodIn>())?;
216-
let name = bytes_to_cstr(buf.as_ref())?;
221+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
222+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
223+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
224+
e
225+
})?;
217226

218227
match self
219228
.fs
@@ -227,7 +236,11 @@ impl<F: FileSystem + Sync> Server<F> {
227236
pub(super) fn mkdir<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
228237
let MkdirIn { mode, umask } = ctx.r.read_obj().map_err(Error::DecodeMessage)?;
229238
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, size_of::<MkdirIn>())?;
230-
let name = bytes_to_cstr(buf.as_ref())?;
239+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
240+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
241+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
242+
e
243+
})?;
231244

232245
match self
233246
.fs
@@ -240,7 +253,11 @@ impl<F: FileSystem + Sync> Server<F> {
240253

241254
pub(super) fn unlink<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
242255
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, 0)?;
243-
let name = bytes_to_cstr(buf.as_ref())?;
256+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
257+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
258+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
259+
e
260+
})?;
244261

245262
match self.fs.unlink(ctx.context(), ctx.nodeid(), name) {
246263
Ok(()) => ctx.reply_ok(None::<u8>, None),
@@ -250,7 +267,11 @@ impl<F: FileSystem + Sync> Server<F> {
250267

251268
pub(super) fn rmdir<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
252269
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, 0)?;
253-
let name = bytes_to_cstr(buf.as_ref())?;
270+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
271+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
272+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
273+
e
274+
})?;
254275

255276
match self.fs.rmdir(ctx.context(), ctx.nodeid(), name) {
256277
Ok(()) => ctx.reply_ok(None::<u8>, None),
@@ -303,7 +324,11 @@ impl<F: FileSystem + Sync> Server<F> {
303324
pub(super) fn link<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
304325
let LinkIn { oldnodeid } = ctx.r.read_obj().map_err(Error::DecodeMessage)?;
305326
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, size_of::<LinkIn>())?;
306-
let name = bytes_to_cstr(buf.as_ref())?;
327+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
328+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
329+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
330+
e
331+
})?;
307332

308333
match self
309334
.fs
@@ -507,14 +532,16 @@ impl<F: FileSystem + Sync> Server<F> {
507532
if size != value.len() as u32 {
508533
return Err(Error::InvalidXattrSize((size, value.len())));
509534
}
535+
let name = bytes_to_cstr(name).map_err(|e| {
536+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
537+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
538+
e
539+
})?;
510540

511-
match self.fs.setxattr(
512-
ctx.context(),
513-
ctx.nodeid(),
514-
bytes_to_cstr(name)?,
515-
value,
516-
flags,
517-
) {
541+
match self
542+
.fs
543+
.setxattr(ctx.context(), ctx.nodeid(), name, value, flags)
544+
{
518545
Ok(()) => ctx.reply_ok(None::<u8>, None),
519546
Err(e) => ctx.reply_error(e),
520547
}
@@ -528,7 +555,11 @@ impl<F: FileSystem + Sync> Server<F> {
528555

529556
let buf =
530557
ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, size_of::<GetxattrIn>())?;
531-
let name = bytes_to_cstr(buf.as_ref())?;
558+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
559+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
560+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
561+
e
562+
})?;
532563

533564
match self.fs.getxattr(ctx.context(), ctx.nodeid(), name, size) {
534565
Ok(GetxattrReply::Value(val)) => ctx.reply_ok(None::<u8>, Some(&val)),
@@ -570,7 +601,11 @@ impl<F: FileSystem + Sync> Server<F> {
570601
mut ctx: SrvContext<'_, F, S>,
571602
) -> Result<usize> {
572603
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, 0)?;
573-
let name = bytes_to_cstr(&buf)?;
604+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
605+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
606+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
607+
e
608+
})?;
574609

575610
match self.fs.removexattr(ctx.context(), ctx.nodeid(), name) {
576611
Ok(()) => ctx.reply_ok(None::<u8>, None),
@@ -878,7 +913,11 @@ impl<F: FileSystem + Sync> Server<F> {
878913
fn create<S: BitmapSlice>(&self, mut ctx: SrvContext<'_, F, S>) -> Result<usize> {
879914
let args: CreateIn = ctx.r.read_obj().map_err(Error::DecodeMessage)?;
880915
let buf = ServerUtil::get_message_body(&mut ctx.r, &ctx.in_header, size_of::<CreateIn>())?;
881-
let name = bytes_to_cstr(&buf)?;
916+
let name = bytes_to_cstr(buf.as_ref()).map_err(|e| {
917+
let _ = ctx.reply_error_explicit(io::Error::from_raw_os_error(libc::EINVAL));
918+
error!("fuse: bytes to cstr error: {:?}, {:?}", buf, e);
919+
e
920+
})?;
882921

883922
match self.fs.create(ctx.context(), ctx.nodeid(), name, args) {
884923
Ok((entry, handle, opts)) => {

0 commit comments

Comments
 (0)