diff --git a/jupyter_server/services/contents/filemanager.py b/jupyter_server/services/contents/filemanager.py
index 96029d96d..e1d2ae1ae 100644
--- a/jupyter_server/services/contents/filemanager.py
+++ b/jupyter_server/services/contents/filemanager.py
@@ -602,6 +602,8 @@ def rename_file(self, old_path, new_path):
                 shutil.move(old_os_path, new_os_path)
         except web.HTTPError:
             raise
+        except FileNotFoundError:
+            raise web.HTTPError(404, f"File or directory does not exist: {old_path}") from None
         except Exception as e:
             raise web.HTTPError(500, f"Unknown error renaming file: {old_path} {e}") from e
 
@@ -1069,6 +1071,8 @@ async def rename_file(self, old_path, new_path):
                 await run_sync(shutil.move, old_os_path, new_os_path)
         except web.HTTPError:
             raise
+        except FileNotFoundError:
+            raise web.HTTPError(404, f"File or directory does not exist: {old_path}") from None
         except Exception as e:
             raise web.HTTPError(500, f"Unknown error renaming file: {old_path} {e}") from e
 
diff --git a/tests/services/contents/test_manager.py b/tests/services/contents/test_manager.py
index 5c3e2eca5..ef8d40a28 100644
--- a/tests/services/contents/test_manager.py
+++ b/tests/services/contents/test_manager.py
@@ -854,6 +854,21 @@ async def test_rename(jp_contents_manager):
     await ensure_async(cm.new_untitled("foo/bar_diff", ext=".ipynb"))
 
 
+async def test_rename_nonexistent(jp_contents_manager):
+    """Test renaming a non-existent file/directory returns 404 error"""
+    cm = jp_contents_manager
+
+    # Test with non-existent file
+    with pytest.raises(HTTPError) as e:
+        await ensure_async(cm.rename("nonexistent_file.txt", "new_name.txt"))
+    assert expected_http_error(e, 404)
+
+    # Test with non-existent directory
+    with pytest.raises(HTTPError) as e:
+        await ensure_async(cm.rename("nonexistent_dir", "new_dir"))
+    assert expected_http_error(e, 404)
+
+
 async def test_delete_root(jp_contents_manager):
     cm = jp_contents_manager
     with pytest.raises(HTTPError) as e: