Fix the error values the libc backend ttyname implementation.

libc'c `ttyname_r` returns its error code rather than setting `errno`;
fix the wrapper code accordingly.

Author: sunfishcode

src/imp/libc/termios/syscalls.rs

@@ -150,11 +150,10 @@ pub(crate) fn isatty(fd: BorrowedFd<'_>) -> bool {
 #[cfg(not(any(target_os = "fuchsia", target_os = "wasi")))]
 pub(crate) fn ttyname(dirfd: BorrowedFd<'_>, buf: &mut [u8]) -> io::Result<usize> {
     unsafe {
-        ret(c::ttyname_r(
-            borrowed_fd(dirfd),
-            buf.as_mut_ptr().cast(),
-            buf.len(),
-        ))?;
-        Ok(CStr::from_ptr(buf.as_ptr().cast()).to_bytes().len())
+        // `ttyname_r` returns its error status rather than using `errno`.
+        match c::ttyname_r(borrowed_fd(dirfd), buf.as_mut_ptr().cast(), buf.len()) {
+            0 => Ok(CStr::from_ptr(buf.as_ptr().cast()).to_bytes().len()),
+            err => Err(io::Errno::from_raw_os_error(err)),
+        }
     }
 }

tests/termios/main.rs

@@ -6,3 +6,5 @@
 
 #[cfg(not(windows))]
 mod isatty;
+#[cfg(not(windows))]
+mod ttyname;

tests/termios/ttyname.rs

@@ -0,0 +1,22 @@
+use rustix::io;
+use rustix::termios::ttyname;
+use std::fs::File;
+
+#[test]
+fn test_ttyname_ok() {
+    let file = File::open("/dev/stdin").unwrap();
+    assert!(ttyname(&file, Vec::new())
+        .unwrap()
+        .into_string()
+        .unwrap()
+        .starts_with("/dev/"));
+}
+
+#[test]
+fn test_ttyname_not_tty() {
+    let file = File::open("Cargo.toml").unwrap();
+    assert_eq!(ttyname(&file, Vec::new()).unwrap_err(), io::Errno::NOTTY);
+
+    let file = File::open("/dev/null").unwrap();
+    assert_eq!(ttyname(&file, Vec::new()).unwrap_err(), io::Errno::NOTTY);
+}