pythongh-134744: Fix fcntl error handling (python#134748)

Fix also reference leak on buffer overflow.

Author: vstinner

Lib/test/test_fcntl.py

@@ -11,7 +11,7 @@
     cpython_only, get_pagesize, is_apple, requires_subprocess, verbose
 )
 from test.support.import_helper import import_module
-from test.support.os_helper import TESTFN, unlink
+from test.support.os_helper import TESTFN, unlink, make_bad_fd
 
 
 # Skip test if no fcntl module.
@@ -274,6 +274,17 @@ def test_fcntl_small_buffer(self):
     def test_fcntl_large_buffer(self):
         self._check_fcntl_not_mutate_len(2024)
 
+    @unittest.skipUnless(hasattr(fcntl, 'F_DUPFD'), 'need fcntl.F_DUPFD')
+    def test_bad_fd(self):
+        # gh-134744: Test error handling
+        fd = make_bad_fd()
+        with self.assertRaises(OSError):
+            fcntl.fcntl(fd, fcntl.F_DUPFD, 0)
+        with self.assertRaises(OSError):
+            fcntl.fcntl(fd, fcntl.F_DUPFD, b'\0' * 10)
+        with self.assertRaises(OSError):
+            fcntl.fcntl(fd, fcntl.F_DUPFD, b'\0' * 2048)
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_ioctl.py

@@ -5,7 +5,7 @@
 import threading
 import unittest
 from test import support
-from test.support import threading_helper
+from test.support import os_helper, threading_helper
 from test.support.import_helper import import_module
 fcntl = import_module('fcntl')
 termios = import_module('termios')
@@ -201,6 +201,17 @@ def test_ioctl_set_window_size(self):
         new_winsz = struct.unpack("HHHH", result)
         self.assertEqual(new_winsz[:2], (20, 40))
 
+    @unittest.skipUnless(hasattr(fcntl, 'FICLONE'), 'need fcntl.FICLONE')
+    def test_bad_fd(self):
+        # gh-134744: Test error handling
+        fd = os_helper.make_bad_fd()
+        with self.assertRaises(OSError):
+            fcntl.ioctl(fd, fcntl.FICLONE, fd)
+        with self.assertRaises(OSError):
+            fcntl.ioctl(fd, fcntl.FICLONE, b'\0' * 10)
+        with self.assertRaises(OSError):
+            fcntl.ioctl(fd, fcntl.FICLONE, b'\0' * 2048)
+
 
 if __name__ == "__main__":
     unittest.main()

Modules/fcntlmodule.c

@@ -128,14 +128,15 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg)
                 Py_END_ALLOW_THREADS
             } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
             if (ret < 0) {
-                if (async_err) {
+                if (!async_err) {
                     PyErr_SetFromErrno(PyExc_OSError);
                 }
                 Py_DECREF(result);
                 return NULL;
             }
             if (ptr[len] != '\0') {
                 PyErr_SetString(PyExc_SystemError, "buffer overflow");
+                Py_DECREF(result);
                 return NULL;
             }
             return result;
@@ -310,14 +311,15 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg,
                 Py_END_ALLOW_THREADS
             } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
             if (ret < 0) {
-                if (async_err) {
+                if (!async_err) {
                     PyErr_SetFromErrno(PyExc_OSError);
                 }
                 Py_DECREF(result);
                 return NULL;
             }
             if (ptr[len] != '\0') {
                 PyErr_SetString(PyExc_SystemError, "buffer overflow");
+                Py_DECREF(result);
                 return NULL;
             }
             return result;