Don't abort on unsuported ioctl() (#17470)

``ioctl`` with unsupported ``request`` or ``argp`` argument now fails
with errno ``EINVAL`` instead of aborting the process.

Fixes: #13810
Closes: #13872

Author: tiran

src/library_syscall.js

@@ -252,7 +252,7 @@ var SyscallsLibrary = {
         if (!stream.tty) return -{{{ cDefine('ENOTTY') }}};
         return 0;
       }
-      default: abort('bad ioctl syscall ' + op);
+      default: return -{{{ cDefine('EINVAL') }}}; // not supported
     }
 #endif // SYSCALLS_REQUIRE_FILESYSTEM
   },

system/lib/wasmfs/syscalls.cpp

@@ -1233,13 +1233,8 @@ int __syscall_ioctl(int fd, int request, ...) {
       // TTY operations that we do nothing for anyhow can just be ignored.
       return -0;
     }
-    case TIOCGPGRP:
-    case TIOCSPGRP: {
-      // TODO We should get/set the group number here.
-      return -EINVAL;
-    }
     default: {
-      abort();
+      return -EINVAL; // not supported
     }
   }
 }

test/other/test_ioctl.c

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2022 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+int main() {
+  // CLOEXEC is not supported
+  assert(ioctl(STDOUT_FILENO, FIOCLEX, NULL) == -1);
+  assert(errno = EINVAL);
+
+  puts("success");
+  return 0;
+}

test/other/test_ioctl.out

@@ -0,0 +1 @@
+success

test/test_other.py

@@ -8414,6 +8414,11 @@ def test_autotools_shared_check(self):
   def test_ioctl_window_size(self):
       self.do_other_test('test_ioctl_window_size.cpp')
 
+  @also_with_wasmfs
+  def test_sys_ioctl(self):
+    # ioctl requires filesystem
+    self.do_other_test('test_ioctl.c', emcc_args=['-sFORCE_FILESYSTEM'])
+
   def test_fd_closed(self):
     self.do_other_test('test_fd_closed.cpp')