Fix broken i53 overflow checks in libsyscall.js. NFC (#24257)

These checks have been broken since they were added in #19711.

 Add some tests for the overload case.

Author: sbc100

src/lib/libsyscall.js

@@ -153,7 +153,11 @@ var SyscallsLibrary = {
   ],
   _mmap_js: (len, prot, flags, fd, offset, allocated, addr) => {
 #if FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM
-    if (isNaN(offset)) return {{{ cDefs.EOVERFLOW }}};
+#if ASSERTIONS
+    // musl's mmap doesn't allow values over a certain limit
+    // see OFF_MASK in mmap.c.
+    assert(!isNaN(offset));
+#endif
     var stream = SYSCALLS.getStreamFromFD(fd);
     var res = FS.mmap(stream, len, offset, prot, flags);
     var ptr = res.ptr;
@@ -631,7 +635,7 @@ var SyscallsLibrary = {
   },
   _msync_js__i53abi: true,
   _msync_js: (addr, len, prot, flags, fd, offset) => {
-    if (isNaN(offset)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(offset)) return -{{{ cDefs.EOVERFLOW }}};
     SYSCALLS.doMsync(addr, SYSCALLS.getStreamFromFD(fd), len, flags, offset);
     return 0;
   },
@@ -670,14 +674,14 @@ var SyscallsLibrary = {
   },
   __syscall_truncate64__i53abi: true,
   __syscall_truncate64: (path, length) => {
-    if (isNaN(length)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(length)) return -{{{ cDefs.EOVERFLOW }}};
     path = SYSCALLS.getStr(path);
     FS.truncate(path, length);
     return 0;
   },
   __syscall_ftruncate64__i53abi: true,
   __syscall_ftruncate64: (fd, length) => {
-    if (isNaN(length)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(length)) return -{{{ cDefs.EOVERFLOW }}};
     FS.ftruncate(fd, length);
     return 0;
   },
@@ -995,7 +999,7 @@ var SyscallsLibrary = {
   },
   __syscall_fallocate__i53abi: true,
   __syscall_fallocate: (fd, mode, offset, len) => {
-    if (isNaN(offset)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(offset) || isNaN(len)) return -{{{ cDefs.EOVERFLOW }}};
     if (mode != 0) {
       return -{{{ cDefs.ENOTSUP }}}
     }

src/lib/libwasmfs_node.js

@@ -159,12 +159,14 @@ addToLibrary({
   _wasmfs_node_truncate__i53abi: true,
   _wasmfs_node_truncate__deps : ['$wasmfsTry'],
   _wasmfs_node_truncate : (path_p, len) => {
+    if (isNaN(len)) return -{{{ cDefs.EOVERFLOW }}};
     return wasmfsTry(() => fs.truncateSync(UTF8ToString(path_p), len));
   },
 
   _wasmfs_node_ftruncate__i53abi: true,
   _wasmfs_node_ftruncate__deps : ['$wasmfsTry'],
   _wasmfs_node_ftruncate : (fd, len) => {
+    if (isNaN(len)) return -{{{ cDefs.EOVERFLOW }}};
     return wasmfsTry(() => fs.ftruncateSync(fd, len));
   },
 

system/lib/wasmfs/memory_backend.h

@@ -25,6 +25,9 @@ class MemoryDataFile : public DataFile {
   int flush() override { return 0; }
   off_t getSize() override { return buffer.size(); }
   int setSize(off_t size) override {
+    if (size > buffer.max_size()) {
+      return -EOVERFLOW;
+    }
     buffer.resize(size);
     return 0;
   }

test/fcntl/test_fcntl_misc.c

@@ -40,10 +40,19 @@ int main() {
 
   err = posix_fallocate(f, -1, 7);
   printf("posix_fallocate 3: %s\n", strerror(err));
-  printf("\n");
 
   err = posix_fallocate(f, 3, -1);
   printf("posix_fallocate 4: %s\n", strerror(err));
 
+  // Values over 2^53 are not representable in JS and
+  // should result in EOVERFLOW.
+  err = posix_fallocate(f, 1, 0x00ffffffffffffff);
+  assert(err == EOVERFLOW);
+  printf("posix_fallocate 5: %s\n", strerror(err));
+
+  err = posix_fallocate(f, 0x00ffffffffffffff, 1);
+  assert(err == EOVERFLOW);
+  printf("posix_fallocate 6: %s\n", strerror(err));
+
   return 0;
 }

test/fcntl/test_fcntl_misc.out

@@ -7,5 +7,6 @@ posix_fallocate 2: 0
 st_size: 10
 
 posix_fallocate 3: Invalid argument
-
 posix_fallocate 4: Invalid argument
+posix_fallocate 5: Value too large for data type
+posix_fallocate 6: Value too large for data type

test/fs/test_mmap.c

@@ -164,6 +164,11 @@ void test_mmap_shared_with_offset() {
   // assert failure if offset is not a multiple of page size
   assert(map == MAP_FAILED);
 
+  map = (char*)mmap(0, textsize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0x00ffffffffffffff);
+  // mmap offset outside MAX_SAFE_INTEGER range.
+  assert(map == MAP_FAILED);
+  assert(errno == EINVAL);
+
   map = (char*)mmap(0, textsize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
   assert(map != MAP_FAILED);
 

test/other/codesize/test_codesize_cxx_wasmfs.size

@@ -1 +1 @@
-169140
+169161

test/other/codesize/test_codesize_files_wasmfs.size

@@ -1 +1 @@
-49964
+49985

test/unistd/truncate.c

@@ -124,5 +124,30 @@ int main() {
   memset(&s, 0, sizeof s);
   errno = 0;
 
+#if __wasm32__ || !defined(WASMFS) || defined(NODEFS) || defined(NODERAWFS)
+  // These last two test don't run against the in-memory wasmfs filesystem
+  // in wasm64 mode since in that case they fail with an malloc abort.
+  //
+  // If we are running in wasm32, or we are using JS-based FS then we detect
+  // these overflows before any attempt at allocation.
+  printf("ftruncate(0x00ffffffffffffff): %d\n", ftruncate(f, 0x00ffffffffffffff));
+  printf("errno: %s\n", strerror(errno));
+  fstat(f, &s);
+  printf("st_size: %lld\n", s.st_size);
+  assert(s.st_size == 0);
+  memset(&s, 0, sizeof s);
+  errno = 0;
+  printf("\n");
+
+  printf("truncate(0x00ffffffffffffff): %d\n", truncate("towrite", 0x00ffffffffffffff));
+  printf("errno: %s\n", strerror(errno));
+  fstat(f, &s);
+  printf("st_size: %lld\n", s.st_size);
+  assert(s.st_size == 0);
+  memset(&s, 0, sizeof s);
+  errno = 0;
+  printf("\n");
+#endif
+
   return 0;
 }