Invalid file descriptor in Vma::new in vma.rs when calling `mmap`

[nuczyc]

Describe the bug

The kernel panics instead of returning an error code when a user-space program calls mmap using a closed or invalid file descriptor (FD).

ruxos/modules/ruxtask/src/vma.rs

Line 189 in 32a8fe8

let f = get_file_like(_fid).expect("invaild fd for vma");

To Reproduce

1. Compile the program and run.

#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>

/*
 * PoC for triggering VMA panic in RuxOS
 * 
 * The crash occurs in Vma::new() when get_file_like() returns None/Err
 * for a file descriptor that should be valid for memory mapping.
 * 
 * This PoC attempts to trigger the panic by:
 * 1. Opening a file
 * 2. Closing it to make the FD invalid
 * 3. Calling mmap() with the now-invalid FD
 * 
 * The kernel should panic when trying to create a VMA with this invalid FD.
 * 
 * Note: This may not trigger if RuxOS validates FDs before VMA creation.
 * The actual trigger depends on the specific implementation of get_file_like().
 */

int main() {
    int fd;
    void *mapped;
    const char *filename = "/tmp/test_file";
    
    // Create a test file
    fd = open(filename, O_CREAT | O_RDWR, 0644);
    if (fd < 0) {
        perror("open");
        return 1;
    }
    
    // Write some data to the file
    if (write(fd, "test", 4) != 4) {
        perror("write");
        close(fd);
        return 1;
    }
    
    // Close the file to make FD invalid
    close(fd);
    
    // Now try to mmap with the invalid FD
    // This should trigger the panic in Vma::new() when get_file_like() fails
    mapped = mmap(NULL, 4096, PROT_READ, MAP_PRIVATE, fd, 0);
    
    if (mapped == MAP_FAILED) {
        // If we reach here, the kernel properly handled the invalid FD
        printf("mmap failed as expected: %s\n", strerror(errno));
    } else {
        printf("mmap succeeded unexpectedly\n");
        munmap(mapped, 4096);
    }
    
    // Clean up
    unlink(filename);
    
    return 0;
}

2.features.txt

alloc
paging
net
multitask
irq
fs

Environment

• RuxOS version:[Merge pull request #219 from syswonder/dev · syswonder/ruxos@32a8fe8](32a8fe8)

Logs

SeaBIOS (version 1.16.3-debian-1.16.3-2)


iPXE (https://ipxe.org) 00:03.0 CA00 PCI2.10 PnP PMM+7EFCAA40+7EF0AA40 CA00
                                                                               


Booting from ROM..
Initialize IDT & GDT...

8888888b.                     .d88888b.   .d8888b.
888   Y88b                   d88P" "Y88b d88P  Y88b
888    888                   888     888 Y88b.
888   d88P 888  888 888  888 888     888  "Y888b.
8888888P"  888  888 `Y8bd8P' 888     888     "Y88b.
888 T88b   888  888   X88K   888     888       "888
888  T88b  Y88b 888 .d8""8b. Y88b. .d88P Y88b  d88P
888   T88b  "Y88888 888  888  "Y88888P"   "Y8888P"

arch = x86_64
platform = x86_64-qemu-q35
target = x86_64-unknown-none
smp = 1
build_mode = debug
log_level = warn

[  0.191439 0 axfs_ramfs::dir:68] AlreadyExists sys
[  0.191992 0:1 ruxruntime::lang_items:14] panicked at modules/ruxtask/src/vma.rs:189:41:
invaild fd for vma: EBADF