Wasip3: File handing implementation

This is an initial implementation of some basic file-handling functions
such as reading, writing, opening directories, iterating them, etc. A
number of previously failing tests are now passing and the
`wasip2_file.c` implementation is now merged with WASIp3 with `file.c`.
Bindings generation is also updated to flag some specific functions as
being lowered synchronously as libc always blocks on the result.

Author: cpetig

cmake/bindings.cmake

@@ -107,6 +107,17 @@ add_custom_target(
       --rename wasi:cli/terminal-stdin@${wasip3-version}=terminal_stdin
       --rename wasi:cli/terminal-stdout@${wasip3-version}=terminal_stdout
       --rename wasi:cli/terminal-stderr@${wasip3-version}=terminal_stderr
+
+      # Disable async bindings generation for some functions which are only
+      # ever called synchronously within libc.
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.metadata-hash"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.metadata-hash-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.stat"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.get-flags"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.open-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.read-directory"
+      "--async=-wasi:clocks/monotonic-clock@${wasip3-version}#wait-until"
+      "--async=-wasi:clocks/monotonic-clock@${wasip3-version}#wait-for"
       ${CMAKE_SOURCE_DIR}/wasi/p3/wit
   COMMAND cmake -E copy wasip3.h ${bottom_half}/headers/public/wasi/__generated_wasip3.h
   COMMAND cmake -E copy wasip3_component_type.o ${bottom_half}/sources

expected/wasm32-wasip2/defined-symbols.txt

@@ -1494,6 +1494,7 @@ vwprintf
 vwscanf
 wall_clock_now
 wall_clock_resolution
+wasi_string_from_c
 wasip2_list_string_free
 wasip2_list_tuple2_string_string_free
 wasip2_list_u32_free
@@ -1502,7 +1503,6 @@ wasip2_option_string_free
 wasip2_string_dup
 wasip2_string_dup_n
 wasip2_string_free
-wasip2_string_from_c
 wasip2_string_set
 wasip2_tuple2_string_string_free
 wcpcpy

expected/wasm32-wasip3/defined-symbols.txt

@@ -331,7 +331,7 @@ __wasilibc_parse_port
 __wasilibc_populate_preopens
 __wasilibc_pthread_self
 __wasilibc_random
-__wasilibc_read_stream3
+__wasilibc_read
 __wasilibc_rename_newat
 __wasilibc_rename_oldat
 __wasilibc_reset_preopens
@@ -345,7 +345,7 @@ __wasilibc_unspecified_addr
 __wasilibc_utimens
 __wasilibc_wasi_family_to_libc
 __wasilibc_wasi_to_sockaddr
-__wasilibc_write_stream3
+__wasilibc_write
 __wasm_call_dtors
 __wcscoll_l
 __wcsftime_l
@@ -1463,21 +1463,25 @@ vswprintf
 vswscanf
 vwprintf
 vwscanf
+wasi_string_from_c
 wasip3_backpressure_dec
 wasip3_backpressure_inc
 wasip3_context_get_0
 wasip3_context_get_1
 wasip3_context_set_0
 wasip3_context_set_1
+wasip3_future_block_on
 wasip3_list_string_free
 wasip3_list_tuple2_string_string_free
 wasip3_list_u8_free
 wasip3_option_string_free
+wasip3_stream_block_on
 wasip3_string_dup
 wasip3_string_dup_n
 wasip3_string_free
 wasip3_string_set
 wasip3_subtask_block_on
+wasip3_subtask_block_on_and_drop
 wasip3_subtask_cancel
 wasip3_subtask_drop
 wasip3_task_cancel
@@ -1495,7 +1499,6 @@ wasip3_thread_yield_cancellable
 wasip3_thread_yield_to_suspended
 wasip3_thread_yield_to_suspended_cancellable
 wasip3_tuple2_string_string_free
-wasip3_waitable_block_on
 wasip3_waitable_join
 wasip3_waitable_set_drop
 wasip3_waitable_set_new

libc-bottom-half/CMakeLists.txt

@@ -140,6 +140,7 @@ else()
     sources/getsockpeername.c
     sources/listen.c
     sources/file_utils.c
+    sources/file.c
     sources/recv.c
     sources/send.c
     sources/shutdown.c
@@ -155,7 +156,6 @@ if(WASI STREQUAL "p2")
   list(APPEND bottom_half_sources
     sources/netdb.c
     sources/wasip2.c
-    sources/wasip2_file.c
     sources/wasip2_stdio.c
   )
 endif()
@@ -164,7 +164,6 @@ if (WASI STREQUAL "p3")
   list(APPEND bottom_half_sources
     sources/wasip3.c
     sources/wasip3_block_on.c
-    sources/wasip3_file.c
     sources/wasip3_stdio.c
   )
 endif()

libc-bottom-half/cloudlibc/src/libc/dirent/dirent_impl.h

@@ -28,7 +28,7 @@ struct _DIR {
   size_t skip;
   size_t offset;
 #elif defined(__wasip3__)
-  filesystem_stream_directory_entry_t stream;
+  filesystem_tuple2_stream_directory_entry_future_result_void_error_code_t stream;
   size_t skip;
   size_t offset;
 #else
@@ -40,4 +40,26 @@ struct _DIR {
   size_t dirent_size;
 };
 
+static void dirent_close_streams(DIR *dirp) {
+#if defined(__wasip1__)
+  // nothing to close ...
+#elif defined(__wasip2__)
+  if (dirp->stream.__handle) {
+    filesystem_directory_entry_stream_drop_own(dirp->stream);
+    dirp->stream.__handle = 0;
+  }
+#elif defined(__wasip3__)
+  if (dirp->stream.f0 != 0) {
+    filesystem_stream_directory_entry_drop_readable(dirp->stream.f0);
+    dirp->stream.f0 = 0;
+  }
+  if (dirp->stream.f1 != 0) {
+    filesystem_future_result_void_error_code_drop_readable(dirp->stream.f1);
+    dirp->stream.f1 = 0;
+  }
+#else
+# error "Unknown WASI version"
+#endif
+}
+
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/fdclosedir.c

@@ -11,12 +11,8 @@ int fdclosedir(DIR *dirp) {
   int fd = dirp->fd;
 #if defined(__wasip1__)
   free(dirp->buffer);
-#elif defined(__wasip2__)
-  if (dirp->stream.__handle != 0)
-    filesystem_directory_entry_stream_drop_own(dirp->stream);
-#elif defined(__wasip3__)
-  if (dirp->stream != 0)
-    filesystem_stream_directory_entry_drop_readable(dirp->stream);
+#elif defined(__wasip2__) || defined(__wasip3__)
+  dirent_close_streams(dirp);
 #else
 # error "Unsupported WASI version"
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/fdopendir.c

@@ -49,7 +49,7 @@ DIR *fdopendir(int fd) {
   dirp->dirent = NULL;
   dirp->dirent_size = 1;
   return dirp;
-#elif defined(__wasip2__)
+#elif defined(__wasip2__) || defined(__wasip3__)
   // Translate the file descriptor to an internal handle
   filesystem_borrow_descriptor_t file_handle;
   if (fd_to_file_handle(fd, &file_handle) < 0) {
@@ -58,6 +58,7 @@ DIR *fdopendir(int fd) {
   }
 
   // Read the directory
+#ifdef __wasip2__
   filesystem_own_directory_entry_stream_t result;
   filesystem_error_code_t error_code;
   bool ok = filesystem_method_descriptor_read_directory(file_handle,
@@ -69,18 +70,16 @@ DIR *fdopendir(int fd) {
     return NULL;
   }
 
-  dirp->fd = fd;
   dirp->stream = result;
+#elif defined(__wasip3__)
+  filesystem_method_descriptor_read_directory(file_handle, &dirp->stream);
+#endif
+  dirp->fd = fd;
   dirp->skip = 0;
   dirp->offset = 0;
   dirp->dirent = NULL;
   dirp->dirent_size = 1;
   return dirp;
-#elif defined(__wasip3__)
-  // TODO(wasip3)
-  errno = ENOTSUP;
-  free(dirp);
-  return NULL;
 #else
 # error "Unsupported WASI version"
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/readdir.c

@@ -31,6 +31,10 @@ static_assert(DT_REG == __WASI_FILETYPE_REGULAR_FILE, "Value mismatch");
 static_assert(DT_UNKNOWN == __WASI_FILETYPE_UNKNOWN, "Value mismatch");
 #endif
 
+#ifdef __wasip3__
+#include <wasi/wasip3_block.h>
+#endif
+
 // Grows a buffer to be large enough to hold a certain amount of data.
 #define GROW(buffer, buffer_size, target_size)      \
   do {                                              \
@@ -140,12 +144,13 @@ struct dirent *readdir(DIR *dirp) {
   }
 }
 
-#elif defined(__wasip2__)
+#elif defined(__wasip2__) || defined(__wasip3__)
 
 static int ensure_has_directory_stream(DIR *dirp, filesystem_borrow_descriptor_t *handle) {
   if (fd_to_file_handle(dirp->fd, handle) < 0)
     return -1;
 
+#ifdef __wasip2__
   if (dirp->stream.__handle != 0)
     return 0;
 
@@ -157,10 +162,15 @@ static int ensure_has_directory_stream(DIR *dirp, filesystem_borrow_descriptor_t
     translate_error(error_code);
     return -1;
   }
+#elif defined(__wasip3__)
+  if (dirp->stream.f0 == 0)
+    filesystem_method_descriptor_read_directory(*handle, &dirp->stream);
+#endif
   return 0;
 }
 
 static struct dirent *readdir_next(DIR *dirp) {
+  bool ok;
   filesystem_metadata_hash_value_t metadata;
   filesystem_error_code_t error_code;
   filesystem_borrow_descriptor_t dir_handle;
@@ -173,7 +183,7 @@ static struct dirent *readdir_next(DIR *dirp) {
   if (dirp->offset == 0) {
     dirp->offset += 1;
     GROW(dirp->dirent, dirp->dirent_size, offsetof(struct dirent, d_name) + 2);
-    bool ok = filesystem_method_descriptor_metadata_hash(dir_handle,
+    ok = filesystem_method_descriptor_metadata_hash(dir_handle,
                                                          &metadata,
                                                          &error_code);
     if (!ok) {
@@ -200,11 +210,12 @@ static struct dirent *readdir_next(DIR *dirp) {
     return dirp->dirent;
   }
 
+#if defined(__wasip2__)
   filesystem_borrow_directory_entry_stream_t stream = filesystem_borrow_directory_entry_stream(dirp->stream);
   filesystem_option_directory_entry_t dir_entry_optional;
-  bool ok = filesystem_method_directory_entry_stream_read_directory_entry(stream,
-                                                                          &dir_entry_optional,
-                                                                          &error_code);
+  ok = filesystem_method_directory_entry_stream_read_directory_entry(stream,
+                                                                     &dir_entry_optional,
+                                                                     &error_code);
   if (!ok) {
     translate_error(error_code);
     return NULL;
@@ -216,6 +227,46 @@ static struct dirent *readdir_next(DIR *dirp) {
 
   filesystem_directory_entry_t dir_entry = dir_entry_optional.val;
 
+#elif defined(__wasip3__)
+  filesystem_directory_entry_t dir_entry;
+
+  // Loop until at least one stream entry is read, or until the stream is closed.
+  bool closed = false;
+  while (1) {
+    size_t amount =
+      wasip3_stream_block_on(
+          filesystem_stream_directory_entry_read(dirp->stream.f0, &dir_entry, 1),
+          dirp->stream.f0,
+          &closed);
+
+    // If something was read, then break out and process that below.
+    if (amount > 0)
+      break;
+
+    // If nothing was read and the stream isn't finished yet, try again.
+    if (!closed)
+      continue;
+
+    // If the stream's result future hasn't been read yet, do so here.
+    if (dirp->stream.f1) {
+      filesystem_result_void_error_code_t result;
+      wasip3_future_block_on(
+          filesystem_future_result_void_error_code_read(dirp->stream.f1, &result),
+          dirp->stream.f1);
+      filesystem_future_result_void_error_code_drop_readable(dirp->stream.f1);
+      dirp->stream.f1 = 0;
+      if (result.is_err)
+        translate_error(result.val.err);
+    }
+
+    // The stream is closed, so return NULL. This'll set `errno` based on the
+    // result of the future above.
+    return NULL;
+  }
+#else
+#error "Unknown WASI version"
+#endif
+
   // Ensure that the dirent is large enough to fit the filename
   size_t the_size = offsetof(struct dirent, d_name);
   GROW(dirp->dirent, dirp->dirent_size, the_size + dir_entry.name.len + 1);
@@ -232,7 +283,11 @@ static struct dirent *readdir_next(DIR *dirp) {
                                                      &dir_entry.name,
                                                      &metadata,
                                                      &error_code);
+#ifdef __wasip2__
   wasip2_string_free(&dir_entry.name);
+#else
+  wasip3_string_free(&dir_entry.name);
+#endif
   if (!ok) {
     translate_error(error_code);
     return NULL;
@@ -251,13 +306,6 @@ struct dirent *readdir(DIR *dirp) {
   }
   return result;
 }
-#elif defined(__wasip3__)
-struct dirent *readdir(DIR *dirp) {
-  // TODO(wasip3)
-  errno = ENOTSUP;
-  free(dirp);
-  return NULL;
-}
 #else
 # error "Unknown WASI version"
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/rewinddir.c

@@ -13,12 +13,8 @@ void rewinddir(DIR *dirp) {
   dirp->cookie = __WASI_DIRCOOKIE_START;
   // Mark entire buffer as processed to force a read of new data.
   dirp->buffer_used = dirp->buffer_processed = dirp->buffer_size;
-#elif defined(__wasip2__)
-  dirp->stream.__handle = 0;
-  dirp->skip = 0;
-  dirp->offset = 0;
-#elif defined(__wasip3__)
-  dirp->stream = 0;
+#elif defined(__wasip2__) || defined(__wasip3__)
+  dirent_close_streams(dirp);
   dirp->skip = 0;
   dirp->offset = 0;
 #else

libc-bottom-half/cloudlibc/src/libc/dirent/seekdir.c

@@ -13,12 +13,8 @@ void seekdir(DIR *dirp, long loc) {
   // Mark entire buffer as processed to force a read of new data.
   // TODO(ed): We could prevent a read if the offset is in the buffer.
   dirp->buffer_used = dirp->buffer_processed = dirp->buffer_size;
-#elif defined(__wasip2__)
-  dirp->stream.__handle = 0;
-  dirp->skip = loc;
-  dirp->offset = 0;
-#elif defined(__wasip3__)
-  dirp->stream = 0;
+#elif defined(__wasip2__) || defined(__wasip3__)
+  dirent_close_streams(dirp);
   dirp->skip = loc;
   dirp->offset = 0;
 #else