Enable pthreads on Emscripten (#17586)

* workerized RA

* Workerized (non-async) web player, using OPFS

This patch eliminates the need for asyncify and uses modern filesystem
APIs instead of the deprecated, unmaintained BrowserFS.

This is a WIP patch because it won't fully work until these two
Emscripten PRs land and are released:

emscripten-core/emscripten#23518
emscripten-core/emscripten#23021

The former fixes an offscreen canvas context recreation bug, and the
latter adds an equivalent to BrowserFS's XHR filesystem (but without
the hazardous running-XHR-on-the-main-thread problem).

The biggest issue is that local storage of users who were using the
old version of the webplayer will be gone when they switch to the new
webplayer.  I don't have a good story for converting the old BrowserFS
IDBFS contents into the new OPFS filesystem (the move is worth doing
because OPFS supports seeking and reading only bits of a file, and
because BrowserFS is dead).

I've kept around the old libretro webplayer under
pkg/emscripten/libretro-classic, and with these make flags you can
build a non-workerized RA that uses asyncify to sleep as before:

make -f Makefile.emscripten libretro=$CORE HAVE_WORKER=0 HAVE_WASMFS=0 PTHREAD=0 HAVE_AL=1

I also moved the default directory for core content on emscripten to
not be a subdirectory of the local filesystem mount, because it's
confusing to have a subdirectory that's lazily fetched and not
mirrored to the local storage.  I think it won't impact existing users
of the classic web player because they already have a retroarch.cfg in
place.

* Get fetchfs working without manifest support

* makefile fixes

* fix scaling, remove zip dependency

* Support asset/cheats/etc downloaders for emscripten

- Add http transfer support for emscripten
  - At the task_http level, not the net_http level --- so no netplay
    or webdav.
- Change default paths to be more like other platforms
- Gives us smaller bundles and a faster boot time
- Had to work around a task queue bug on Emscripten
  - I made the smallest possible change to do it, but it may be better
    to fix in rthread.c

* Load an emscripten file_packager package on first run

If no ozone assets are present, load a libretro_minimal package
created using Emscripten's built-in file packager.

* updated readme, removed indexer from wasmfs libretro-web

* Put back zip dependency, load asset bundle into opfs on first run

* fix upload path

* Remove unused function

* easy testing setup for two multithreaded conditions

1. make PROXY_TO_PTHREAD=1 (slower)
2. make PROXY_TO_PTHREAD=0 (bad audio, because doesn't sleep in
openal.c)

* Remove condition on sleep in openal

also make input_driver check existence of drv->axis, drv->button
before calling them.

* Fix resizing under EGL

* Don't force config file path on emscripten

* Add time.h include to netplay, default HAVE_NETPLAYDISCOVERY to 0

* Remove nearly all proxied joypad calls under emscripten

* Fix file uploads under firefox

* Fix safari API uses, but Safari still hangs in OPFS filesystem mount

I think this can be fixed by moving the backend creation off the main
thread.

* Move filesystem init into emscripten C entry point

* Setup filesystems off of main thread

* re-set default player to async

Also improve Safari compatibility under proxy-to-pthread condition

* Safari upload file fixes

* Remove some excess prints

* Fix typo

Author: JoeOsborn

Makefile.common

@@ -1536,7 +1536,10 @@ ifeq ($(HAVE_GL_CONTEXT), 1)
    endif
 
    ifeq ($(HAVE_EMSCRIPTEN), 1)
-      OBJ += gfx/drivers_context/emscriptenegl_ctx.o
+      ifeq ($(HAVE_EGL), 1)
+        OBJ += gfx/drivers_context/emscriptenegl_ctx.o
+      endif
+      OBJ += gfx/drivers_context/emscriptenwebgl_ctx.o
    endif
 
    ifeq ($(HAVE_MALI_FBDEV), 1)
@@ -2203,11 +2206,16 @@ ifeq ($(HAVE_NETWORKING), 1)
           $(LIBRETRO_COMM_DIR)/net/net_socket.o \
           core_updater_list.o \
           network/natt.o \
-          tasks/task_http.o \
           tasks/task_netplay_lan_scan.o \
           tasks/task_netplay_nat_traversal.o \
           tasks/task_netplay_find_content.o
 
+   ifeq ($(HAVE_EMSCRIPTEN), 1)
+     OBJ += tasks/task_http_emscripten.o
+   else
+     OBJ += tasks/task_http.o
+   endif
+
    ifeq ($(HAVE_MENU), 1)
       OBJ += tasks/task_pl_thumbnail_download.o
    endif

Makefile.emscripten

@@ -19,12 +19,18 @@ HAVE_PATCH = 1
 HAVE_DSP_FILTER = 1
 HAVE_VIDEO_FILTER = 1
 HAVE_OVERLAY = 1
+HAVE_NETWORKING = 1
+HAVE_LIBRETRODB = 1
+HAVE_COMPRESSION = 1
+HAVE_UPDATE_ASSETS = 1
+HAVE_ONLINE_UPDATER = 1
 HAVE_GLSL = 1
 HAVE_SCREENSHOTS = 1
 HAVE_REWIND = 1
 HAVE_AUDIOMIXER = 1
 HAVE_CC_RESAMPLER = 1
-HAVE_EGL    = 1
+HAVE_EGL    ?= 1
+HAVE_OPENGLES = 1
 HAVE_RJPEG  = 0
 HAVE_RPNG   = 1
 HAVE_EMSCRIPTEN = 1
@@ -48,6 +54,12 @@ HAVE_7ZIP = 1
 HAVE_BSV_MOVIE = 1
 HAVE_AL = 1
 HAVE_CHD ?= 0
+HAVE_WASMFS ?= 0
+PROXY_TO_PTHREAD ?= 0
+HAVE_NETPLAYDISCOVERY ?= 0
+
+DEFINES += -DHAVE_NETWORKING -DHAVE_ONLINE_UPDATER -DHAVE_UPDATE_ASSETS -DHAVE_COMPRESSION
+DEFINES += -DHAVE_UPDATE_CORE_INFO
 
 # WARNING -- READ BEFORE ENABLING
 # The rwebaudio driver is known to have several audio bugs, such as
@@ -61,12 +73,12 @@ HAVE_RWEBAUDIO = 0
 GL_DEBUG ?= 0
 
 # enable javascript filesystem tracking
-FS_DEBUG = 1
+FS_DEBUG = 0
 
 HAVE_OPENGLES ?= 1
 HAVE_OPENGLES3 ?= 0
 
-ASYNC ?= 0
+ASYNC ?= 1
 LTO ?= 0
 PTHREAD ?= 0
 
@@ -92,61 +104,86 @@ _cmd_toggle_menu,_cmd_reload_config,_cmd_toggle_grab_mouse,_cmd_toggle_game_focu
 _cmd_set_volume,_cmd_set_shader,_cmd_cheat_set_code,_cmd_cheat_get_code,_cmd_cheat_toggle_index,_cmd_cheat_get_code_state,_cmd_cheat_realloc,\
 _cmd_cheat_get_size,_cmd_cheat_apply_cheats
 
-LIBS    := -s USE_ZLIB=1
-LDFLAGS := -L. --no-heap-copy -s $(LIBS) -s STACK_SIZE=$(STACK_SIZE) -s INITIAL_MEMORY=$(INITIAL_HEAP) \
-           -s EXPORTED_RUNTIME_METHODS=callMain,FS,PATH,ERRNO_CODES,stringToNewUTF8,UTF8ToString \
+EXPORTS := callMain,FS,PATH,ERRNO_CODES,ENV,stringToNewUTF8,UTF8ToString,Browser,GL
+
+LIBS    := -s USE_ZLIB=1 -lbrowser.js
+
+ifeq ($(HAVE_WASMFS), 1)
+   DEFINES += -DHAVE_WASMFS=1
+   LIBS += -sWASMFS -sFORCE_FILESYSTEM=1 -lfetchfs.js -lopfs.js
+   EXPORTS := $(EXPORTS),FETCHFS,OPFS
+   ifeq ($(PTHREAD),0)
+   $(error ERROR: WASMFS requires threading support)
+   endif
+endif
+
+ifeq ($(PROXY_TO_PTHREAD),1)
+   LIBS += -sUSE_ES6_IMPORT_META=0 -sENVIRONMENT=worker,web
+   LIBS += -sPROXY_TO_PTHREAD -sOFFSCREENCANVAS_SUPPORT
+   DEFINES += -DUSE_OFFSCREENCANVAS=1 -DPROXY_TO_PTHREAD=1
+else
+   override ASYNC = 1
+endif
+   
+ifeq ($(HAVE_SDL2), 1)
+   LIBS += -s USE_SDL=2
+   DEFINES += -DHAVE_SDL2
+endif
+
+
+LDFLAGS := -L. --no-heap-copy -s STACK_SIZE=$(STACK_SIZE) -s INITIAL_MEMORY=$(INITIAL_HEAP) \
+           -s EXPORTED_RUNTIME_METHODS=$(EXPORTS) \
            -s ALLOW_MEMORY_GROWTH=1 -s EXPORTED_FUNCTIONS="$(EXPORTED_FUNCTIONS)" \
            -s MODULARIZE=1 -s EXPORT_ES6=1 -s EXPORT_NAME="libretro_$(subst -,_,$(LIBRETRO))" \
+           -s DISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=0 \
            --extern-pre-js emscripten/pre.js \
            --js-library emscripten/library_rwebcam.js \
+           -gsource-map -g2 \
            --js-library emscripten/library_platform_emscripten.js
 
+ifeq ($(HAVE_OPENGLES), 1)
+   ifeq ($(HAVE_OPENGLES3), 1)
+      LDFLAGS += -s FULL_ES3=1 -s MIN_WEBGL_VERSION=2 -s MAX_WEBGL_VERSION=2 -lGL
+   else
+      LDFLAGS += -s FULL_ES2=1 -s MIN_WEBGL_VERSION=1 -s MAX_WEBGL_VERSION=2 -lGL
+   endif
+endif
+
+ifeq ($(GL_DEBUG), 1)
+   LDFLAGS += -s GL_ASSERTIONS=1 -s GL_DEBUG=1 -DHAVE_GL_DEBUG_ES=1
+endif
+
+ifeq ($(FS_DEBUG), 1)
+   LDFLAGS += -s FS_DEBUG=1
+endif
+
 ifeq ($(HAVE_RWEBAUDIO), 1)
    LDFLAGS += --js-library emscripten/library_rwebaudio.js
    DEFINES += -DHAVE_RWEBAUDIO
 endif
+
 ifeq ($(HAVE_AL), 1)
    LDFLAGS += -lopenal
    DEFINES += -DHAVE_AL
-   override ASYNC = 1
 endif
 
 ifneq ($(PTHREAD), 0)
-   LDFLAGS += -s MAXIMUM_MEMORY=1073741824 -pthread -s PTHREAD_POOL_SIZE=$(PTHREAD)
-   CFLAGS += -pthread
+   LDFLAGS += -s WASM_MEM_MAX=1073741824 -pthread -s PTHREAD_POOL_SIZE=$(PTHREAD)
+   CFLAGS += -pthread -s SHARED_MEMORY
    HAVE_THREADS=1
 else
    HAVE_THREADS=0
 endif
 
+
 ifeq ($(ASYNC), 1)
+   DEFINES += -DEMSCRIPTEN_ASYNCIFY
    LDFLAGS += -s ASYNCIFY=$(ASYNC) -s ASYNCIFY_STACK_SIZE=8192 
    ifeq ($(DEBUG), 1)
       LDFLAGS += -s ASYNCIFY_DEBUG=1 # -s ASYNCIFY_ADVISE
    endif
 endif
 
-ifeq ($(HAVE_OPENGLES), 1)
-   ifeq ($(HAVE_OPENGLES3), 1)
-      LDFLAGS += -s FULL_ES3=1 -s MIN_WEBGL_VERSION=2 -s MAX_WEBGL_VERSION=2
-   else
-      LDFLAGS += -s FULL_ES2=1
-   endif
-endif
-
-ifeq ($(GL_DEBUG), 1)
-   LDFLAGS += -s GL_ASSERTIONS=1 -s GL_DEBUG=1
-endif
-
-ifeq ($(FS_DEBUG), 1)
-   LDFLAGS += -s FS_DEBUG=1
-endif
-
-ifeq ($(HAVE_SDL2), 1)
-   LIBS += -s USE_SDL=2
-   DEFINES += -DHAVE_SDL2
-endif
-
 include Makefile.common
 
 CFLAGS += $(DEF_FLAGS) -Ideps -Ideps/stb
@@ -183,8 +220,10 @@ RARCH_OBJ := $(addprefix $(OBJDIR)/,$(OBJ))
 
 all: $(TARGET)
 
-$(TARGET): $(RARCH_OBJ) $(libretro)
-	@$(if $(libretro), mv -f $(libretro) $(libretro_new),)
+$(libretro_new) : $(libretro)
+	mv -f $(libretro) $(libretro_new)
+
+$(TARGET): $(RARCH_OBJ) $(libretro_new)
 	@$(if $(Q), $(shell echo echo "LD $@ \<obj\> $(libretro_new) $(LIBS) $(LDFLAGS)"),)
 	$(Q)$(LD) -o $@ $(RARCH_OBJ) $(libretro_new) $(LIBS) $(LDFLAGS)
 

config.def.h

@@ -1833,7 +1833,11 @@
 #define DEFAULT_BUILDBOT_SERVER_URL ""
 #endif
 
+#ifdef EMSCRIPTEN
+#define DEFAULT_BUILDBOT_ASSETS_SERVER_URL "https://buildbot.libretro.com/assets/"
+#else
 #define DEFAULT_BUILDBOT_ASSETS_SERVER_URL "http://buildbot.libretro.com/assets/"
+#endif
 
 #define DEFAULT_DISCORD_APP_ID "475456035851599874"
 

emscripten/library_platform_emscripten.js

@@ -15,28 +15,6 @@ var LibraryPlatformEmscripten = {
       }
    },
 
-   PlatformEmscriptenWatchCanvasSize: function() {
-      RPE.observer = new ResizeObserver(function(e) {
-         var width, height;
-         var entry = e.find(i => i.target == Module.canvas);
-         if (!entry) return;
-         if (entry.devicePixelContentBoxSize) {
-            width = entry.devicePixelContentBoxSize[0].inlineSize;
-            height = entry.devicePixelContentBoxSize[0].blockSize;
-         } else {
-            width = Math.round(entry.contentRect.width * window.devicePixelRatio);
-            height = Math.round(entry.contentRect.height * window.devicePixelRatio);
-         }
-         Module.setCanvasSize(width, height);
-         Module.print("Setting real canvas size: " + width + " x " + height);
-      });
-      RPE.observer.observe(Module.canvas);
-      window.addEventListener("resize", function(e) {
-         RPE.observer.unobserve(Module.canvas);
-         RPE.observer.observe(Module.canvas);
-      }, false);
-   },
-
    PlatformEmscriptenPowerStateInit: function() {
       if (!navigator.getBattery) return;
       navigator.getBattery().then(function(battery) {