Default to X11 as GLFW_PLATFORM on Linux to avoid (#74)

* Default to `X11` as `GLFW_PLATFORM` on Linux to avoid

Wayland/libdecor ASAN leak which causes
unresponsiveness. See issue #68
#68

Summary: ASAN reports leaks seemingly from the Wayland
setup or decoration stack 
(libdecor/GTK/Pango/Fontconfig via GLFW)
that persist until process exit.

This does not seem to be related to render/present stalls;
override with GLFW_PLATFORM if you explicitly want Wayland.

Maybe it can work with Wayland at some point

* `glfw` `3.4` (#75)

it is required for the window hints

* turn off wayland for now

Author: jamylak

.github/workflows/linux-smoke-tests.yml

@@ -38,11 +38,24 @@ jobs:
           sudo apt-get install -y \
             build-essential cmake ninja-build \
             libgtest-dev libspdlog-dev \
-            libglfw3 libglfw3-dev libvulkan-dev \
+            libvulkan-dev \
             glslang-tools glslang-dev libglm-dev \
             mesa-vulkan-drivers xvfb pkg-config \
+            libx11-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev \
             libavcodec-dev libavformat-dev libavutil-dev libswscale-dev
 
+      - name: Build and install GLFW 3.4 (Linux X11 platform hints required)
+        run: |
+          git clone --depth 1 --branch 3.4 https://github.com/glfw/glfw.git /tmp/glfw
+          # Build X11-only to avoid Wayland issue
+          # See issue #68: https://github.com/jamylak/vsdf/issues/68
+          cmake -S /tmp/glfw -B /tmp/glfw/build -G Ninja \
+            -DGLFW_BUILD_TESTS=OFF -DGLFW_BUILD_EXAMPLES=OFF \
+            -DGLFW_BUILD_DOCS=OFF -DGLFW_INSTALL=ON \
+            -DGLFW_BUILD_WAYLAND=OFF
+          cmake --build /tmp/glfw/build
+          sudo cmake --install /tmp/glfw/build
+
       - name: Configure
         run: cmake -B build -DCMAKE_BUILD_TYPE=Debug -DENABLE_SANITIZERS=ON -DBUILD_TESTS=ON -G Ninja .
 

.github/workflows/release.yml

@@ -26,10 +26,22 @@ jobs:
         sudo apt-get install -y --no-install-recommends \
           build-essential cmake ninja-build \
           libspdlog-dev \
-          libglfw3 libglfw3-dev libvulkan-dev \
+          libvulkan-dev \
           glslang-tools glslang-dev libglm-dev \
           pkg-config \
+          libx11-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev \
           libavcodec-dev libavformat-dev libavutil-dev libswscale-dev patchelf
+    - name: Build and install GLFW 3.4 (Linux X11 platform hints required)
+      run: |
+        git clone --depth 1 --branch 3.4 https://github.com/glfw/glfw.git /tmp/glfw
+        # Build X11-only to avoid Wayland issue
+        # See issue #68: https://github.com/jamylak/vsdf/issues/68
+        cmake -S /tmp/glfw -B /tmp/glfw/build -G Ninja \
+          -DGLFW_BUILD_TESTS=OFF -DGLFW_BUILD_EXAMPLES=OFF \
+          -DGLFW_BUILD_DOCS=OFF -DGLFW_INSTALL=ON \
+          -DGLFW_BUILD_WAYLAND=OFF
+        cmake --build /tmp/glfw/build
+        sudo cmake --install /tmp/glfw/build
 
     - name: Configure
       run: cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-Wno-error=array-bounds" -G Ninja .
@@ -116,7 +128,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     env:
-      VK_ICD_FILENAMES: /usr/share/vulkan/icd.d/lvp_icd.x86_64.json
+      VK_ICD_FILENAMES: /usr/share/vulkan/icd.d/lvp_icd.json
 
     steps:
     - name: Download artifact

default.nix

@@ -16,6 +16,7 @@
 }:
 
 let
+  _ = assert lib.versionAtLeast glfw.version "3.4"; null;
   volk = fetchFromGitHub {
     owner = "zeux";
     repo = "volk";

flake.nix

@@ -31,7 +31,9 @@
         };
       in
       {
-        devShells.default = pkgs.mkShell rec {
+        devShells.default =
+          assert pkgs.lib.versionAtLeast pkgs.glfw.version "3.4";
+          pkgs.mkShell rec {
           name = "vsdf";
 
           packages = with pkgs; [

include/glfwutils.h

@@ -3,6 +3,8 @@
 
 #define GLFW_INCLUDE_NONE
 #include <GLFW/glfw3.h>
+#include <cctype>
+#include <cstdlib>
 #include <stdexcept>
 #include <string>
 
@@ -18,6 +20,41 @@ namespace glfwutils {
  * Must be called once before creating windows.
  */
 static void initGLFW() {
+#if defined(__linux__)
+#if (GLFW_VERSION_MAJOR < 3) ||                                              \
+    (GLFW_VERSION_MAJOR == 3 && GLFW_VERSION_MINOR < 4)
+#error "GLFW 3.4+ is required on Linux to force X11 via GLFW_PLATFORM. See issue #68: https://github.com/jamylak/vsdf/issues/68"
+#endif
+    const char *platformEnv = std::getenv("GLFW_PLATFORM");
+    if (platformEnv && platformEnv[0] != '\0') {
+        std::string platform = platformEnv;
+        for (char &ch : platform) {
+            ch = static_cast<char>(
+                std::tolower(static_cast<unsigned char>(ch)));
+        }
+        int platformHint = 0;
+        if (platform == "x11") {
+            platformHint = GLFW_PLATFORM_X11;
+        } else if (platform == "wayland") {
+            platformHint = GLFW_PLATFORM_WAYLAND;
+        } else if (platform == "null") {
+            platformHint = GLFW_PLATFORM_NULL;
+        } else {
+            throw std::runtime_error(
+                "Invalid GLFW_PLATFORM value on Linux: " + platform +
+                " (expected x11, wayland, null)");
+        }
+        glfwInitHint(GLFW_PLATFORM, platformHint);
+    } else {
+        // Default to X11 on Linux to avoid Wayland/libdecor ASAN leak.
+        // See issue #68: https://github.com/jamylak/vsdf/issues/68
+        // Summary: ASAN reports leaks seemingly from the Wayland decoration stack
+        // (libdecor/GTK/Pango/Fontconfig via GLFW) that persist until process exit.
+        // This is not related to render/present stalls; override with GLFW_PLATFORM
+        // if you explicitly want Wayland.
+        glfwInitHint(GLFW_PLATFORM, GLFW_PLATFORM_X11);
+    }
+#endif
     if (!glfwInit()) {
         throw std::runtime_error("Failed to initialize GLFW");
     }

linux-tests.Dockerfile

@@ -10,20 +10,33 @@ RUN apt-get update && \
     cmake \
     ninja-build \
     make \
+    git \
+    ca-certificates \
     libgtest-dev \
     libspdlog-dev \
-    libglfw3 libglfw3-dev \
     libvulkan-dev \
     glslang-tools \
     glslang-dev \
     libglm-dev \
     pkg-config \
+    libx11-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev \
     libavcodec-dev \
     libavformat-dev \
     libavutil-dev \
     libswscale-dev
     # && rm -rf /var/lib/apt/lists/*
 
+# GLFW 3.4+ is required for GLFW_PLATFORM hints on Linux.
+# Build X11-only to avoid Wayland issue
+# See issue #68: https://github.com/jamylak/vsdf/issues/68
+RUN git clone --depth 1 --branch 3.4 https://github.com/glfw/glfw.git /tmp/glfw && \
+    cmake -S /tmp/glfw -B /tmp/glfw/build -G Ninja \
+      -DGLFW_BUILD_TESTS=OFF -DGLFW_BUILD_EXAMPLES=OFF \
+      -DGLFW_BUILD_DOCS=OFF -DGLFW_INSTALL=ON \
+      -DGLFW_BUILD_WAYLAND=OFF && \
+    cmake --build /tmp/glfw/build && \
+    cmake --install /tmp/glfw/build
+
 COPY . /app
 
 RUN mkdir -p build && \