Enable sanitizers for macOS, Linux, Windows (#1559)

Enable Address sanitizer as pointed by
#1557

- Add macOS, Windows and Linux sanitizer builds
- up ndk to v28
- misc build fixes
- fix reported memory leaks and bugs

Memleaks + CI issue :
#1575

Author: CedricGuillemet

.github/asan_suppress.txt

@@ -0,0 +1,7 @@
+# Suppress leaks from dbus. No direct leak visible with BN.
+# command to dump leaks and suppress annecessary :
+# ASAN_OPTIONS=detect_leaks=1 LSAN_OPTIONS=suppressions=../../../.github/asan_suppress.txt ./UnitTests
+# ASAN_OPTIONS=detect_leaks=1 LSAN_OPTIONS=suppressions=../../../.github/asan_suppress.txt xvfb-run ./Playground app:///Scripts/validation_native.js
+leak:bmalloc_allocate_casual
+leak:reallocate_for_length
+leak:dbus_message_new_empty_header

.github/jobs/android.yml

@@ -9,7 +9,10 @@ jobs:
     timeoutInMinutes: 45
     pool:
       vmImage: ${{ parameters.vmImage }}
-      
+
+    variables:
+      SANITIZER_FLAG: ${{ coalesce(replace(format('{0}', parameters.Sanitizers), 'True', 'ON'), 'OFF') }}
+
     steps:
     - template: cmake.yml
       parameters:
@@ -26,7 +29,7 @@ jobs:
           workingDirectory: 'Apps/Playground/Android'
           gradleWrapperFile: 'Apps/Playground/Android/gradlew'
           gradleOptions: '-Xmx1536m'
-          options: '-PJSEngine=${{ parameters.JSEngine }} -PARM64Only -PNDK_VERSION=$(NDK_VERSION)'
+          options: '-PJSEngine=${{ parameters.JSEngine }} -PARM64Only -PNDK_VERSION=$(NDK_VERSION) -PSANITIZERS=$(SANITIZER_FLAG)'
           publishJUnitResults: false
           tasks: 'assembleRelease'
       displayName: 'Build Playground ${{ parameters.JSEngine }}'

.github/jobs/linux.yml

@@ -4,6 +4,7 @@ parameters:
   CC: ''
   CXX: ''
   JSEngine: ''
+  enableSanitizers: false
 
 jobs:
   - job: ${{ parameters.name }}
@@ -12,6 +13,7 @@ jobs:
       vmImage: ${{ parameters.vmImage }}
 
     variables:
+      SANITIZER_FLAG: ${{ coalesce(replace(format('{0}', parameters.enableSanitizers), 'True', 'ON'), 'OFF') }}
       CC: ${{ parameters.CC }}
       CXX: ${{ parameters.CXX }}
 
@@ -22,19 +24,29 @@ jobs:
 
     - script: |
         sudo apt-get update
-        sudo apt-get install libjavascriptcoregtk-4.1-dev libgl1-mesa-dev libcurl4-openssl-dev libwayland-dev
+        sudo apt-get install libjavascriptcoregtk-4.1-dev libgl1-mesa-dev libcurl4-openssl-dev libwayland-dev clang
       displayName: 'Install packages'
 
     - script: |
-        cmake -G Ninja -B build -D JAVASCRIPTCORE_LIBRARY=/usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.1.so -D NAPI_JAVASCRIPT_ENGINE=${{ parameters.JSEngine }} -D CMAKE_BUILD_TYPE=RelWithDebInfo -D BX_CONFIG_DEBUG=ON -D CMAKE_UNITY_BUILD=$(UNITY_BUILD) -D OpenGL_GL_PREFERENCE=GLVND -D BABYLON_DEBUG_TRACE=ON
+        cmake -G Ninja -B build -D JAVASCRIPTCORE_LIBRARY=/usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.1.so -D NAPI_JAVASCRIPT_ENGINE=${{ parameters.JSEngine }} -D CMAKE_BUILD_TYPE=RelWithDebInfo -D BX_CONFIG_DEBUG=ON -D CMAKE_UNITY_BUILD=$(UNITY_BUILD) -D OpenGL_GL_PREFERENCE=GLVND -D BABYLON_DEBUG_TRACE=ON -D ENABLE_SANITIZERS=$(SANITIZER_FLAG) .
         ninja -C build
       displayName: 'Build X11'
     
+# Memory leaks on CI is disabled due to memory leaks reported with xvfb and impossible to add to ignore list.
+# See https://github.com/BabylonJS/BabylonNative/issues/1575
+
     - script: |
         cd build/Apps/Playground
+        # Command line to suppress false positive memory leaks :
+        # ASAN_OPTIONS=detect_leaks=1 LSAN_OPTIONS=suppressions=../../../.github/asan_suppress.txt xvfb-run ./Playground app:///Scripts/validation_native.js
         xvfb-run ./Playground app:///Scripts/validation_native.js
       displayName: 'Validation Tests'
 
+    - script: |
+        cd build/Apps/UnitTests
+        xvfb-run ./UnitTests
+      displayName: 'Unit Tests'
+
     - task: PublishBuildArtifacts@1
       inputs:
         artifactName: '${{ parameters.name }} Rendered Pictures'
@@ -47,8 +59,3 @@ jobs:
         pathtoPublish: 'build/Apps/Playground/Errors'
       displayName: 'Publish Tests ${{ parameters.name }} Errors'
       condition: failed()
-
-    - script: |
-        cd build/Apps/UnitTests
-        xvfb-run ./UnitTests
-      displayName: 'Unit Tests'

.github/jobs/macos.yml

@@ -1,13 +1,17 @@
 parameters:
   name: ''
   vmImage: ''
+  enableSanitizers: false
 
 jobs:
   - job: ${{ parameters.name }}
     timeoutInMinutes: 30
     pool:
       vmImage: ${{ parameters.vmImage }}
-  
+
+    variables:
+      SANITIZER_FLAG: ${{ coalesce(replace(format('{0}', parameters.enableSanitizers), 'True', 'ON'), 'OFF') }}
+
     steps:
     - template: cmake.yml
       parameters:
@@ -18,7 +22,7 @@ jobs:
       displayName: 'Select XCode $(XCODE_VERSION)'
 
     - script: |
-        cmake -G Xcode -B buildmacOS -D CMAKE_UNITY_BUILD=$(UNITY_BUILD) -D BABYLON_DEBUG_TRACE=ON
+        cmake -G Xcode -B buildmacOS -D CMAKE_UNITY_BUILD=$(UNITY_BUILD) -D BABYLON_DEBUG_TRACE=ON -D ENABLE_SANITIZERS=$(SANITIZER_FLAG)
       displayName: 'Generate macOS solution'
 
     - script: |

.github/jobs/win32.yml

@@ -15,13 +15,17 @@ parameters:
 - name: graphics_api
   type: string
   default: D3D11
+- name: enableSanitizers
+  type: boolean
+  default: false
 
 jobs:
   - job: ${{ parameters.name }}
     timeoutInMinutes: 60
     pool:
       vmImage: ${{ parameters.vmImage }}
     variables:
+      SANITIZER_FLAG: ${{ coalesce(replace(format('{0}', parameters.enableSanitizers), 'True', 'ON'), 'OFF') }}
       ${{ if eq(parameters.napiType, 'jsi') }}:
         napiSuffix: '_JSI'
         jsEngineDefine: '-DNAPI_JAVASCRIPT_ENGINE=JSI'
@@ -40,7 +44,7 @@ jobs:
 
       # BGFX_CONFIG_MAX_FRAME_BUFFERS is set so enough Framebuffers are available before V8 starts disposing unused ones
       - script: |
-          cmake -G "Visual Studio 17 2022" -B build${{ variables.solutionName }} -A ${{ parameters.platform }} ${{ variables.jsEngineDefine }} -D BX_CONFIG_DEBUG=ON -D GRAPHICS_API=${{ parameters.graphics_api }} -D CMAKE_UNITY_BUILD=$(UNITY_BUILD) -D BGFX_CONFIG_MAX_FRAME_BUFFERS=256 -D BABYLON_DEBUG_TRACE=ON
+          cmake -G "Visual Studio 17 2022" -B build${{ variables.solutionName }} -A ${{ parameters.platform }} ${{ variables.jsEngineDefine }} -D BX_CONFIG_DEBUG=ON -D GRAPHICS_API=${{ parameters.graphics_api }} -D CMAKE_UNITY_BUILD=$(UNITY_BUILD) -D BGFX_CONFIG_MAX_FRAME_BUFFERS=256 -D BABYLON_DEBUG_TRACE=ON -D ENABLE_SANITIZERS=$(SANITIZER_FLAG)
         displayName: 'Generate ${{ variables.solutionName }} solution'
     
       - task: MSBuild@1
@@ -59,9 +63,18 @@ jobs:
           reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\UnitTests.exe" /v DumpType /t REG_DWORD /d 2
           reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\UnitTests.exe" /v DumpCount /t REG_DWORD /d 1
           reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\UnitTests.exe" /v DumpFolder /t REG_SZ /d "$(Build.ArtifactStagingDirectory)/Dumps"
-
         displayName: 'Enable Crash Dumps'
 
+      - powershell: |
+          $vs = vswhere -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
+          $msvc = Get-ChildItem "$vs\VC\Tools\MSVC" | Sort-Object Name -Descending | Select-Object -First 1
+          $asanPath = "$($msvc.FullName)\bin\Hostx64\x64"
+
+          Write-Host "Adding ASAN runtime folder to PATH: $asanPath"
+          Write-Host "##vso[task.prependpath]$asanPath"
+        displayName: "Add ASAN runtime DLLs to PATH"
+        condition: ${{ parameters.enableSanitizers }}
+
       - script: |
           cd build${{ variables.solutionName }}\Apps\Playground
           cd RelWithDebInfo

Apps/Playground/X11/App.cpp

@@ -296,5 +296,6 @@ int main(int _argc, const char* const* _argv)
 
     XUnmapWindow(display, window);
     XDestroyWindow(display, window);
+    XCloseDisplay(display);
     return 0;
 }

Apps/UnitTests/X11/App.cpp

@@ -53,5 +53,7 @@ int main()
     Babylon::DebugTrace::EnableDebugTrace(true);
     Babylon::DebugTrace::SetTraceOutput([](const char* trace) { printf("%s\n", trace); fflush(stdout); });
 
-    return RunTests(config);
+    int ret = RunTests(config);
+    XCloseDisplay(display);
+    return ret;
 }