Fix-test-suite-not-run-on-macos (#6631)

Fixes to run test_suite.yml during macos CI (if "test suite mac" label is present).

* Make sure the Contents/Frameworks folder is created.

* Set WEBOTS_HOME_PATH such that tests can find Makefile.include during CI.

* Change WEBOTS_HOME to Webots.app and add logic to test_suite.py to find webots executable.

* Get qt_warning_filters.conf from correct location on macos.

* Ignore (but warn about) error trying to connect to background Webots on MacOS.

* On Mac, timeout test after 60 minutes instead of 10.

* Use macos-13 runner instead of macos-14. The former has twice as much RAM as the latter.

* Install urdfdom during macos CI because it is needed by tests suite.

* Assert that move succeeds.

* Print age of ode_tmp.dif so we can see how long we should wait on macos during CI.

* Allow up to 20 seconds for world reload.

* Skip billboard.wbt test during macos CI. The test passes locally, so the failure is presumably due to the CI server being headless.

* Only increase timeout on macos.

---------

Co-authored-by: CoolSpy3 <[email protected]>
Co-authored-by: Olivier Michel <[email protected]>

Author: 3 people

.github/workflows/test_suite_mac.yml

@@ -19,7 +19,7 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test webots build') }}
     strategy:
       matrix:
-        os: [macos-14]
+        os: [macos-13]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
@@ -33,7 +33,7 @@ jobs:
     if: ${{ github.event_name == 'push' || github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'test distribution') || contains(github.event.pull_request.labels.*.name, 'test suite') }}
     strategy:
       matrix:
-        os: [macos-14]
+        os: [macos-13]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
@@ -67,10 +67,10 @@ jobs:
           distribution/*.dmg
   test-suite:
     needs: build
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'test suite') }}
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'test suite mac') }}
     strategy:
       matrix:
-        os: [macos-14]
+        os: [macos-13]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
@@ -89,17 +89,21 @@ jobs:
       with:
         python-version: 3.11
     - name: Test
-      run:
+      run: |
+        brew install urdfdom
         export WEBOTS_DISABLE_SAVE_SCREEN_PERSPECTIVE_ON_CLOSE=true
+        python scripts/packaging/update_urls.py $(git log -1 --format='%H')
         export WEBOTS_HOME=/Applications/Webots.app
-        # we currently can't run the test suite because of https://github.com/cyberbotics/webots/issues/2110
-        # python tests/test_suite.py
+        export TESTS_HOME=$PWD # required by cache group in the test suite
+        export WEBOTS_HOME_PATH=$PWD # required for tests to find Makefile.include
+        export BRANCH_HASH=$(git log -1 --format='%H')
+        python tests/test_suite.py
   delete-artifacts:
     needs: [build, test-suite]
     if: ${{ always() && !contains(github.event.pull_request.labels.*.name, 'test distribution')  && !contains(github.event.pull_request.labels.*.name, 'test webots build') }}
     strategy:
       matrix:
-        os: [macos-14]
+        os: [macos-13]
     runs-on: ubuntu-latest
     steps:
     - name: Delete artifacts

src/webots/gui/main.cpp

@@ -147,8 +147,12 @@ int main(int argc, char *argv[]) {
 #endif
     );
 
-  // load qt warning filters from file
+#ifdef __APPLE__
+  QString qtFiltersFilePath = QDir::fromNativeSeparators(webotsDirPath + "/Contents/Resources/qt_warning_filters.conf");
+#else
   QString qtFiltersFilePath = QDir::fromNativeSeparators(webotsDirPath + "/resources/qt_warning_filters.conf");
+#endif
+  // load qt warning filters from file
   QFile qtFiltersFile(qtFiltersFilePath);
   if (qtFiltersFile.open(QIODevice::ReadOnly)) {
     QTextStream in(&qtFiltersFile);

tests/default/controllers/test_suite_supervisor/test_suite_supervisor.py

@@ -240,6 +240,9 @@ def run(self):
 
         # 30 seconds before executing the next world, 60 seconds for the robot_window_html test
         delay = 60 if self.currentSimulationFilename.endswith('/robot_window_html.wbt') else 30
+        if sys.platform == "darwin":
+            # Longer timeout on MacOS because Webots takes longer to start there during CI.
+            delay *= 10
         timeout = time.time() + delay
 
         running_controllers_pid = []

tests/physics/controllers/determinism/determinism.c

@@ -28,8 +28,8 @@ int main(int argc, char **argv) {
 
   if (file_exists(ode_tmp_dif)) {
     double delta = difftime(now, file_get_creation_time(ode_tmp_dif));
-    // printf("delta time = %f\n", delta);
-    if (delta > 2.0) {
+    // printf("%s is %f seconds old.\n", ode_tmp_dif, delta);
+    if (delta > 20.0) {
       ts_assert_boolean_equal(remove_file(ode_tmp_dif), "Cannot remove ODE tmp dif file");
       ts_assert_boolean_not_equal(file_exists(ode_tmp_dif), "Cannot remove ODE tmp dif file");
     }
@@ -60,7 +60,7 @@ int main(int argc, char **argv) {
           exit(EXIT_FAILURE);
         }
       } else {
-        move_file(ode_dif, ode_tmp_dif);
+        ts_assert_boolean_equal(move_file(ode_dif, ode_tmp_dif), "Cannot move ODF dif file to ODE tmp dif file");
         wb_supervisor_world_reload();
         wb_robot_cleanup();
         exit(EXIT_SUCCESS);

tests/test_suite.py

@@ -86,6 +86,8 @@ def setupWebots():
 
     if sys.platform == 'win32':
         webotsFullPath = os.path.join(os.path.normpath(os.environ['WEBOTS_HOME']), 'msys64', 'mingw64', 'bin', 'webots.exe')
+    elif sys.platform == 'darwin':
+        webotsFullPath = os.path.join(os.path.normpath(os.environ['WEBOTS_HOME']), 'Contents', 'MacOS', 'webots')
     else:
         webotsBinary = 'webots'
         if 'WEBOTS_HOME' in os.environ:
@@ -198,12 +200,14 @@ def generateWorldsList(groupName):
         for filename in filenames:
             # speaker test not working on github action because of missing sound drivers
             # robot window and movie recording test not working on BETA Ubuntu 22.04 GitHub Action environment
+            # billboard test not working in macos GitHub Action environment
             if (not filename.endswith('_temp.wbt') and
                     not ('GITHUB_ACTIONS' in os.environ and (
                         filename.endswith('speaker.wbt') or
                         filename.endswith('local_proto_with_texture.wbt') or
                         (filename.endswith('robot_window_html.wbt') and is_ubuntu_22_04) or
-                        (filename.endswith('supervisor_start_stop_movie.wbt') and is_ubuntu_22_04)
+                        (filename.endswith('supervisor_start_stop_movie.wbt') and is_ubuntu_22_04) or
+                        (filename.endswith('billboard.wbt') and sys.platform == 'darwin')
                     ))):
                 worldsList.append(filename)
 
@@ -244,7 +248,11 @@ def runGroupTest(groupName, firstSimulation, worldsCount, failures):
     command = Command(webotsArguments)
 
     # redirect stdout and stderr to files
-    command.runTest(timeout=10 * 60)  # 10 minutes
+    timeoutMinutes = 10
+    if sys.platform == "darwin":
+        # Longer timeout on MacOS because Webots takes longer to start there during CI.
+        timeoutMinutes = 60
+    command.runTest(timeout=timeoutMinutes * 60)
 
     if command.isTimeout or command.returncode != 0:
         if command.isTimeout: