Fix & improve test worlds (#5996)

* Backport

* test

* test

* test

* Faster test

* Fix

* Test

* Fix

* Fix

* Fix

* Restore full test

* Fix worlds

* Cleanup

* Remove checkout

* Fixes

* use diff rather than git diff

* Fix branch name

* Show error

* Faster failure

* Install runtime dependencies

* Fix

* Fix

* Finalize

* Cleanup

Author: ad-daniel

.github/workflows/test_suite_linux.yml

@@ -127,7 +127,6 @@ jobs:
         name: test-results-${{ matrix.os }}
         path: |
           tests/
-
   test-ros:
     needs: build
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test ros') || github.event_name == 'schedule' }}
@@ -171,44 +170,33 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test worlds') || github.event_name == 'schedule' }}
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
     - name: Download Artifacts
       uses: actions/download-artifact@v2
       with:
         name: build-ubuntu-20.04
         path: artifact
+    - name: Get branch name
+      id: branch-name
+      uses: tj-actions/[email protected]
     - name: Extract Webots and Cache
       run: |
         tar xjf artifact/webots-*-x86-64*.tar.bz2 -C artifact
+        cp -R artifact/ untouched-artifact/
         mkdir -p ~/.cache/Cyberbotics/Webots/
         unzip -q artifact/assets-*.zip -d ~/.cache/Cyberbotics/Webots/assets
-    - name: Install Webots Dependencies
-      run: sudo scripts/install/linux_runtime_dependencies.sh
     - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
     - name: Update World, Check Warnings and Validate Cache
       run: |
-        git config --global user.name ${GITHUB_ACTOR}
-        git config --global user.email ${GITHUB_ACTOR}@github.com
+        wget https://raw.githubusercontent.com/cyberbotics/webots/${{ steps.branch-name.outputs.current_branch }}/scripts/install/linux_runtime_dependencies.sh && sudo bash linux_runtime_dependencies.sh
         export LIBGL_ALWAYS_SOFTWARE=true
         export WEBOTS_DISABLE_SAVE_SCREEN_PERSPECTIVE_ON_CLOSE=true
-        xvfb-run --auto-servernum python scripts/packaging/update_urls.py $(git log -1 --format='%H')
-        echo "Updated urls with commit $(git log -1 --format='%H')"
-        git add -u && git commit -m "Fake-commit the updated urls so that no diff exists prior to running the test"
         export WEBOTS_HOME=$PWD/artifact/webots
-        if [[ -d ~/.cache/Cyberbotics/Webots/assets/ ]]; then cached_files_before=$(ls ~/.cache/Cyberbotics/Webots/assets/ | wc -l); else cached_files_before=0; fi
-        xvfb-run --auto-servernum python tests/test_worlds.py
-        if [[ "$(git diff --name-only *.wbt *.wbproj | wc -l)" -ne "0" ]]; then echo Some world and/or wbproj files are not up to date: "$(git diff *.wbt *.wbproj)"; exit -1; fi
-        if [[ -d ~/.cache/Cyberbotics/Webots/assets/ ]]; then cached_files_after=$(ls ~/.cache/Cyberbotics/Webots/assets/ | wc -l); else cached_files_after=0; fi
-        # Note: need an offset of 22 because world 'url.wbt' uses non-cached assets on purpose
-        if [ "$cached_files_after" -ne "$((cached_files_before + 22))" ]; then
-          echo "Some assets are not included in the cache (before $cached_files_before, after $cached_files_after)";
-          echo "Asset log content is:"
-          cat ~/.cache/Cyberbotics/Webots/assets/log.txt
-          exit 1
-        fi
+        wget https://raw.githubusercontent.com/cyberbotics/webots/${{ steps.branch-name.outputs.current_branch }}/tests/test_worlds.py
+        xvfb-run --auto-servernum python3 test_worlds.py
+        if [[ "$(diff -qr artifact/ untouched-artifact/ | wc -l)" -ne "0" ]]; then echo Some world and/or wbproj files are not up to date: "$(diff -qr artifact/ untouched-artifact/)"; exit 1; fi
   delete-artifacts:
     needs: [build, test-suite, test-ros, test-worlds]
     if: ${{ always() && !contains(github.event.pull_request.labels.*.name, 'test distribution')  && !contains(github.event.pull_request.labels.*.name, 'test webots build') }}

.github/workflows/test_suite_linux_develop.yml

@@ -163,43 +163,33 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test worlds') || github.event_name == 'schedule' }}
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
-      with:
-         ref: develop
     - name: Download Artifacts
       uses: actions/download-artifact@v2
       with:
         name: build-ubuntu-20.04
         path: artifact
+    - name: Get branch name
+      id: branch-name
+      uses: tj-actions/[email protected]
     - name: Extract Webots and Cache
       run: |
         tar xjf artifact/webots-*-x86-64*.tar.bz2 -C artifact
+        cp -R artifact/ untouched-artifact/
         mkdir -p ~/.cache/Cyberbotics/Webots/
         unzip -q artifact/assets-*.zip -d ~/.cache/Cyberbotics/Webots/assets
-    - name: Install Webots Dependencies
-      run: sudo scripts/install/linux_runtime_dependencies.sh
     - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
     - name: Update World, Check Warnings and Validate Cache
       run: |
-        git config --global user.name ${GITHUB_ACTOR}
-        git config --global user.email ${GITHUB_ACTOR}@github.com
+        wget https://raw.githubusercontent.com/cyberbotics/webots/${{ steps.branch-name.outputs.current_branch }}/scripts/install/linux_runtime_dependencies.sh && sudo bash linux_runtime_dependencies.sh
         export LIBGL_ALWAYS_SOFTWARE=true
         export WEBOTS_DISABLE_SAVE_SCREEN_PERSPECTIVE_ON_CLOSE=true
-        xvfb-run --auto-servernum python scripts/packaging/update_urls.py $(git log -1 --format='%H')
-        git add -u && git commit -m "Fake-commit the updated urls so that no diff exists prior to running the test"
         export WEBOTS_HOME=$PWD/artifact/webots
-        if [[ -d ~/.cache/Cyberbotics/Webots/assets/ ]]; then cached_files_before=$(ls ~/.cache/Cyberbotics/Webots/assets/ | wc -l); else cached_files_before=0; fi
-        xvfb-run --auto-servernum python tests/test_worlds.py
-        if [[ "$(git diff --name-only *.wbt *.wbproj | wc -l)" -ne "0" ]]; then echo Some world and/or wbproj files are not up to date: "$(git diff *.wbt *.wbproj)"; exit -1; fi
-        if [[ -d ~/.cache/Cyberbotics/Webots/assets/ ]]; then cached_files_after=$(ls ~/.cache/Cyberbotics/Webots/assets/ | wc -l); else cached_files_after=0; fi
-        # Note: need an offset of 22 because world 'url.wbt' uses non-cached assets on purpose
-        if [ "$cached_files_after" -ne "$((cached_files_before + 22))" ]; then
-          echo "Some assets are not included in the cache (before $cached_files_before, after $cached_files_after)";
-          exit 1
-        fi
+        sudo apt install -y wget && wget https://raw.githubusercontent.com/cyberbotics/webots/${{ steps.branch-name.outputs.current_branch }}/tests/test_worlds.py
+        xvfb-run --auto-servernum python3 test_worlds.py
+        if [[ "$(diff -qr artifact/ untouched-artifact/ | wc -l)" -ne "0" ]]; then echo Some world and/or wbproj files are not up to date: "$(diff -qr artifact/ untouched-artifact/)"; exit 1; fi
   delete-artifacts:
     needs: [build, test-suite, test-ros, test-worlds]
     if: ${{ always() && !contains(github.event.pull_request.labels.*.name, 'test distribution')  && !contains(github.event.pull_request.labels.*.name, 'test webots build') }}

projects/robots/mobsya/thymio/worlds/thymio2.wbt

@@ -28,7 +28,5 @@ TexturedBackgroundLight {
 CircleArena {
 }
 Thymio2 {
-  controller "thymio2_demo"
-  controllerArgs []
   window "thymio2"
 }

projects/robots/mobsya/thymio/worlds/thymio2_ball.wbt

@@ -30,7 +30,6 @@ CircleArena {
 }
 Thymio2 {
   translation 0 0 1
-  controller "thymio2_demo"
   window "thymio2"
 }
 Thymio2Ball {

projects/robots/mobsya/thymio/worlds/thymio2_pen.wbt

@@ -39,7 +39,6 @@ RectangleArena {
   }
 }
 Thymio2 {
-  controller "thymio2_demo"
   window "thymio2"
   bodySlot [
     Thymio2Pen {

projects/samples/robotbenchmark/maze_runner/worlds/maze_runner.wbt

@@ -242,7 +242,6 @@ DEF THYMIO2 Thymio2 {
   translation -0.279509 0.02923 0.610695
   rotation 0.0014800006661173696 0.9997794499798363 0.020949009428711335 -0.014665
   controller "move_in_maze"
-  controllerArgs []
 }
 MazeRunnerBenchmark {
 }

projects/samples/robotbenchmark/obstacle_avoidance/worlds/obstacle_avoidance.wbt

@@ -58,7 +58,6 @@ DEF THYMIO2 Thymio2 {
   translation 0 -2.68 0
   rotation 0 0 -1 -1.570795
   controller "avoid_obstacles"
-  controllerArgs []
   bodySlot [
     Compass {
     }

projects/samples/robotbenchmark/robot_programming/worlds/robot_programming.wbt

@@ -38,7 +38,6 @@ CircleArena {
 DEF THYMIO2 Thymio2 {
   rotation 0 0 1 3.14159
   controller "move"
-  controllerArgs []
 }
 RobotProgrammingBenchmark {
 }

tests/test_worlds.py

@@ -19,11 +19,19 @@
 import os
 import fnmatch
 import sys
+import platform
 from subprocess import Popen, PIPE, TimeoutExpired
 
 if sys.version_info[0] != 3 or sys.version_info[1] < 7:
     sys.exit('This script requires Python version 3.7')
 
+if platform.system() == 'Darwin':
+    CACHE_DIR = os.path.join(os.path.expanduser('~'), 'Library', 'Caches', 'Cyberbotics', 'Webots', 'assets')
+elif platform.system() == 'Linux':
+    CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'Cyberbotics', 'Webots', 'assets')
+elif platform.system() == 'Windows':
+    CACHE_DIR = os.path.join(os.path.expanduser('~'), 'AppData', 'Local', 'Cyberbotics', 'Webots', 'cache', 'assets')
+
 
 class TestWorldsWarnings(unittest.TestCase):
     """Unit test of the worlds."""
@@ -64,10 +72,12 @@ def setUp(self):
                 sys.exit(1)
             self.webotsFullPath = os.path.normpath(self.webotsFullPath)
 
-    def test_worlds_warnings(self):
-        """Test all the 'projects' worlds for loading warnings."""
+    def test_worlds_warnings_and_cache(self):
+        """Test all the 'projects' worlds for loading warnings and if they reference un-cached assets."""
         problematicWorlds = []
         crashedWorlds = []
+        worldsWithNonCachedAssets = []
+        cacheSizeBefore = len(os.listdir(CACHE_DIR))
         for i in range(len(self.worlds)):
             print('Testing: %d/%d: %s' % (i + 1, len(self.worlds), self.worlds[i]))
             self.process = Popen([
@@ -93,13 +103,34 @@ def test_worlds_warnings(self):
                 problematicWorlds.append(self.worlds[i])
             if errors and self.crashError in str(errors):
                 crashedWorlds.append(self.worlds[i])
+
+            # url.wbt and camera.wbt are exceptional as they contain non-cached assets on purpose, adapt cache size accordingly
+            worldName = os.path.basename(self.worlds[i])
+            if worldName == 'url.wbt':
+                cacheSizeBefore += 22
+            if worldName == 'camera.wbt':
+                cacheSizeBefore += 26
+
+            cacheSizeAfter = len(os.listdir(CACHE_DIR))
+            if cacheSizeAfter - cacheSizeBefore > 0:
+                worldsWithNonCachedAssets.append(self.worlds[i])
+                cacheSizeBefore = cacheSizeAfter  # update the counter so it doesn't trigger again for the same reason
+
         if crashedWorlds:
             print('\n\t'.join(['Impossible to test the following worlds because they crash when loading:'] + crashedWorlds))
         self.assertTrue(
             not problematicWorlds,
             msg='\n\t'.join(['The following worlds have unwanted warnings:'] + problematicWorlds)
         )
 
+        if worldsWithNonCachedAssets:
+            print('\nThe following worlds reference non-cached assets:')
+            for world in worldsWithNonCachedAssets:
+                print('- ' + world)
+
+        if crashedWorlds or problematicWorlds or worldsWithNonCachedAssets:
+            self.fail('Problematic worlds have been found.')
+
 
 if __name__ == '__main__':
     unittest.main()