Merge pull request #4655 from crazyserver/MOBILE-4875

MOBILE-4875 behat: Improve behat parallel run detection

Author: dpalou

.github/scripts/print_behat_tags_json.sh

@@ -3,25 +3,62 @@
 # Search tags.
 declare -A tags
 
+# If the first argument is -d or --debug, enable debug output.
+debug=$([[ "$1" == "-d" || "$1" == "--debug" ]] && echo "1" || echo "")
+
+parallelrunmissing=false
 for feature in `find ./src/ -iname *.feature`
 do
     if [[ "$1" == "snapshots" ]] && ! grep -q -i "the UI should match the snapshot" "$feature"; then
         continue
     fi
-    tag=`head -n 1 $feature | sed -E s/\\\\s+.*//`
-    tags[$tag]=$tag
+
+    featuretags=`head -n 1 $feature`
+    # Count non-blank lines in feature file also ignoring comments on the start of the line.
+    lines=`grep -c -v -e '^[[:space:]]*$' -e '^[[:space:]]*#' $feature`
+    parallelruninfile=false
+    for tag in $featuretags; do
+        # Only include @app_parallel_run_ tags.
+        if [[ "$tag" =~ @app_parallel_run_ ]]; then
+            parallelruninfile=true
+            if [[ -n "${tags[$tag]}" ]]; then
+                tags[$tag]=$((${tags[$tag]} + lines))
+            else
+                tags[$tag]="$lines"
+            fi
+            break
+        fi
+    done
+
+    if [ "$parallelruninfile" = false ] ; then
+        # Invalid or missing tag in $feature
+        parallelrunmissing=true
+    fi
 done
 
 # Serialize to JSON.
-tags_json="["
-
-for tag in "${tags[@]}"
+tags_json=""
+for tag in $(printf "%s\n" "${!tags[@]}" | sort);
 do
+    if [ ! -z "$debug" ]; then
+        lines=${tags[$tag]}
+        echo "$tag - $lines lines"
+    fi
+
     tags_json+="\"$tag\","
 done
 
 tags_json="${tags_json%?}"
-tags_json+="]"
+
+# If there's any invalid tags, use the negation of all other tags.
+negation=""
+if [ "$parallelrunmissing" = true ] ; then
+    for tag in $(printf "%s\n" "${!tags[@]}" | sort);
+    do
+        negation+="~${tag}&&"
+    done
+    tags_json="${tags_json},\"${negation%??}\""
+fi
 
 # Print to console.
-echo $tags_json
+echo "[${tags_json}]"

.github/workflows/acceptance.yml

@@ -70,6 +70,7 @@ jobs:
               run: |
                   if [ -z $BEHAT_TAGS ]; then
                     tags_json=`.github/scripts/print_behat_tags_json.sh`
+                    echo "Generated tags: $tags_json"
                     echo "tags=$tags_json" >> $GITHUB_OUTPUT;
                   else
                     echo "tags=[\"$BEHAT_TAGS\"]" >> $GITHUB_OUTPUT;
@@ -181,24 +182,43 @@ jobs:
               run: moodle-plugin-ci add-config 'define("TEST_MOD_BIGBLUEBUTTONBN_MOCK_SERVER", "http://localhost:8002/hash" . sha1($CFG->wwwroot));'
 
             - name: Run Behat tests
+              id: run-behat-tests
               run: moodle-plugin-ci behat --auto-rerun 3 --profile chrome --tags="@app&&~@local&&$BEHAT_TAGS"
               env:
                   BEHAT_TAGS: ${{ matrix.tags }}
                   MOODLE_BEHAT_SELENIUM_IMAGE: selenium/standalone-chrome:latest
 
+            - name: Rename snapshot failure files
+              id: filename
+              if: ${{ failure() && steps.run-behat-tests.conclusion == 'failure' }}
+              run: |
+                  PARALLEL_NAME=$(echo "${{ matrix.tags }}" | sed 's/^@app_parallel_run_//')
+                  echo "PARALLEL_NAME=$PARALLEL_NAME" >> $GITHUB_OUTPUT
+                  echo "❌ $PARALLEL_NAME failed" >> $GITHUB_STEP_SUMMARY
+
             - name: Upload Snapshot failures
               uses: actions/upload-artifact@v4
-              if: ${{ failure() }}
+              if: ${{ failure() && steps.run-behat-tests.conclusion == 'failure' }}
+              env:
+                  PARALLEL_NAME: ${{ steps.filename.outputs.PARALLEL_NAME }}
               with:
-                  name: snapshot_failures-${{ matrix.tags }}
+                  name: snapshot_failures-${{ env.PARALLEL_NAME }}
                   path: moodle/public/local/moodleappbehat/tests/behat/snapshots/failures/*
+                  if-no-files-found: ignore
 
             - name: Upload Behat failures
               uses: actions/upload-artifact@v4
-              if: ${{ failure() }}
+              if: ${{ failure() && steps.run-behat-tests.conclusion == 'failure' }}
+              env:
+                  PARALLEL_NAME: ${{ steps.filename.outputs.PARALLEL_NAME }}
               with:
-                  name: behat_failures-${{ matrix.tags }}
+                  name: behat_failures-${{ env.PARALLEL_NAME }}
                   path: moodledata/behat_dump/*
+                  if-no-files-found: ignore
+
+            - name: Mark cancelled jobs as failed.
+              if: ${{ cancelled() }}
+              run: exit 1
 
     complete:
         runs-on: ubuntu-latest
@@ -214,5 +234,6 @@ jobs:
                   rm ./artifacts/build -rf
                   if [ -n "$(ls -A ./artifacts)" ]; then
                     echo "There were some failures in the previous jobs"
+                    echo "### ❌ There were some failures in the previous jobs" >> $GITHUB_STEP_SUMMARY
                     exit 1
                   fi

.github/workflows/testing.yml

@@ -24,13 +24,15 @@ jobs:
           result=$(cat scripts/langindex.json | grep \"TBD\" | wc -l); test $result -eq 0
           if [ $result -ne 0 ]; then
             echo "::error::There are lang strings to be decided on langindex.json"
+            echo "❌ There are lang strings to be decided on langindex.json" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
           gulp
           langcount=`jq -r '. | length' src/assets/lang/en.json`
           langindexcount=`jq -r '. | length' scripts/langindex.json`
           if [ $langcount -ne $langindexcount ]; then
             echo "::error::Lang file has $langcount while langindex $langindexcount"
+            echo "❌ Lang file has $langcount while langindex $langindexcount" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
 
@@ -49,6 +51,7 @@ jobs:
           done
           if [ $found -ne 0 ]; then
             echo "::error::Found $found missing langkeys"
+            echo "❌ Found $found missing langkeys" >> $GITHUB_STEP_SUMMARY
             exit 1
           fi
       - name: Run Linters (ignore warnings)
@@ -68,6 +71,7 @@ jobs:
           cat circular-dependencies
           lines=$(cat circular-dependencies | wc -l)
           echo "Total circular dependencies: $lines"
+          echo "⚠️ Total circular dependencies: $lines" >> $GITHUB_STEP_SUMMARY
           test $lines -eq 80
       - name: JavaScript code compatibility
         run: |
@@ -80,5 +84,6 @@ jobs:
           # acorn is used by es-check but cannot check for this feature only, so we need to check it manually.
           if grep -qE '(^|[^-])\bstatic[ ]*\{' www/*.js cordova-plugin-moodleapp/www/*.js; then
             echo "::error::Static initialization blocks are not supported in Chrome 93 and iOS 15."
+            echo "❌ Static initialization blocks are not supported in Chrome 93 and iOS 15." >> $GITHUB_STEP_SUMMARY
             exit 1
           fi

src/addons/block/activitymodules/tests/behat/block_activitymodules.feature

@@ -1,4 +1,4 @@
-@addon_block_activity_modules @app @block @block_activity_modules @javascript @lms_upto5.0
+@app_parallel_run_blocks @addon_block_activity_modules @app @block @block_activity_modules @javascript @lms_upto5.0
 Feature: Basic tests of activity modules block
 
   Background:

src/addons/block/activityresults/tests/behat/block_activityresults.feature

@@ -1,4 +1,4 @@
-@addon_block_activity_results @app @block @block_activity_results @javascript
+@app_parallel_run_blocks @addon_block_activity_results @app @block @block_activity_results @javascript
 Feature: Basic tests of activity results block
 
   Background:

src/addons/block/badges/tests/behat/block_badges.feature

@@ -1,4 +1,4 @@
-@addon_block_badges @app @block @block_badges @javascript
+@app_parallel_run_badges @addon_block_badges @app @block @block_badges @javascript
 Feature: Basic tests of badges block
 
   Background:

src/addons/block/blogmenu/tests/behat/block_blogmenu.feature

@@ -1,4 +1,4 @@
-@addon_block_blog_menu @app @block @block_blog_menu @javascript
+@app_parallel_run_blog @addon_block_blog_menu @app @block @block_blog_menu @javascript
 Feature: Basic tests of blog menu block
 
   Background:

src/addons/block/blogrecent/tests/behat/block_blogrecent.feature

@@ -1,4 +1,4 @@
-@addon_block_blog_recent @app @block @block_blog_recent @javascript
+@app_parallel_run_blog @addon_block_blog_recent @app @block @block_blog_recent @javascript
 Feature: Basic tests of blog recent block
 
   Background:

src/addons/block/blogtags/tests/behat/block_blogtags.feature

@@ -1,4 +1,4 @@
-@addon_block_blog_tags @app @block @block_blog_tags @javascript
+@app_parallel_run_blog @addon_block_blog_tags @app @block @block_blog_tags @javascript
 Feature: Basic tests of blog tags block
 
   Background:

src/addons/block/calendarmonth/tests/behat/block_calendar_month.feature

@@ -1,4 +1,4 @@
-@addon_block_calendar_month @app @block @block_calendar_month @javascript
+@app_parallel_run_calendar @addon_block_calendar_month @app @block @block_calendar_month @javascript
 Feature: View the calendar block and check it links to the calendar page
 
   Background: