Merge remote-tracking branch 'origin/rc/barnard' into update-vm-barnard

laurci · laurci · commit 58aad139c7fb · 2025-06-24T07:50:56.000Z
diff --git a/.github/workflows/build_and_run_chain_simulator_and_execute_system_test.yml b/.github/workflows/build_and_run_chain_simulator_and_execute_system_test.yml
@@ -117,6 +117,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: 1.23.6
+          cache: false
         id: go
 
       - name: Checkout mx-chain-go
@@ -142,7 +143,17 @@ jobs:
           TS=$(date +'%Y_%^B_%d__%H_%M_%S'); 
           echo "TS=$TS" >> "$GITHUB_ENV"
           echo "TIMESTAMP=$TS" >> "$GITHUB_ENV"
+
+          # Fix any interrupted dpkg operations first
+          sudo dpkg --configure -a || true
+
+          # Clean up any partial packages
+          sudo apt-get clean
+          sudo apt-get autoclean
+
+          # Update package lists and install rclone
           sudo apt-get update -y && sudo apt-get install -y rclone
+
           mkdir -p ~/.config/rclone
           cat > ~/.config/rclone/rclone.conf <<EOF
           [r2]
@@ -276,17 +287,55 @@ jobs:
         run: |
           set +e
           pytest mx-chain-testing-suite/scenarios/ --html=report.html --self-contained-html --continue-on-collection-errors
-          PYTEST_EXIT_CODE=$?
+          INITIAL_PYTEST_EXIT_CODE=$?
           set -e
-          echo "PYTEST_EXIT_CODE=$PYTEST_EXIT_CODE" >> $GITHUB_ENV
-          echo "Pytest exit code: $PYTEST_EXIT_CODE"
+
+          echo "Initial pytest exit code: $INITIAL_PYTEST_EXIT_CODE"
+
+          # Parse the HTML report to determine actual test status
+          FINAL_PYTEST_EXIT_CODE=$INITIAL_PYTEST_EXIT_CODE
+
           if [ -f "report.html" ]; then
             echo "Report generated successfully."
+
+            # Extract test summary from HTML report using multiple parsing approaches
+            FAILED_TESTS=$(grep -E '[0-9]+ Failed|Failed.*[0-9]+' report.html | head -1 | grep -o '[0-9]\+' | head -1 || echo "0")
+            ERROR_TESTS=$(grep -E '[0-9]+ Errors|Errors.*[0-9]+' report.html | head -1 | grep -o '[0-9]\+' | head -1 || echo "0")
+            RERUN_TESTS=$(grep -E '[0-9]+ Reruns|Reruns.*[0-9]+' report.html | head -1 | grep -o '[0-9]\+' | head -1 || echo "0")
+
+            # Fallback: check for "0 Failed" pattern directly
+            if grep -q "0 Failed" report.html && [ "$INITIAL_PYTEST_EXIT_CODE" -ne 0 ]; then
+              echo "Detected '0 Failed' pattern in report despite non-zero exit code - likely successful reruns"
+              FAILED_TESTS=0
+            fi
+
+            echo "Failed tests: $FAILED_TESTS"
+            echo "Error tests: $ERROR_TESTS" 
+            echo "Rerun tests: $RERUN_TESTS"
+
+            # Determine final status based on actual test results
+            if [ "$FAILED_TESTS" -eq 0 ] && [ "$ERROR_TESTS" -eq 0 ]; then
+              if [ "$RERUN_TESTS" -gt 0 ]; then
+                echo "✅ All tests passed after $RERUN_TESTS reruns - considering as SUCCESS"
+              else
+                echo "✅ All tests passed on first attempt"
+              fi
+              FINAL_PYTEST_EXIT_CODE=0
+            else
+              echo "❌ Tests have genuine failures: $FAILED_TESTS failed, $ERROR_TESTS errors"
+              FINAL_PYTEST_EXIT_CODE=1
+            fi
+
+            # Move report to reports directory
             mkdir -p ./reports
             mv report.html ./reports/
           else
-            echo "Report not found."
+            echo "❌ Report not found - using original exit code"
+            FINAL_PYTEST_EXIT_CODE=$INITIAL_PYTEST_EXIT_CODE
           fi
+
+          echo "PYTEST_EXIT_CODE=$FINAL_PYTEST_EXIT_CODE" >> $GITHUB_ENV
+          echo "Final pytest exit code: $FINAL_PYTEST_EXIT_CODE"
           
       - name: Stage report for R2
         run: |
@@ -362,8 +411,11 @@ jobs:
             done
             echo "</ul></body></html>" >> index.html
             git add index.html
-            git commit -m "Update Index of Reports"
-            git push origin gh-pages --force
+            if git commit -m "Update Index of Reports"; then
+              git push origin gh-pages --force
+            else
+              echo "No changes to commit. Nothing to push."
+            fi
           else
             mkdir -p docs
             cd docs
@@ -479,4 +531,80 @@ jobs:
             exit 1
           else
             echo "Tests passed successfully."
-          fi
+          fi
+          
+
+      - name: Cleanup Workspace and Processes
+        if: always()
+        run: |
+          echo "🧹 Starting SAFE cleanup for job ${{ github.run_id }}..."
+          
+          # Get current working directory to limit cleanup scope
+          WORKSPACE_DIR=$(pwd)
+          echo "Workspace directory: $WORKSPACE_DIR"
+          
+          # 1. Kill only OUR ChainSimulator processes (by checking working directory)
+          echo "Killing ChainSimulator processes from this workspace..."
+          for pid in $(pgrep -f "chainsimulator"); do
+            if [ -n "$pid" ]; then
+              CWD=$(pwdx $pid 2>/dev/null | awk '{print $2}' || echo "")
+              if [[ "$CWD" == *"$WORKSPACE_DIR"* ]]; then
+                echo "Killing chainsimulator process $pid from our workspace"
+                kill -TERM $pid 2>/dev/null || true
+                sleep 2
+                kill -9 $pid 2>/dev/null || true
+              fi
+            fi
+          done
+          
+          # 2. Kill only OUR pytest processes (by checking if they're in our workspace)
+          echo "Killing pytest processes from this workspace..."
+          for pid in $(pgrep -f "pytest.*scenarios"); do
+            if [ -n "$pid" ]; then
+              CWD=$(pwdx $pid 2>/dev/null | awk '{print $2}' || echo "")
+              if [[ "$CWD" == *"$WORKSPACE_DIR"* ]]; then
+                echo "Killing pytest process $pid from our workspace"
+                kill -TERM $pid 2>/dev/null || true
+              fi
+            fi
+          done
+          
+          # 3. Remove only OUR temporary files (with unique identifiers)
+          echo "Removing temporary files from this job..."
+          rm -f /tmp/chainsim_init_${{ github.run_id }}.log || true
+          rm -f /tmp/chain_simulator_initialized_${{ github.run_id }}.lock || true
+          # Only remove temp files that might be from this job (be very specific)
+          rm -f /tmp/chainsim_init.log || true  # Only if we created it without job ID
+          rm -f /tmp/chain_simulator_initialized.lock || true  # Only if we created it without job ID
+          
+          # 4. Clean up ONLY our workspace directories
+          echo "Cleaning up workspace directories..."
+          rm -rf "$WORKSPACE_DIR/mx-chain-go" || true
+          rm -rf "$WORKSPACE_DIR/mx-chain-simulator-go" || true
+          rm -rf "$WORKSPACE_DIR/mx-chain-testing-suite" || true
+          rm -rf "$WORKSPACE_DIR/reports" || true
+          rm -rf "$WORKSPACE_DIR/r2_upload" || true
+          rm -f "$WORKSPACE_DIR/report.html" || true
+          
+          # 5. Clean up Python cache ONLY in our workspace
+          echo "Cleaning Python cache in workspace..."
+          find "$WORKSPACE_DIR" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+          find "$WORKSPACE_DIR" -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true
+          find "$WORKSPACE_DIR" -type f -name "*.pyc" -delete 2>/dev/null || true
+
+          # 6. Check disk space (informational only)
+          echo "Checking remaining disk space..."
+          df -h "$WORKSPACE_DIR" || true
+          
+          # 7. Show what processes are still running FROM OUR WORKSPACE (for debugging)
+          echo "Checking for remaining processes from our workspace:"
+          for pid in $(pgrep -f "chainsimulator\|pytest.*scenarios"); do
+            if [ -n "$pid" ]; then
+              CWD=$(pwdx $pid 2>/dev/null | awk '{print $2}' || echo "unknown")
+              if [[ "$CWD" == *"$WORKSPACE_DIR"* ]]; then
+                echo "⚠️ Process $pid still running from our workspace: $CWD"
+              fi
+            fi
+          done
+          
+          echo "✅ Safe cleanup completed for job ${{ github.run_id }}!"
diff --git a/.github/workflows/build_and_test_on_macos.yml b/.github/workflows/build_and_test_on_macos.yml
@@ -17,6 +17,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: 1.23.6
+          cache: false
         id: go
 
       - name: Check out code into the Go module directory
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
@@ -20,6 +20,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: 1.23.6
+          cache: false
         id: go
 
       - name: Check out code into the Go module directory
diff --git a/testscommon/dataRetriever/poolFactory.go b/testscommon/dataRetriever/poolFactory.go
@@ -2,7 +2,6 @@ package dataRetriever
 
 import (
 	"fmt"
-	"os"
 	"time"
 
 	"github.com/multiversx/mx-chain-core-go/marshal"
@@ -15,8 +14,8 @@ import (
 	"github.com/multiversx/mx-chain-go/dataRetriever/shardedData"
 	"github.com/multiversx/mx-chain-go/dataRetriever/txpool"
 	"github.com/multiversx/mx-chain-go/storage/cache"
-	storageFactory "github.com/multiversx/mx-chain-go/storage/factory"
 	"github.com/multiversx/mx-chain-go/storage/storageunit"
+	"github.com/multiversx/mx-chain-go/testscommon"
 	"github.com/multiversx/mx-chain-go/testscommon/txcachemocks"
 	"github.com/multiversx/mx-chain-go/trie/factory"
 )
@@ -84,32 +83,18 @@ func createPoolHolderArgs(numShards uint32, selfShard uint32) dataPool.DataPoolA
 
 	cacherConfig = storageunit.CacheConfig{Capacity: 50000, Type: storageunit.LRUCache}
 	cacher, err := cache.NewCapacityLRU(10, 10000)
-	panicIfError("Create trieSync cacher", err)
-
-	tempDir, _ := os.MkdirTemp("", "integrationTests")
-
-	dbConfig := config.DBConfig{
-		FilePath:          tempDir,
-		Type:              string(storageunit.LvlDBSerial),
-		BatchDelaySeconds: 4,
-		MaxBatchSize:      10000,
-		MaxOpenFiles:      10,
-	}
-
-	persisterFactory, err := storageFactory.NewPersisterFactory(dbConfig)
-	panicIfError("Create persister factory", err)
+	panicIfError("CreatePoolsHolder", err)
 
-	persister, err := persisterFactory.CreateWithRetries(tempDir)
-	panicIfError("Create trieSync DB", err)
+	db := testscommon.NewMemDbMock()
 	tnf := factory.NewTrieNodeFactory()
 
 	adaptedTrieNodesStorage, err := storageunit.NewStorageCacherAdapter(
 		cacher,
-		persister,
+		db,
 		tnf,
 		&marshal.GogoProtoMarshalizer{},
 	)
-	panicIfError("Create AdaptedTrieNodesStorage", err)
+	panicIfError("CreatePoolsHolder", err)
 
 	trieNodesChunks, err := storageunit.NewCache(cacherConfig)
 	panicIfError("CreatePoolsHolder", err)
