fix: reducing space consumption in github runners (#3900)

* feat: add disk space usage reporting and cleanup steps in local testing workflow

* feat: add disk space cleanup step before build in local testing workflow

* chore: adding changelog file 3900.fixed.md [dependabot-skip]

* feat: enhance disk space reporting in local testing workflow

* fix: update disk usage reporting to show current directory size

* fix: improve disk usage reporting with error handling in local testing workflow

* fix: remove redundant disk usage checks and streamline reporting in local testing workflow

* fix: replace cp with mv for log file handling in local and remote scripts

* fix: update log file handling to create directories before moving files in local and remote scripts

* fix: update tar command to remove files after compression in log collection scripts

* fix: update pytest arguments to exclude logs on passed tests in local and remote workflows

* fix: add disk usage checks for runner directories in local testing workflow

* fix: add disk usage check for CodeQL directory in local testing workflow

* fix: remove CodeQL directory after disk usage check in local testing workflow

* fix: enhance disk usage reporting and cleanup for CodeQL directory in local testing workflow

* fix: streamline disk usage reporting and cleanup for CodeQL directory in local testing workflow

---------

Co-authored-by: pyansys-ci-bot <[email protected]>

Author: germa89

.ci/collect_mapdl_logs_locals.sh

@@ -2,17 +2,18 @@
 mkdir "$LOG_NAMES" && echo "Successfully generated directory $LOG_NAMES"
 
 echo "Copying the log files..."
-cp *.log ./"$LOG_NAMES"/ || echo "No log files could be found"
-cp *apdl.out ./"$LOG_NAMES"/ || echo "No APDL log files could be found"
-cp *pymapdl.apdl ./"$LOG_NAMES"/ || echo "No PYMAPDL APDL log files could be found"
+mv *.log ./"$LOG_NAMES"/ || echo "No log files could be found"
+mv *apdl.out ./"$LOG_NAMES"/ || echo "No APDL log files could be found"
+mv *pymapdl.apdl ./"$LOG_NAMES"/ || echo "No PYMAPDL APDL log files could be found"
 
 echo "Copying the profiling files..."
-cp -r prof ./"$LOG_NAMES"/prof || echo "No profile files could be found"
+mkdir -p ./"$LOG_NAMES"/prof
+mv prof/* ./"$LOG_NAMES"/prof || echo "No profile files could be found"
 
 echo "Copying the JSONL files..."
-cp *.jsonl ./"$LOG_NAMES"/ || echo "No JSONL files could be found"
+mv *.jsonl ./"$LOG_NAMES"/ || echo "No JSONL files could be found"
 
 ls -la ./"$LOG_NAMES"
 
 echo "Tar files..."
-tar cvzf ./"$LOG_NAMES".tgz ./"$LOG_NAMES" || echo "Failed to compress"
+tar --remove-files -cvzf ./"$LOG_NAMES".tgz ./"$LOG_NAMES" || echo "Failed to compress"

.ci/collect_mapdl_logs_remote.sh

@@ -3,43 +3,41 @@ if [[ $MAPDL_VERSION == *"ubuntu"* ]] ; then
     echo "It is an ubuntu based image"
     export FILE=/jobs/file
     export WDIR='/jobs/'
-
 else
     echo "It is a CentOS based image"
     export FILE=file
     export WDIR=""
-
 fi;
 
-
 mkdir "$LOG_NAMES" && echo "Successfully generated directory $LOG_NAMES"
 
 ####
 echo "Collecting MAPDL logs..."
 
 (docker exec "$MAPDL_INSTANCE" /bin/bash -c "mkdir -p /mapdl_logs && echo 'Successfully created directory inside docker container'") || echo "Failed to create a directory inside docker container for logs."
-(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.out' > /dev/null ;then cp -f /file*.out /mapdl_logs && echo 'Successfully copied out files.'; fi") || echo "Failed to copy the 'out' files into a local file"
-(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.err' > /dev/null ;then cp -f /file*.err /mapdl_logs && echo 'Successfully copied err files.'; fi") || echo "Failed to copy the 'err' files into a local file"
-(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.log' > /dev/null ;then cp -f /file*.log /mapdl_logs && echo 'Successfully copied log files.'; fi") || echo "Failed to copy the 'log' files into a local file"
-(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$WDIR*.crash' > /dev/null ;then cp -f /*.crash /mapdl_logs && echo 'Successfully copied crash files.'; fi") || echo "Failed to copy the 'crash' files into a local file"
+(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.out' > /dev/null ;then mv -f /file*.out /mapdl_logs && echo 'Successfully moved out files.'; fi") || echo "Failed to move the 'out' files into a local file"
+(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.err' > /dev/null ;then mv -f /file*.err /mapdl_logs && echo 'Successfully moved err files.'; fi") || echo "Failed to move the 'err' files into a local file"
+(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.log' > /dev/null ;then mv -f /file*.log /mapdl_logs && echo 'Successfully moved log files.'; fi") || echo "Failed to move the 'log' files into a local file"
+(docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$WDIR*.crash' > /dev/null ;then mv -f $WDIR*.crash /mapdl_logs && echo 'Successfully moved crash files.'; fi") || echo "Failed to move the 'crash' files into a local file"
 
 docker cp "$MAPDL_INSTANCE":/mapdl_logs/. ./"$LOG_NAMES"/. || echo "Failed to copy the 'log-build-docs' files into a local directory"
 
 ####
 echo "Collecting local build logs..."
 
 echo "Collecting docker run log..."
-cp log.txt ./"$LOG_NAMES"/log.txt || echo "MAPDL run docker log not found."
+mv log.txt ./"$LOG_NAMES"/log.txt || echo "MAPDL run docker log not found."
 
-echo "Copying docker launch log..."
-cp mapdl_launch_0.log ./"$LOG_NAMES"/mapdl_launch_0.log || echo "MAPDL launch docker log not found."
-cp mapdl_launch_1.log ./"$LOG_NAMES"/mapdl_launch_1.log || echo "MAPDL launch docker log not found."
+echo "Moving docker launch log..."
+mv mapdl_launch_0.log ./"$LOG_NAMES"/mapdl_launch_0.log || echo "MAPDL launch docker log not found."
+mv mapdl_launch_1.log ./"$LOG_NAMES"/mapdl_launch_1.log || echo "MAPDL launch docker log not found."
 
-echo "Copying the profiling files..."
-cp -r prof ./"$LOG_NAMES"/prof || echo "No profile files could be found"
+echo "Moving the profiling files..."
+mkdir -p ./"$LOG_NAMES"/prof
+mv prof/* ./"$LOG_NAMES"/prof || echo "No profile files could be found"
 
-echo "Copying the JSONL files..."
-cp *.jsonl ./"$LOG_NAMES"/ || echo "No JSONL files could be found"
+echo "Moving the JSONL files..."
+mv *.jsonl ./"$LOG_NAMES"/ || echo "No JSONL files could be found"
 
 echo "Collecting file structure..."
 ls -R > ./"$LOG_NAMES"/files_structure.txt || echo "Failed to copy file structure to a file"
@@ -48,4 +46,4 @@ echo "Collecting docker file structure..."
 docker exec "$MAPDL_INSTANCE" /bin/bash -c "ls -R" > ./"$LOG_NAMES"/docker_files_structure.txt || echo "Failed to copy the docker structure into a local file"
 
 echo "Tar files..."
-tar cvzf ./"$LOG_NAMES".tgz ./"$LOG_NAMES" || echo "Failed to compress"
+tar --remove-files -cvzf ./"$LOG_NAMES".tgz ./"$LOG_NAMES" || echo "Failed to compress"

.github/workflows/test-local.yml

@@ -120,7 +120,7 @@ jobs:
       TESTING_MINIMAL: ${{ inputs.testing-minimal }}
       P_SCHEMA: "/ansys_inc/v241/ansys/ac4/schema"
       PYTEST_TIMEOUT: 120 # seconds. Limit the duration for each unit test
-      PYTEST_ARGUMENTS: '-vvv -rxXsa --color=yes --durations=30 --random-order --random-order-bucket=class --maxfail=10 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html --timeout=180 --profile-svg --profile'
+      PYTEST_ARGUMENTS: '-vvv -rxXsa --color=yes --durations=30 --random-order --random-order-bucket=class --maxfail=10 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html --timeout=180 --profile-svg --profile --report-log-exclude-logs-on-passed-tests'
       OMPI_ALLOW_RUN_AS_ROOT: 1
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
@@ -132,6 +132,15 @@ jobs:
         password: ${{ secrets.token }}
 
     steps:
+      - name: "Freeing some space and show space consumption (pre-test)"
+        shell: bash
+        run: |
+          echo "Deleting CodeQL..."
+          rm -rf /__t/CodeQL || echo "CodeQL not found"
+
+          echo "Disk space:"
+          df -h
+
       - name: "Install Git and checkout project"
         uses: actions/[email protected]
         with:
@@ -162,13 +171,13 @@ jobs:
         shell: bash
         if: inputs.testing-minimal == true
         run: |
-          apt-get update && apt install -y libgomp1 graphviz
+          apt-get update && apt install -y libgomp1 graphviz && apt-get clean
 
       - name: "Installing OS packages"
         shell: bash
         if: inputs.testing-minimal == false
         run: |
-          apt-get update && apt install -y libgl1-mesa-glx xvfb libgomp1 graphviz
+          apt-get update && apt install -y libgl1-mesa-glx xvfb libgomp1 graphviz && apt-get clean
 
       - name: "Setup Python"
         uses: actions/setup-python@v5
@@ -289,3 +298,9 @@ jobs:
           name: ${{ inputs.file-name }}.xml
           path: ./${{ inputs.file-name }}.xml
 
+      - name: "Show space consumption (post-test)"
+        if: always()
+        shell: bash
+        run: |
+          echo "Disk space:"
+          df -h

.github/workflows/test-remote.yml

@@ -63,10 +63,19 @@ jobs:
       PYMAPDL_DB_PORT2: 21003  # default won't work on GitHub runners
       DPF_PORT: 21004
       DPF_START_SERVER: False
-      PYTEST_ARGUMENTS: '-vvv -rxXsa --color=yes --durations=30 --random-order --random-order-bucket=class --maxfail=10 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html --timeout=180 --profile-svg --profile'
+      PYTEST_ARGUMENTS: '-vvv -rxXsa --color=yes --durations=30 --random-order --random-order-bucket=class --maxfail=10 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html --timeout=180 --profile-svg --profile --report-log-exclude-logs-on-passed-tests'
       MAPDL_PACKAGE: ghcr.io/ansys/mapdl
 
     steps:
+      - name: "Freeing some space and show space consumption (pre-test)"
+        shell: bash
+        run: |
+          echo "Deleting CodeQL..."
+          rm -rf /__t/CodeQL || echo "CodeQL not found"
+
+          echo "Disk space:"
+          df -h
+
       - name: "Install Git and checkout project"
         uses: actions/[email protected]
 
@@ -263,4 +272,11 @@ jobs:
           LOG_NAMES: logs-${{ inputs.file-name }}
         shell: bash
         run: |
-          .ci/display_logs_remote.sh
+          .ci/display_logs_remote.sh
+
+      - name: "Show space consumption (post-test)"
+        if: always()
+        shell: bash
+        run: |
+          echo "Disk space:"
+          df -h

doc/changelog.d/3900.fixed.md

@@ -0,0 +1 @@
+fix: reducing space consumption in GitHub runners