Implement dynamic timeout adjustment for workflows

Githubguy132010 · Githubguy132010 · commit f3ca0c1d157f · 2025-02-13T16:49:50.000+01:00
Related to #74 Implement dynamic timeout adjustments for workflows to accommodate varying build times. * **Scripts**: - Add `scripts/record_run_time.sh` to record the duration of each workflow run. - Add `scripts/adjust_timeout.sh` to calculate average run time and adjust the timeout value in workflow files. - Add `scripts/monitor_system_load.sh` to monitor system load and adjust the timeout value. - Add `scripts/analyze_build_complexity.sh` to analyze build complexity and adjust the timeout value. - Add `scripts/calculate_optimal_timeout.sh` to calculate the optimal timeout time using the output from other scripts. * **Workflow Files**: - Modify `.github/workflows/build-check.yaml` to include steps for recording start and end times, monitoring system load, analyzing build complexity, and adjusting timeout based on historical run times. - Modify `.github/workflows/build.yaml` to include steps for recording start and end times, monitoring system load, analyzing build complexity, and adjusting timeout based on historical run times.
diff --git a/.github/workflows/build-check.yaml b/.github/workflows/build-check.yaml
@@ -41,6 +41,22 @@ jobs:
             pacman -S --noconfirm --needed git archiso grub qemu
           "
 
+      - name: Record Start Time
+        run: |
+          ./scripts/record_run_time.sh start
+
+      - name: Monitor System Load
+        run: |
+          ./scripts/monitor_system_load.sh
+
+      - name: Analyze Build Complexity
+        run: |
+          ./scripts/analyze_build_complexity.sh
+
+      - name: Adjust Timeout Based on Historical Run Times
+        run: |
+          ./scripts/adjust_timeout.sh
+
       - name: Test Build
         id: build
         run: |
@@ -89,6 +105,10 @@ jobs:
             sha1sum \"\$iso_file\" > checksum.sha1
           "
 
+      - name: Record End Time
+        run: |
+          ./scripts/record_run_time.sh end
+
       - name: Clean Up
         if: always()
         run: |
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -32,7 +32,21 @@ jobs:
           echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
           echo "VERSION=$(date +'%Y.%m.%d')" >> $GITHUB_ENV
 
+      - name: Record Start Time
+        run: |
+          ./scripts/record_run_time.sh start
+
+      - name: Monitor System Load
+        run: |
+          ./scripts/monitor_system_load.sh
+
+      - name: Analyze Build Complexity
+        run: |
+          ./scripts/analyze_build_complexity.sh
 
+      - name: Adjust Timeout Based on Historical Run Times
+        run: |
+          ./scripts/adjust_timeout.sh
 
       - name: Set up Arch Linux Container
         run: |
@@ -171,6 +185,10 @@ jobs:
             ${{ env.WORKSPACE }}/out/*.iso
             ${{ env.WORKSPACE }}/out/*.sha*sum
 
+      - name: Record End Time
+        run: |
+          ./scripts/record_run_time.sh end
+
       - name: Clean Up
         if: always()
         run: |
diff --git a/scripts/adjust_timeout.sh b/scripts/adjust_timeout.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Script to adjust the timeout value in workflow files based on historical run times
+
+# Define the log file to store historical run times
+LOG_FILE="workflow_run_times.log"
+
+# Function to calculate the average run time from the log file
+calculate_average_run_time() {
+  if [ ! -f $LOG_FILE ]; then
+    echo "Log file not found. Using default timeout value."
+    return 0
+  fi
+
+  total_time=0
+  count=0
+
+  while read -r line; do
+    total_time=$((total_time + line))
+    count=$((count + 1))
+  done < $LOG_FILE
+
+  if [ $count -eq 0 ]; then
+    echo "No historical run times found. Using default timeout value."
+    return 0
+  fi
+
+  average_time=$((total_time / count))
+  echo "Average run time: $average_time seconds"
+  echo $average_time
+}
+
+# Function to adjust the timeout value in the workflow files
+adjust_timeout_value() {
+  average_time=$(calculate_average_run_time)
+  if [ $average_time -eq 0 ]; then
+    return
+  fi
+
+  timeout_minutes=$((average_time / 60))
+  echo "Setting timeout value to $timeout_minutes minutes"
+
+  # Update the timeout value in the workflow files
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $timeout_minutes/" .github/workflows/build-check.yaml
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $timeout_minutes/" .github/workflows/build.yaml
+}
+
+# Adjust the timeout value
+adjust_timeout_value
diff --git a/scripts/analyze_build_complexity.sh b/scripts/analyze_build_complexity.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Script to analyze build complexity and adjust the timeout value
+
+# Function to count the number of files in the build
+count_files() {
+  find . -type f | wc -l
+}
+
+# Function to count the number of lines of code in the build
+count_lines_of_code() {
+  find . -type f -name '*.sh' -o -name '*.yaml' -o -name '*.yml' -o -name '*.conf' -o -name '*.cfg' | xargs wc -l | tail -n 1 | awk '{print $1}'
+}
+
+# Function to count the number of dependencies in the build
+count_dependencies() {
+  grep -r 'dependencies:' . | wc -l
+}
+
+# Function to adjust the timeout value based on build complexity
+adjust_timeout_based_on_complexity() {
+  num_files=$(count_files)
+  num_lines_of_code=$(count_lines_of_code)
+  num_dependencies=$(count_dependencies)
+
+  echo "Number of files: $num_files"
+  echo "Number of lines of code: $num_lines_of_code"
+  echo "Number of dependencies: $num_dependencies"
+
+  complexity_score=$((num_files + num_lines_of_code + num_dependencies))
+
+  if [ $complexity_score -gt 10000 ]; then
+    echo "High build complexity detected. Increasing timeout value."
+    timeout_minutes=180
+  else
+    echo "Normal build complexity. Using default timeout value."
+    timeout_minutes=120
+  fi
+
+  # Update the timeout value in the workflow files
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $timeout_minutes/" .github/workflows/build-check.yaml
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $timeout_minutes/" .github/workflows/build.yaml
+}
+
+# Adjust the timeout value based on build complexity
+adjust_timeout_based_on_complexity
diff --git a/scripts/calculate_optimal_timeout.sh b/scripts/calculate_optimal_timeout.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Script to calculate the optimal timeout time using the output from other scripts
+
+# Function to calculate the optimal timeout time
+calculate_optimal_timeout() {
+  # Get the average run time from the log file
+  average_run_time=$(./scripts/adjust_timeout.sh)
+
+  # Get the system load adjustment
+  system_load_adjustment=$(./scripts/monitor_system_load.sh)
+
+  # Get the build complexity adjustment
+  build_complexity_adjustment=$(./scripts/analyze_build_complexity.sh)
+
+  # Calculate the optimal timeout time
+  optimal_timeout=$((average_run_time + system_load_adjustment + build_complexity_adjustment))
+
+  echo "Optimal timeout time: $optimal_timeout minutes"
+
+  # Update the timeout value in the workflow files
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $optimal_timeout/" .github/workflows/build-check.yaml
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $optimal_timeout/" .github/workflows/build.yaml
+}
+
+# Calculate the optimal timeout time
+calculate_optimal_timeout
diff --git a/scripts/monitor_system_load.sh b/scripts/monitor_system_load.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Script to monitor system load and adjust the timeout value
+
+# Function to get the current CPU usage
+get_cpu_usage() {
+  top -bn1 | grep "Cpu(s)" | \
+  sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | \
+  awk '{print 100 - $1}'
+}
+
+# Function to get the current memory usage
+get_memory_usage() {
+  free | grep Mem | awk '{print $3/$2 * 100.0}'
+}
+
+# Function to adjust the timeout value based on system load
+adjust_timeout_based_on_load() {
+  cpu_usage=$(get_cpu_usage)
+  memory_usage=$(get_memory_usage)
+
+  echo "Current CPU usage: $cpu_usage%"
+  echo "Current memory usage: $memory_usage%"
+
+  if (( $(echo "$cpu_usage > 80.0" | bc -l) )) || (( $(echo "$memory_usage > 80.0" | bc -l) )); then
+    echo "High system load detected. Increasing timeout value."
+    timeout_minutes=180
+  else
+    echo "Normal system load. Using default timeout value."
+    timeout_minutes=120
+  fi
+
+  # Update the timeout value in the workflow files
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $timeout_minutes/" .github/workflows/build-check.yaml
+  sed -i "s/timeout-minutes: [0-9]\+/timeout-minutes: $timeout_minutes/" .github/workflows/build.yaml
+}
+
+# Adjust the timeout value based on system load
+adjust_timeout_based_on_load
diff --git a/scripts/record_run_time.sh b/scripts/record_run_time.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Script to record the duration of each workflow run
+
+# Define the log file to store historical run times
+LOG_FILE="workflow_run_times.log"
+
+# Function to record the start time of the workflow
+record_start_time() {
+  START_TIME=$(date +%s)
+  echo "Workflow started at: $(date -d @$START_TIME)"
+}
+
+# Function to record the end time of the workflow and calculate the duration
+record_end_time() {
+  END_TIME=$(date +%s)
+  DURATION=$((END_TIME - START_TIME))
+  echo "Workflow ended at: $(date -d @$END_TIME)"
+  echo "Workflow duration: $DURATION seconds"
+
+  # Store the duration in the log file
+  echo $DURATION >> $LOG_FILE
+}
+
+# Check the argument passed to the script
+if [ "$1" == "start" ]; then
+  record_start_time
+elif [ "$1" == "end" ]; then
+  record_end_time
+else
+  echo "Invalid argument. Use 'start' or 'end'."
+  exit 1
+fi
