ci: improve Hugo process management to address EPIPE errors in Github Actions. - Proactive monitoring: Detecting when Hugo dies

jstirnaman · jstirnaman · commit 215ecfb7f9ad · 2025-07-28T17:41:57.000-05:00
during execution
    - Resource management: Reducing memory pressure in
  CI that causes process termination
    - Signal handling: Properly cleaning up processes
  on unexpected termination
    - Timeout adjustments: Giving more time for
  operations in CI environments
  3. The Test Setup Validation Failure: This was
  happening because the before() hook was failing when
  it couldn't communicate with a dead Hugo process.
  Your health monitoring will catch this earlier and
  provide better error messages.
diff --git a/.github/actions/validate-links/action.yml b/.github/actions/validate-links/action.yml
@@ -37,30 +37,42 @@ runs:
           ${{ inputs.cache-key }}-
     
     - name: Run link validation
-      id: validate
+      shell: bash
       run: |
-        echo "Testing files: ${{ inputs.files }}"
-        if [[ -n "${{ inputs.product-name }}" ]]; then
-          echo "Product: ${{ inputs.product-name }}"
-        fi
+        # Set CI-specific environment variables
+        export CI=true
+        export GITHUB_ACTIONS=true
+        export NODE_OPTIONS="--max-old-space-size=4096"
         
-        if [[ "${{ inputs.cache-enabled }}" == "true" ]]; then
-          echo "📦 Cache enabled for this validation run"
-        fi
+        # Add timeout to prevent hanging
+        timeout 30m node cypress/support/run-e2e-specs.js ${{ inputs.files }} \
+          --spec cypress/e2e/content/article-links.cy.js || {
+          exit_code=$?
+          echo "::error::Link validation failed with exit code $exit_code"
+          
+          # Check for specific error patterns
+          if [ -f hugo.log ]; then
+            echo "::group::Hugo Server Logs"
+            cat hugo.log
+            echo "::endgroup::"
+          fi
+          
+          if [ -f /tmp/broken_links_report.json ]; then
+            echo "::group::Broken Links Report"
+            cat /tmp/broken_links_report.json
+            echo "::endgroup::"
+          fi
+          
+          exit $exit_code
+        }
         
-        # Run the validation
-        if node cypress/support/run-e2e-specs.js \
-            --spec "cypress/e2e/content/article-links.cy.js" \
-            ${{ inputs.files }}; then
-          echo "failed=false" >> $GITHUB_OUTPUT
-        else
-          echo "failed=true" >> $GITHUB_OUTPUT
-          exit 1
-        fi
-      shell: bash
-      env:
-        CI: true
-        CACHE_ENABLED: ${{ inputs.cache-enabled }}
+    - name: Upload logs on failure
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: validation-logs-${{ inputs.product-name }}
+        path: |
+          hugo.log
     
     - name: Upload broken links report
       if: failure()
diff --git a/cypress/support/run-e2e-specs.js b/cypress/support/run-e2e-specs.js
@@ -123,7 +123,13 @@ async function main() {
     console.log(`Performing cleanup before exit with code ${code}...`);
     if (hugoProc && hugoStarted) {
       try {
-        hugoProc.kill('SIGKILL'); // Use SIGKILL to ensure immediate termination
+        // Use SIGTERM first, then SIGKILL if needed
+        hugoProc.kill('SIGTERM');
+        setTimeout(() => {
+          if (!hugoProc.killed) {
+            hugoProc.kill('SIGKILL');
+          }
+        }, 1000);
       } catch (err) {
         console.error(`Error killing Hugo process: ${err.message}`);
       }
@@ -138,6 +144,10 @@ async function main() {
     console.error(`Uncaught exception: ${err.message}`);
     cleanupAndExit(1);
   });
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+    cleanupAndExit(1);
+  });
 
   const { fileArgs, specArgs } = parseArgs(process.argv);
   if (fileArgs.length === 0) {
@@ -354,13 +364,24 @@ async function main() {
     initializeReport();
 
     console.log(`Running Cypress tests for ${urlList.length} URLs...`);
+    
+    // Add CI-specific configuration
+    const isCI = process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true';
+    
     const cypressOptions = {
       reporter: 'junit',
       browser: 'chrome',
       config: {
         baseUrl: `http://localhost:${HUGO_PORT}`,
-        video: true,
-        trashAssetsBeforeRuns: false, // Prevent trash errors
+        video: !isCI, // Disable video in CI to reduce resource usage
+        trashAssetsBeforeRuns: false,
+        // Add CI-specific timeouts
+        defaultCommandTimeout: isCI ? 15000 : 10000,
+        pageLoadTimeout: isCI ? 45000 : 30000,
+        responseTimeout: isCI ? 45000 : 30000,
+        // Reduce memory usage in CI
+        experimentalMemoryManagement: true,
+        numTestsKeptInMemory: isCI ? 1 : 5,
       },
       env: {
         // Pass URLs as a comma-separated string for backward compatibility
@@ -377,8 +398,28 @@ async function main() {
       cypressOptions.spec = specArgs.join(',');
     }
 
+    // Add error handling for Hugo process monitoring during Cypress execution
+    let hugoHealthCheckInterval;
+    if (hugoProc && hugoStarted) {
+      hugoHealthCheckInterval = setInterval(() => {
+        if (hugoProc.killed || hugoProc.exitCode !== null) {
+          console.error('❌ Hugo server died during Cypress execution');
+          if (hugoHealthCheckInterval) {
+            clearInterval(hugoHealthCheckInterval);
+          }
+          cypressFailed = true;
+          // Don't exit immediately, let Cypress finish gracefully
+        }
+      }, 5000);
+    }
+
     const results = await cypress.run(cypressOptions);
 
+    // Clear health check interval
+    if (hugoHealthCheckInterval) {
+      clearInterval(hugoHealthCheckInterval);
+    }
+
     // Process broken links report
     const brokenLinksCount = displayBrokenLinksReport();
 
@@ -496,6 +537,22 @@ async function main() {
   } catch (err) {
     console.error(`❌ Cypress execution error: ${err.message}`);
 
+    // Handle EPIPE errors specifically
+    if (err.code === 'EPIPE' || err.message.includes('EPIPE')) {
+      console.error('🔧 EPIPE Error Detected:');
+      console.error('   • This usually indicates the Hugo server process was terminated unexpectedly');
+      console.error('   • Common causes in CI:');
+      console.error('     - Memory constraints causing process termination');
+      console.error('     - CI runner timeout or resource limits');
+      console.error('     - Hugo server crash due to build errors');
+      console.error(`   • Check Hugo logs: ${HUGO_LOG_FILE}`);
+      
+      // Try to provide more context about Hugo server state
+      if (hugoProc) {
+        console.error(`   • Hugo process state: killed=${hugoProc.killed}, exitCode=${hugoProc.exitCode}`);
+      }
+    }
+
     // Provide more detailed error information
     if (err.stack) {
       console.error('📋 Error Stack Trace:');
@@ -552,28 +609,44 @@ async function main() {
     if (hugoProc && hugoStarted && typeof hugoProc.kill === 'function') {
       console.log(`Stopping Hugo server (fast shutdown: ${cypressFailed})...`);
 
-      if (cypressFailed) {
-        hugoProc.kill('SIGKILL');
-        console.log('Hugo server forcibly terminated');
-      } else {
-        const shutdownTimeout = setTimeout(() => {
-          console.error(
-            'Hugo server did not shut down gracefully, forcing termination'
-          );
+      try {
+        if (cypressFailed) {
+          // Fast shutdown for failed tests
           hugoProc.kill('SIGKILL');
-          process.exit(exitCode);
-        }, 2000);
+          console.log('Hugo server forcibly terminated');
+        } else {
+          // Graceful shutdown for successful tests
+          const shutdownTimeout = setTimeout(() => {
+            console.error(
+              'Hugo server did not shut down gracefully, forcing termination'
+            );
+            try {
+              hugoProc.kill('SIGKILL');
+            } catch (killErr) {
+              console.error(`Error force-killing Hugo: ${killErr.message}`);
+            }
+            process.exit(exitCode);
+          }, 3000); // Increased timeout for CI
 
-        hugoProc.kill('SIGTERM');
+          hugoProc.kill('SIGTERM');
 
-        hugoProc.on('close', () => {
-          clearTimeout(shutdownTimeout);
-          console.log('Hugo server shut down successfully');
-          process.exit(exitCode);
-        });
+          hugoProc.on('close', () => {
+            clearTimeout(shutdownTimeout);
+            console.log('Hugo server shut down successfully');
+            process.exit(exitCode);
+          });
 
-        // Return to prevent immediate exit
-        return;
+          hugoProc.on('error', (err) => {
+            console.error(`Error during Hugo shutdown: ${err.message}`);
+            clearTimeout(shutdownTimeout);
+            process.exit(exitCode);
+          });
+
+          // Return to prevent immediate exit
+          return;
+        }
+      } catch (shutdownErr) {
+        console.error(`Error during Hugo server shutdown: ${shutdownErr.message}`);
       }
     } else if (hugoStarted) {
       console.log('Hugo process was started but is not available for cleanup');
