Fix daily link checker (github#28346)

* reduce linkinator concurrency from 300 to 100

* reduce it and use self-hosted

* try again

* actually do concurrency of 10

* move the override to the Action workflow

* Trying `DISABLE_RENDERING_CACHE`

* back to 100?

* super low concurrency

* 3??

* 1??

* try self-hosted + 100 concurrency

Co-authored-by: Peter Bengtsson <[email protected]>

Author: rsese

.github/workflows/check-all-english-links.yml

@@ -17,7 +17,7 @@ jobs:
   check_all_english_links:
     name: Check all links
     if: github.repository == 'github/docs-internal'
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
     env:
       GITHUB_TOKEN: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }}
       FIRST_RESPONDER_PROJECT: Docs content first responder
@@ -56,10 +56,21 @@ jobs:
           # The default is 10s. But because this runs overnight, we can
           # be a lot more patient.
           REQUEST_TIMEOUT: 20000
+          # The default is 300 which works OK on a fast macbook pro
+          # but so well in Actions.
+          LINKINATOR_CONCURRENCY: 100
         run: |
-          node server.mjs &
+          node server.mjs > /tmp/stdout.log 2> /tmp/stderr.log &
           sleep 5
-          curl --retry-connrefused --retry 3 -I http://localhost:4000/
+          curl --retry-connrefused --retry 4 -I http://localhost:4000/
+
+      - if: ${{ failure() }}
+        name: Debug server outputs on errors
+        run: |
+          echo "____STDOUT____"
+          cat /tmp/stdout.log
+          echo "____STDERR____"
+          cat /tmp/stderr.log
 
       - name: Run script
         run: |

script/check-english-links.js

@@ -43,6 +43,8 @@ const DISPLAY_MAX_LENGTH = parseInt(process.env.DISPLAY_MAX_LENGTH || '30000', 1
 // Links with these codes may or may not really be broken.
 const retryStatusCodes = [429, 503, 'Invalid']
 
+const LINKINATOR_CONCURRENCY = parseInt(process.env.LINKINATOR_CONCURRENCY || '300')
+
 program
   .description('Check all links in the English docs.')
   .option(
@@ -73,7 +75,7 @@ const enterpriseReleasesToSkip = new RegExp(`${root}.+?[/@](${deprecated.join('|
 
 const config = {
   path: program.opts().path || englishRoot,
-  concurrency: 300,
+  concurrency: LINKINATOR_CONCURRENCY,
   // If this is a dry run, turn off recursion.
   recurse: !program.opts().dryRun,
   silent: true,