Lint xrefs and urls

.github/workflows/doc-build.yml

@@ -14,20 +14,6 @@ on:
     - cron: '0 0 * * *'
 
 jobs:
-  check-urls:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check URLs
-        run: bash ./scripts/check_urls.sh
-
-  check-xrefs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check Links
-        run: bash ./scripts/check_xrefs.sh
-
   build:
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     permissions:

.github/workflows/lint.yml

@@ -64,6 +64,30 @@ jobs:
 
         exit $RC
 
+  lint-urls:
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-linter
+      submodules: 'none'
+      fetch-depth: 0
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        ./scripts/lint_urls.sh
+
+  lint-xrefs:
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-linter
+      submodules: 'none'
+      fetch-depth: 0
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        ./scripts/lint_xrefs.sh
+
   android-java-format:
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     permissions:

scripts/lint_urls.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -euo pipefail
+
+status=0
+green='\e[1;32m'; red='\e[1;31m'; cyan='\e[1;36m'; yellow='\e[1;33m'; reset='\e[0m'
+user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
+max_jobs=10
+pids=()
+
+running_jobs() {
+  jobs -rp | wc -l
+}
+
+while IFS=: read -r filepath url; do
+  fpath="$filepath"
+  (
+    code=$(curl -k -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -I "$url") || code=000
+    if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then
+      code=$(curl -k -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -r 0-0 -A "$user_agent" "$url") || code=000
+    fi
+    if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then
+      request_id=$(curl -sS -G -H 'Accept: application/json' \
+        --data-urlencode "host=$url" \
+        --data-urlencode "max_nodes=1" \
+        --data-urlencode "node=us3.node.check-host.net" \
+        https://check-host.net/check-http \
+        | jq -r .request_id) || request_id=""
+      if [ -n "$request_id" ]; then
+        sleep 5
+        for _ in {1..5}; do
+          new_code=$(curl -sS -H 'Accept: application/json' \
+            "https://check-host.net/check-result/$request_id" \
+            | jq -r -e '.[][0][3]') || new_code=000
+          [[ "$new_code" =~ ^[0-9]+$ ]] || new_code=000
+          if [ "$new_code" -ge 200 ] && [ "$new_code" -lt 400 ]; then
+            code=$new_code
+            break
+          fi
+          sleep 5
+        done
+      fi
+    fi
+    if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then
+      printf "${red}%s${reset} ${yellow}%s${reset} %s\n" "$code" "$url" "$fpath" >&2
+      exit 1
+    else
+      printf "${green}%s${reset} ${cyan}%s${reset} %s\n" "$code" "$url" "$fpath"
+      exit 0
+    fi
+  ) &
+  pids+=($!)
+  while [ "$(running_jobs)" -ge "$max_jobs" ]; do
+    sleep 1
+  done
+done < <(
+  git --no-pager grep --no-color -I -P -o \
+    '(?!.*@lint-ignore)(?<!git\+)(?<!\$\{)https?://(?![^\s<>\")]*[<>\{\}\$])[^[:space:]<>\")\[\]\(\\]+' \
+    -- '*' \
+    ':(exclude).*' \
+    ':(exclude,glob)**/.*' \
+    ':(exclude,glob)**/*.lock' \
+    ':(exclude,glob)**/*.svg' \
+    ':(exclude,glob)**/*.xml' \
+    ':(exclude,glob)**/*.gradle*' \
+    ':(exclude,glob)**/*gradle*' \
+    ':(exclude,glob)**/third-party/**' \
+    ':(exclude,glob)**/third_party/**' \
+  | sed -E 's/[^/[:alnum:]]+$//' \
+  | grep -Ev '://(0\.0\.0\.0|127\.0\.0\.1|localhost)([:/])' \
+  | grep -Ev 'fwdproxy:8080' \
+  || true
+)
+
+for pid in "${pids[@]}"; do
+  wait "$pid" 2>/dev/null || {
+    case $? in
+      1) status=1 ;;
+      127) ;;  # ignore "not a child" noise
+      *) exit $? ;;
+    esac
+  }
+done
+
+exit $status

scripts/check_xrefs.sh → scripts/lint_xrefs.sh

@@ -23,15 +23,16 @@ while IFS=: read -r filepath link; do
     status=1
   fi
 done < <(
-  git --no-pager grep --no-color -I -o -E \
-    '\[[^]]+\]\([^[:space:])]*/[^[:space:])]*\)|href="[^"]*/[^"]*"|src="[^"]*/[^"]*"' \
+  git --no-pager grep --no-color -I -P -o \
+    '(?!.*@lint-ignore)(?:\[[^]]+\]\([^[:space:])]*/[^[:space:])]*\)|href="[^"]*/[^"]*"|src="[^"]*/[^"]*")' \
     -- '*' \
     ':(exclude).*' \
     ':(exclude)**/.*' \
     ':(exclude)**/*.lock' \
     ':(exclude)**/*.svg' \
     ':(exclude)**/*.xml' \
-    ':(exclude)**/third-party/**' \
+    ':(exclude,glob)**/third-party/**' \
+    ':(exclude,glob)**/third_party/**' \
   | grep -Ev 'https?://' \
   | sed -E \
       -e 's#([^:]+):\[[^]]+\]\(([^)]+)\)#\1:\2#' \