diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index cae4168dbc7..8e5519d33ef 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -14,20 +14,6 @@ on: - cron: '0 0 * * *' jobs: - check-urls: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Check URLs - run: bash ./scripts/check_urls.sh - - check-xrefs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Check Links - run: bash ./scripts/check_xrefs.sh - build: uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main permissions: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f174ddb6dc7..63c4fd1ea66 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -64,6 +64,30 @@ jobs: exit $RC + lint-urls: + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + with: + runner: linux.2xlarge + docker-image: executorch-ubuntu-22.04-linter + submodules: 'none' + fetch-depth: 0 + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + ./scripts/lint_urls.sh + + lint-xrefs: + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + with: + runner: linux.2xlarge + docker-image: executorch-ubuntu-22.04-linter + submodules: 'none' + fetch-depth: 0 + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + ./scripts/lint_xrefs.sh + android-java-format: uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main permissions: diff --git a/scripts/check_urls.sh b/scripts/check_urls.sh deleted file mode 100755 index ad6c1440ebe..00000000000 --- a/scripts/check_urls.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -set -euo pipefail - -status=0 -green='\e[1;32m'; red='\e[1;31m'; cyan='\e[1;36m'; yellow='\e[1;33m'; reset='\e[0m' -user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" -last_filepath= - -while IFS=: read -r filepath url; do - if [ "$filepath" != "$last_filepath" ]; then - printf '\n%s:\n' "$filepath" - last_filepath=$filepath - fi - code=$(curl -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -I "$url") || code=000 - if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then - code=$(curl -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -r 0-0 -A "$user_agent" "$url") || code=000 - fi - if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then - request_id=$(curl -sS -H 'Accept: application/json' "https://check-host.net/check-http?host=$url&max_nodes=1&node=us3.node.check-host.net" \ - | jq -r .request_id) - for _ in {1..3}; do - code=$(curl -sS -H 'Accept: application/json' "https://check-host.net/check-result/$request_id" \ - | jq -r -e '.[][0][3]') || code=000 - [[ "$code" =~ ^[0-9]+$ ]] || code=000 - sleep 3 - done - fi - if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then - printf "${red}%s${reset} ${yellow}%s${reset}\n" "$code" "$url" >&2 - status=1 - else - printf "${green}%s${reset} ${cyan}%s${reset}\n" "$code" "$url" - fi -done < <( - git --no-pager grep --no-color -I -P -o \ - '(?\")]*[\{\}\$])[^[:space:]<>\")\[\]\(]+' \ - -- '*' \ - ':(exclude).*' \ - ':(exclude)**/.*' \ - ':(exclude)**/*.lock' \ - ':(exclude)**/*.svg' \ - ':(exclude)**/*.xml' \ - ':(exclude)**/*.gradle*' \ - ':(exclude)**/*gradle*' \ - ':(exclude)**/third-party/**' \ - | sed -E 's/[^/[:alnum:]]+$//' \ - | grep -Ev '://(0\.0\.0\.0|127\.0\.0\.1|localhost)([:/])' \ - | grep -Ev 'fwdproxy:8080' \ - || true -) - -exit $status diff --git a/scripts/lint_urls.sh b/scripts/lint_urls.sh new file mode 100755 index 00000000000..761e96ea7cd --- /dev/null +++ b/scripts/lint_urls.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set -euo pipefail + +status=0 +green='\e[1;32m'; red='\e[1;31m'; cyan='\e[1;36m'; yellow='\e[1;33m'; reset='\e[0m' +user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" +max_jobs=10 +pids=() + +running_jobs() { + jobs -rp | wc -l +} + +while IFS=: read -r filepath url; do + fpath="$filepath" + ( + code=$(curl -k -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -I "$url") || code=000 + if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then + code=$(curl -k -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -r 0-0 -A "$user_agent" "$url") || code=000 + fi + if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then + request_id=$(curl -sS -G -H 'Accept: application/json' \ + --data-urlencode "host=$url" \ + --data-urlencode "max_nodes=1" \ + --data-urlencode "node=us3.node.check-host.net" \ + https://check-host.net/check-http \ + | jq -r .request_id) || request_id="" + if [ -n "$request_id" ]; then + sleep 5 + for _ in {1..5}; do + new_code=$(curl -sS -H 'Accept: application/json' \ + "https://check-host.net/check-result/$request_id" \ + | jq -r -e '.[][0][3]') || new_code=000 + [[ "$new_code" =~ ^[0-9]+$ ]] || new_code=000 + if [ "$new_code" -ge 200 ] && [ "$new_code" -lt 400 ]; then + code=$new_code + break + fi + sleep 5 + done + fi + fi + if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then + printf "${red}%s${reset} ${yellow}%s${reset} %s\n" "$code" "$url" "$fpath" >&2 + exit 1 + else + printf "${green}%s${reset} ${cyan}%s${reset} %s\n" "$code" "$url" "$fpath" + exit 0 + fi + ) & + pids+=($!) + while [ "$(running_jobs)" -ge "$max_jobs" ]; do + sleep 1 + done +done < <( + git --no-pager grep --no-color -I -P -o \ + '(?!.*@lint-ignore)(?\")]*[<>\{\}\$])[^[:space:]<>\")\[\]\(\\]+' \ + -- '*' \ + ':(exclude).*' \ + ':(exclude,glob)**/.*' \ + ':(exclude,glob)**/*.lock' \ + ':(exclude,glob)**/*.svg' \ + ':(exclude,glob)**/*.xml' \ + ':(exclude,glob)**/*.gradle*' \ + ':(exclude,glob)**/*gradle*' \ + ':(exclude,glob)**/third-party/**' \ + ':(exclude,glob)**/third_party/**' \ + | sed -E 's/[^/[:alnum:]]+$//' \ + | grep -Ev '://(0\.0\.0\.0|127\.0\.0\.1|localhost)([:/])' \ + | grep -Ev 'fwdproxy:8080' \ + || true +) + +for pid in "${pids[@]}"; do + wait "$pid" 2>/dev/null || { + case $? in + 1) status=1 ;; + 127) ;; # ignore "not a child" noise + *) exit $? ;; + esac + } +done + +exit $status diff --git a/scripts/check_xrefs.sh b/scripts/lint_xrefs.sh similarity index 82% rename from scripts/check_xrefs.sh rename to scripts/lint_xrefs.sh index 69e083a8a67..047be392c06 100755 --- a/scripts/check_xrefs.sh +++ b/scripts/lint_xrefs.sh @@ -23,15 +23,16 @@ while IFS=: read -r filepath link; do status=1 fi done < <( - git --no-pager grep --no-color -I -o -E \ - '\[[^]]+\]\([^[:space:])]*/[^[:space:])]*\)|href="[^"]*/[^"]*"|src="[^"]*/[^"]*"' \ + git --no-pager grep --no-color -I -P -o \ + '(?!.*@lint-ignore)(?:\[[^]]+\]\([^[:space:])]*/[^[:space:])]*\)|href="[^"]*/[^"]*"|src="[^"]*/[^"]*")' \ -- '*' \ ':(exclude).*' \ ':(exclude)**/.*' \ ':(exclude)**/*.lock' \ ':(exclude)**/*.svg' \ ':(exclude)**/*.xml' \ - ':(exclude)**/third-party/**' \ + ':(exclude,glob)**/third-party/**' \ + ':(exclude,glob)**/third_party/**' \ | grep -Ev 'https?://' \ | sed -E \ -e 's#([^:]+):\[[^]]+\]\(([^)]+)\)#\1:\2#' \