Fix Terraform documentation URLs and improve link checker robustness (#3988)

strickvl · web-flow · commit cb1fb7ee028d · 2025-09-23T12:47:48.000+02:00
Updates HashiCorp Terraform documentation URLs from legacy terraform.io domain
to the new developer.hashicorp.com location. HashiCorp has consolidated their
documentation under a unified developer portal.

Additionally enhances the link checker script to handle rate-limiting more
gracefully, particularly for HashiCorp domains which frequently return HTTP 429
responses during CI runs. The improvements include:

- Add soft-pass for HTTP 429 responses from configurable domains
- Implement domain-specific HEAD request skipping for sites that dislike HEAD
- Add custom User-Agent header support for better server compatibility
- Respect Retry-After headers and implement backoff strategies
- Add CLI flags for domain-specific policies (--ignore-429-domain, --no-head-domain)

These changes prevent CI flakiness from false-positive link failures while
maintaining proper validation for actual broken links.

Changes:
- Update all Terraform documentation URLs in markdown files
- Enhance link_checker.py with rate-limiting resilience
- Add domain-specific request handling policies
- Improve HTTP retry logic with proper backoff
diff --git a/docs/book/getting-started/hello-world.md b/docs/book/getting-started/hello-world.md
@@ -82,7 +82,7 @@ The fastest way to create a cloud stack is through the **Infrastructure-as-Code*
 
 You'll need:
 
-* [Terraform](https://www.terraform.io/downloads.html) version 1.9+ installed locally
+* [Terraform](https://developer.hashicorp.com/terraform/install) version 1.9+ installed locally
 * Authentication configured for your preferred cloud provider (AWS, GCP, or Azure)
 * Appropriate permissions to create resources in your cloud account
 
diff --git a/docs/book/how-to/infrastructure-deployment/infrastructure-as-code/README.md b/docs/book/how-to/infrastructure-deployment/infrastructure-as-code/README.md
@@ -10,7 +10,7 @@ the practice of managing and provisioning infrastructure through code\
 instead of through manual processes.
 
 In this section, we will show you how to integrate ZenML with popular\
-IaC tools such as [Terraform](https://www.terraform.io/).
+IaC tools such as [Terraform](https://developer.hashicorp.com/terraform).
 
 ![Screenshot of ZenML stack on Terraform Registry](../../../.gitbook/assets/terraform_providers_screenshot.png)
 
diff --git a/docs/book/how-to/infrastructure-deployment/stack-deployment/deploy-a-cloud-stack-with-terraform.md b/docs/book/how-to/infrastructure-deployment/stack-deployment/deploy-a-cloud-stack-with-terraform.md
@@ -47,7 +47,7 @@ ZENKEY_...
 Finally, you will need the following on the machine where you will be running\
 Terraform:
 
-* [Terraform](https://www.terraform.io/downloads.html) installed on your machine (version at least 1.9).
+* [Terraform](https://developer.hashicorp.com/terraform/install) installed on your machine (version at least 1.9).
 * the ZenML Terraform stack modules assume you are already locally authenticated with your cloud provider through the provider's CLI or SDK tool and have permissions to create the resources that the modules will provision. This is different depending on the cloud provider you are using and is covered in the following sections.
 
 ## How to use the Terraform stack deployment modules
diff --git a/docs/link_checker.py b/docs/link_checker.py
@@ -19,6 +19,7 @@
    - Validates links by making HTTP requests
    - Supports parallel validation for better performance
    - Provides detailed error reporting for broken links
+   - Soft-passes HTTP 429 responses for specific domains (e.g., HashiCorp docs) to avoid CI flakiness
 
 Usage Examples:
     # Find all links containing 'docs.zenml.io' in a directory
@@ -39,6 +40,9 @@
     # Use custom URL path mappings
     python link_checker.py --dir docs --replace-links --url-mapping user-guide=user-guides
 
+    # Soft-pass 429 for HashiCorp docs and skip HEAD for those domains (defaults)
+    python link_checker.py --dir docs --substring http --validate-links --ci-mode
+
 Arguments:
     --dir: Directory containing markdown files to scan
     --files: List of specific markdown files to scan
@@ -49,6 +53,9 @@
     --timeout: Timeout for HTTP requests in seconds (default: 10)
     --url-mapping: Path segment mappings in format old=new (can be used multiple times)
     --ci-mode: CI mode: only report broken links and exit with error code on failures
+    --ignore-429-domain: Domain for which HTTP 429 should be treated as a soft pass (can be used multiple times)
+    --no-head-domain: Domain for which to skip HEAD and use GET directly (can be used multiple times)
+    --user-agent: Custom User-Agent header to use for HTTP requests
 
 Note:
     The 'requests' package is required for link validation. Install it with:
@@ -61,6 +68,7 @@
 import sys
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Dict, List, Optional, Tuple
+from urllib.parse import urlparse
 
 try:
     import requests
@@ -71,6 +79,22 @@
 except ImportError:
     HAS_REQUESTS = False
 
+# Default policies for troublesome domains that frequently rate-limit automated traffic.
+# These defaults can be extended via CLI flags.
+DEFAULT_IGNORE_429_DOMAINS = {
+    "developer.hashicorp.com",
+    "terraform.io",
+    "www.terraform.io",
+}
+DEFAULT_NO_HEAD_DOMAINS = {
+    "developer.hashicorp.com",
+    "terraform.io",
+    "www.terraform.io",
+}
+DEFAULT_USER_AGENT = (
+    "ZenML-LinkChecker/1.0 (+https://github.com/zenml-io/zenml)"
+)
+
 
 def find_markdown_files(directory: str) -> List[str]:
     """Find all markdown files in the given directory and its subdirectories."""
@@ -227,14 +251,21 @@ def is_local_development_url(url: str) -> bool:
 
 
 def check_link_validity(
-    url: str, timeout: int = 10
+    url: str,
+    timeout: int = 10,
+    ignore_429_domains: Optional[set] = None,
+    no_head_domains: Optional[set] = None,
+    user_agent: Optional[str] = None,
 ) -> Tuple[str, bool, Optional[str], Optional[int]]:
     """
     Check if a URL is valid by making an HTTP request.
 
     Args:
         url: The URL to check
         timeout: Request timeout in seconds
+        ignore_429_domains: Domains for which HTTP 429 should be considered a soft pass
+        no_head_domains: Domains for which to skip HEAD and only use GET
+        user_agent: Custom User-Agent header
 
     Returns:
         Tuple of (url, is_valid, error_message, status_code)
@@ -258,36 +289,66 @@ def check_link_validity(
     if is_local_development_url(cleaned_url):
         return url, True, None, None
 
-    # Configure session with retries
+    parsed = urlparse(cleaned_url)
+    hostname = (parsed.hostname or "").lower()
+    ignore_429 = bool(ignore_429_domains) and hostname in ignore_429_domains
+    skip_head = bool(no_head_domains) and hostname in no_head_domains
+
+    # Configure session with retries. We respect Retry-After and avoid raising
+    # on final status, returning the last response instead. This allows us
+    # to interpret 429 as a soft-pass for specific domains.
     session = requests.Session()
     retries = Retry(
         total=3,
-        backoff_factor=0.5,
+        backoff_factor=1.0,
+        respect_retry_after_header=True,
         status_forcelist=[429, 500, 502, 503, 504],
         allowed_methods=["HEAD", "GET"],
+        raise_on_status=False,
     )
     session.mount("http://", HTTPAdapter(max_retries=retries))
     session.mount("https://", HTTPAdapter(max_retries=retries))
+    session.headers.update(
+        {
+            "User-Agent": user_agent or DEFAULT_USER_AGENT,
+            "Accept": "*/*",
+        }
+    )
 
     try:
-        # First try with HEAD request
-        response = session.head(
-            cleaned_url, timeout=timeout, allow_redirects=True
-        )
+        # Strategy: HEAD first unless the domain is known to dislike HEAD,
+        # then fallback to GET if needed. Some sites rate-limit or block HEAD.
+        response = None
+        if not skip_head:
+            response = session.head(
+                cleaned_url, timeout=timeout, allow_redirects=True
+            )
+        else:
+            response = session.get(
+                cleaned_url, timeout=timeout, allow_redirects=True
+            )
 
-        # If HEAD fails, try GET
+        # If HEAD fails (>=400), try GET
         if response.status_code >= 400:
             response = session.get(
                 cleaned_url, timeout=timeout, allow_redirects=True
             )
 
+        # Soft-pass 429 for configured domains
+        if response.status_code == 429 and ignore_429:
+            return (
+                url,
+                True,
+                "429 rate-limited (soft-pass for domain)",
+                429,
+            )
+
         is_valid = response.status_code < 400
 
         # Additional check for Gitbook URLs that return 200 for non-existent pages
         if is_valid and "docs.zenml.io" in cleaned_url:
             # We need to check for "noindex" meta tag which indicates a 404 page in Gitbook
             try:
-                # Use GET to fetch the page content
                 content_response = session.get(cleaned_url, timeout=timeout)
                 content = content_response.text.lower()
 
@@ -314,18 +375,35 @@ def check_link_validity(
         )
 
     except requests.RequestException as e:
+        # If we hit retry exhaustion with 429, soft-pass for configured domains
+        if "429" in str(e) and ignore_429:
+            return (
+                url,
+                True,
+                "429 rate-limited (soft-pass for domain)",
+                429,
+            )
         return url, False, str(e), None
 
 
 def validate_urls(
-    urls: List[str], max_workers: int = 10
+    urls: List[str],
+    max_workers: int = 10,
+    timeout: int = 10,
+    ignore_429_domains: Optional[set] = None,
+    no_head_domains: Optional[set] = None,
+    user_agent: Optional[str] = None,
 ) -> Dict[str, Tuple[bool, Optional[str], Optional[int]]]:
     """
     Validate multiple URLs in parallel.
 
     Args:
         urls: List of URLs to validate
         max_workers: Maximum number of parallel workers
+        timeout: Request timeout
+        ignore_429_domains: Domains for which HTTP 429 should be considered a soft pass
+        no_head_domains: Domains for which to skip HEAD and only use GET
+        user_agent: Custom User-Agent header
 
     Returns:
         Dictionary of {url: (is_valid, error_message, status_code)}
@@ -336,8 +414,6 @@ def validate_urls(
     results = {}
 
     # Count and report GitHub links that will be skipped in validation
-    from urllib.parse import urlparse
-
     github_urls = [
         url
         for url in urls
@@ -369,7 +445,14 @@ def validate_urls(
         # Submit all URLs (GitHub links will be auto-skipped in check_link_validity)
         for url in urls:
             future_to_url[
-                executor.submit(check_link_validity, url, timeout=15)
+                executor.submit(
+                    check_link_validity,
+                    url,
+                    timeout=timeout,
+                    ignore_429_domains=ignore_429_domains,
+                    no_head_domains=no_head_domains,
+                    user_agent=user_agent,
+                )
             ] = url
 
         # Process results
@@ -384,7 +467,12 @@ def validate_urls(
                         f"  Checked URL {i}/{len(urls)} [github.com]: ✓ Skipped (automatically marked valid)"
                     )
                 else:
-                    status = "✅ Valid" if is_valid else f"❌ {error_message}"
+                    if is_valid and status_code == 429:
+                        status = "✅ Valid (429 soft-pass)"
+                    else:
+                        status = (
+                            "✅ Valid" if is_valid else f"❌ {error_message}"
+                        )
                     domain = (
                         url.split("/")[2]
                         if "://" in url and "/" in url.split("://", 1)[1]
@@ -482,6 +570,11 @@ def replace_links_in_file(
     dry_run: bool = False,
     validate_links: bool = False,
     url_mappings: Dict[str, str] = None,
+    *,
+    timeout: int = 10,
+    ignore_429_domains: Optional[set] = None,
+    no_head_domains: Optional[set] = None,
+    user_agent: Optional[str] = None,
 ) -> Dict[str, Tuple[str, bool, Optional[str]]]:
     """
     Replace relative links in the file with absolute URLs.
@@ -492,6 +585,10 @@ def replace_links_in_file(
         dry_run: If True, don't actually modify the file
         validate_links: If True, validate the generated links
         url_mappings: Dictionary of path segment mappings {old: new}
+        timeout: HTTP timeout for validation requests
+        ignore_429_domains: Domains for which HTTP 429 should be considered a soft pass
+        no_head_domains: Domains for which to skip HEAD and only use GET
+        user_agent: Custom User-Agent header
 
     Returns:
         Dictionary of {original_link: (new_link, is_valid, error_message)}
@@ -596,7 +693,13 @@ def should_replace_link(link: str) -> bool:
     # Validate links if requested
     validation_results = {}
     if validate_links and transformed_urls:
-        validation_results = validate_urls(transformed_urls)
+        validation_results = validate_urls(
+            transformed_urls,
+            timeout=timeout,
+            ignore_429_domains=ignore_429_domains,
+            no_head_domains=no_head_domains,
+            user_agent=user_agent,
+        )
 
         # Update the replacements dictionary with validation results
         for rel_link, (trans_link, _, _) in replacements.items():
@@ -731,6 +834,23 @@ def main():
         action="store_true",
         help="CI mode: only report broken links and exit with error code on failures",
     )
+    parser.add_argument(
+        "--ignore-429-domain",
+        action="append",
+        default=[],
+        help="Domain for which to consider HTTP 429 as a soft pass (can be used multiple times). Defaults include developer.hashicorp.com and terraform.io.",
+    )
+    parser.add_argument(
+        "--no-head-domain",
+        action="append",
+        default=[],
+        help="Domain for which to skip HEAD and only use GET (can be used multiple times). Defaults include developer.hashicorp.com and terraform.io.",
+    )
+    parser.add_argument(
+        "--user-agent",
+        default=DEFAULT_USER_AGENT,
+        help="User-Agent to use for HTTP requests.",
+    )
     args = parser.parse_args()
 
     # Check for requests module if validation is enabled
@@ -766,6 +886,14 @@ def main():
         if not args.ci_mode:
             print(f"Scanning {len(files_to_scan)} specified markdown files")
 
+    # Merge defaults with CLI-provided domain policies
+    ignore_429_domains = DEFAULT_IGNORE_429_DOMAINS.union(
+        set(args.ignore_429_domain or [])
+    )
+    no_head_domains = DEFAULT_NO_HEAD_DOMAINS.union(
+        set(args.no_head_domain or [])
+    )
+
     if args.replace_links:
         # Replace links mode
         total_replacements = 0
@@ -780,6 +908,10 @@ def main():
                     args.dry_run,
                     args.validate_links,
                     url_mappings,
+                    timeout=args.timeout,
+                    ignore_429_domains=ignore_429_domains,
+                    no_head_domains=no_head_domains,
+                    user_agent=args.user_agent,
                 )
                 if replacements:
                     if not args.ci_mode:
@@ -887,7 +1019,13 @@ def main():
         if args.validate_links and links_to_validate:
             if not args.ci_mode:
                 print(f"\nValidating {len(links_to_validate)} links...")
-            validation_results = validate_urls(list(set(links_to_validate)))
+            validation_results = validate_urls(
+                list(set(links_to_validate)),
+                timeout=args.timeout,
+                ignore_429_domains=ignore_429_domains,
+                no_head_domains=no_head_domains,
+                user_agent=args.user_agent,
+            )
 
             valid_count = sum(
                 1 for result in validation_results.values() if result[0]
