Skip to content

Automatically check for broken links #1048

New issue
New issue
Open
Task
Open
Automatically check for broken links#1048
Task
Labels
enhancementNew feature or request
@SimiHunjan

Description

@SimiHunjan
SimiHunjan
opened on Jul 30, 2025

Problem

Checking for broken links is not automated and reported by users manually navigating the pages and clicking on links.

Create a script that scans all Markdown (.md) files in the repository and checks for broken links. This will help find broken links in the repo.

This script can later be added to CI to fail PRs with broken links in docs.

Solution

Example:

require("dotenv").config();
const axios = require("axios");
const axiosRetry = require("axios-retry").default;
const { Octokit } = require("@octokit/rest");
const path = require("path");

const REPO_OWNER = "hiero-ledger";
const REPO_NAME = "hiero-sdk-java";
const BRANCH = "main";

const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
axiosRetry(axios, { retries: 3, retryDelay: axiosRetry.exponentialDelay });

const brokenLinks = [];

// Status code → description map
const STATUS_DESCRIPTIONS = {
  400: "Bad Request",
  401: "Unauthorized",
  403: "Forbidden",
  404: "Not Found",
  405: "Method Not Allowed",
  408: "Request Timeout",
  429: "Too Many Requests",
  500: "Internal Server Error",
  502: "Bad Gateway",
  503: "Service Unavailable",
  504: "Gateway Timeout",
};

// Get all .md files in the repo recursively
async function getMarkdownFiles(path = "") {
  const response = await octokit.repos.getContent({
    owner: REPO_OWNER,
    repo: REPO_NAME,
    path,
  });

  const files = [];

  for (const item of response.data) {
    if (item.type === "file" && item.name.endsWith(".md")) {
      files.push({ download_url: item.download_url, repo_path: item.path });
    } else if (item.type === "dir") {
      const nestedFiles = await getMarkdownFiles(item.path);
      files.push(...nestedFiles);
    }
  }

  return files;
}

// Extract markdown links
function extractLinks(markdown) {
  const regex = /\[.*?\]\((.*?)\)/g;
  const links = new Set();
  let match;
  while ((match = regex.exec(markdown)) !== null) {
    const link = match[1];
    if (!link.startsWith("mailto:")) {
      links.add(link);
    }
  }
  return [...links];
}

// Resolve relative links to GitHub blob URL
function resolveRelativeLink(repoPath, relLink) {
  const baseDir = path.posix.dirname(repoPath);
  const normalizedPath = path.posix.normalize(
    path.posix.join(baseDir, relLink)
  );
  return `https://github.com/${REPO_OWNER}/${REPO_NAME}/blob/${BRANCH}/${normalizedPath}`;
}

// Check if a link works
async function checkLink(url) {
  try {
    const res = await axios.head(url, {
      timeout: 20000,
      validateStatus: () => true,
    });
    if (res.status >= 400) {
      const reason = STATUS_DESCRIPTIONS[res.status] || "Unknown Error";
      console.log(`[BROKEN] ${url} - ${res.status} ${reason}`);
      brokenLinks.push({ url, status: res.status, reason });
    }
  } catch (err) {
    console.log(`[ERROR] ${url} - Request failed: ${err.message}`);
    brokenLinks.push({ url, status: "Request failed", reason: err.message });
  }
}

// Main
(async () => {
  console.log(`🔍 Crawling markdown files in ${REPO_OWNER}/${REPO_NAME}...\n`);

  let mdFiles = [];
  try {
    mdFiles = await getMarkdownFiles();
  } catch (err) {
    console.error("❌ Failed to fetch markdown files:", err.message);
    process.exit(1);
  }

  const allLinks = new Set();

  for (const { download_url, repo_path } of mdFiles) {
    try {
      const res = await axios.get(download_url, { timeout: 15000 });
      const links = extractLinks(res.data);
      for (const link of links) {
        if (link.startsWith("http")) {
          allLinks.add(link);
        } else if (!link.startsWith("#")) {
          allLinks.add(resolveRelativeLink(repo_path, link));
        }
      }
    } catch (err) {
      console.error(`[FAILED TO LOAD MD] ${download_url}`);
    }
  }

  console.log(`\n🔗 Found ${allLinks.size} unique links. Checking...\n`);

  for (const link of allLinks) {
    await checkLink(link);
  }

  console.log(`\n✅ Done. ${brokenLinks.length} broken links found.`);
})();

Output:

🔍 Crawling markdown files in hiero-ledger/hiero-sdk-java...


🔗 Found 93 unique links. Checking...

[BROKEN] https://api.scorecard.dev/projects/github.com/hiero-ledger/hiero-sdk-java/badge - 405 Method Not Allowed
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/docs/sdk/discord - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ConstructClientExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/GenerateKeyExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/GenerateKeyWithMnemonicExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/GetAddressBookExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/GetExchangeRatesExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/LoggerFunctionalitiesExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/SignTransactionExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/TransactionSerializationExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ScheduleExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ScheduledTransferExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ScheduleIdenticalTransactionExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ScheduleMultiSigTransactionExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ScheduledTransactionMultiSigThresholdExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateAccountExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateAccountThresholdKeyExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateAccountWithAliasExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateAccountWithAliasAndReceiverSignatureRequiredExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/AccountCreationWaysExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/AccountCreateWithHtsExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/AutoCreateAccountTransferTransactionExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/AccountAliasExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/AccountAllowanceExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/GetAccountInfoExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/GetAccountBalanceExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/UpdateAccountPublicKeyExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/DeleteAccountExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/StakingExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/StakingWithUpdateExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/MultiSigOfflineExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateTopicExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/TopicWithAdminKeyExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ConsensusPubSubExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ConsensusPubSubChunkedExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ConsensusPubSubWithSubmitKeyExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/TransferCryptoExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/MultiAppTransferExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/TransferTokensExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/TransferUsingEvmAddressExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CustomFeesExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ExemptCustomFeesExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/NftAddRemoveAllowancesExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ZeroTokenOperationsExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ChangeRemoveTokenKeys.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/TokenRejectExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateFileExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/FileAppendChunkedExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/GetFileContentsExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/DeleteFileExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateSimpleContractExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/CreateStatefulContractExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ContractNoncesExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/SolidityPrecompileExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/ValidateChecksumExample.java - 404 Not Found
[BROKEN] https://github.com/hiero-ledger/hiero-sdk-java/blob/main/examples/src/main/java/org/hiero/sdk/java/examples/PrngExample.java - 404 Not Found

✅ Done. 56 broken links found.

Alternatives

No response

Metadata

Metadata

Assignees

No one assigned

    Labels

    enhancementNew feature or request

    Type

    Task

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions