feat(modules): implement skipped modules handling and validation

KristjanESPERANTO · KristjanESPERANTO · commit 4ddbc041eafd · 2025-09-28T14:32:44.000+02:00
diff --git a/container/Dockerfile b/container/Dockerfile
@@ -10,6 +10,7 @@ set -e
 git config --global --add safe.directory /workspace
 git log -1
 export GITHUB_TOKEN="${GITHUB_TOKEN}"
+pip install requests
 npm clean-install
 node --run all
 EOF
diff --git a/docs/index.css b/docs/index.css
@@ -605,3 +605,51 @@ input:checked + .slider::before {
   padding: 0.5rem;
   cursor: pointer;
 }
+
+/* Styling for skipped/failed modules */
+.card.skipped {
+  border-left: 4px solid #dc3545;
+  background-image: repeating-linear-gradient(140deg, #0000, #f002 2%);
+  opacity: 0.85;
+}
+
+.card.skipped .header {
+  background-color: #fdeaea;
+  border-bottom: 1px solid #f5c6cb;
+}
+
+.card.skipped .header .title {
+  color: #721c24;
+}
+
+.card.skipped .error-message {
+  color: #dc3545;
+  font-weight: 500;
+  padding: 0.5rem;
+  background-color: #f8d7da;
+  border: 1px solid #f5c6cb;
+  border-radius: 4px;
+  margin: 0.5rem;
+  font-size: 0.9rem;
+}
+
+/* Dark mode styles for skipped modules */
+body.dark .card.skipped {
+  background-color: #2d1b1b;
+  border-left-color: #dc3545;
+}
+
+body.dark .card.skipped .header {
+  background-color: #3d2222;
+  border-bottom-color: #5a2828;
+}
+
+body.dark .card.skipped .header .title {
+  color: #ff6b6b;
+}
+
+body.dark .card.skipped .error-message {
+  background-color: #4a2626;
+  border-color: #5a2828;
+  color: #ff8a80;
+}
diff --git a/docs/script.js b/docs/script.js
@@ -1,4 +1,5 @@
 let allModules = [];
+let skippedModules = [];
 let filteredModuleList = [];
 const cardTemplate = document.getElementById("card-template");
 const resetButton = document.getElementById("reset-button");
@@ -33,6 +34,23 @@ function toggleMenu () {
 function createCard (moduleData) {
   const card = document.importNode(cardTemplate.content, true);
 
+  // Skipped module special handling
+  if (moduleData.skipped) {
+    card.querySelector(".card").classList.add("skipped");
+    card.querySelector(".name").textContent = moduleData.name || "Unknown Module";
+    card.querySelector(".name").href = moduleData.url || "#";
+    card.querySelector(".description").innerHTML = `<span style='color:red;font-weight:bold'>Error: Module could not be loaded.</span><br>${moduleData.error || "Unknown Error"}`;
+    // Remove other info sections
+    card.querySelector(".maintainer").textContent = moduleData.maintainer || "?";
+    [".stars", ".tags", ".img-container", ".info", ".outdated-note"].forEach((selector) => {
+      const element = card.querySelector(selector);
+      if (element) {
+        element.remove();
+      }
+    });
+    return card;
+  }
+
   /* Set the header data */
   card.querySelector(".name").href = moduleData.url;
   card.querySelector(".name").textContent = moduleData.name;
@@ -132,7 +150,7 @@ function updateModuleCardContainer () {
   let moduleCounter = filteredModuleList.length;
 
   filteredModuleList.forEach((moduleData) => {
-    if (!moduleData.outdated || showOutdated.checked) {
+    if ((!moduleData.outdated || showOutdated.checked) && (!moduleData.skipped || showOutdated.checked)) {
       try {
         const cardNode = createCard(moduleData);
         if (cardNode) {
@@ -229,7 +247,8 @@ function displayStatistics (data) {
 
 function filterBySearchText (searchText) {
   const searchLower = searchText.toLowerCase();
-  filteredModuleList = allModules.filter((card) => {
+  const allModulesList = allModules.concat(skippedModules);
+  filteredModuleList = allModulesList.filter((card) => {
     const cardText = card.text
       ? card.text.toLowerCase()
       : "";
@@ -295,8 +314,8 @@ function filterByTag (tag) {
 function addCategoryFilter () {
   const categoryFilter = document.getElementById("category-filter");
   const categories = [...new Set(allModules.map((module) => module.category))];
-
   categories.sort();
+  categories.push("Problematic Modules");
 
   categories.forEach((category) => {
     const option = document.createElement("option");
@@ -308,15 +327,15 @@ function addCategoryFilter () {
   categoryFilter.addEventListener("change", () => {
     const selectedCategory = categoryFilter.value;
     if (selectedCategory === "all") {
-      filteredModuleList = allModules;
+      filteredModuleList = allModules.concat(skippedModules);
+    } else if (selectedCategory === "Problematic Modules") {
+      filteredModuleList = skippedModules;
     } else {
       filteredModuleList = allModules.filter((module) => module.category === selectedCategory);
     }
 
     searchInput.value = "";
-
     removeSelectedMarkingFromTagsAndCards();
-
     updateModuleCardContainer();
   });
 }
@@ -338,7 +357,7 @@ moduleCardContainer.addEventListener("click", (event) => {
 resetButton.addEventListener("click", () => {
   resetCategoryFilter();
   const root = document.querySelector(":root");
-  filteredModuleList = allModules;
+  filteredModuleList = allModules.concat(skippedModules);
   searchInput.value = "";
   showOutdated.checked = true;
   removeSelectedMarkingFromTagsAndCards();
@@ -360,7 +379,7 @@ searchInput.addEventListener("input", () => {
     if (searchInput.value) {
       filterBySearchText(searchInput.value);
     } else {
-      filteredModuleList = allModules;
+      filteredModuleList = allModules.concat(skippedModules);
       updateModuleCardContainer();
     }
   }, 180);
@@ -413,15 +432,27 @@ async function initiate () {
     const response = await fetch(modulesFile);
     const data = await response.json();
     allModules = data;
-    filteredModuleList = data;
-    sortData(sortDropdown.value);
-    updateModuleCardContainer();
-    displayTagButtonContainer();
-    addCategoryFilter();
   } catch (error) {
-    console.error("Error fetching data:", error);
+    allModules = [];
+    console.error("Error fetching modules:", error);
+  }
+
+  // Load skipped modules
+  try {
+    const skippedRes = await fetch("data/skipped_modules.json");
+    const skippedRaw = await skippedRes.json();
+    skippedModules = skippedRaw.map((moduleObj) => ({...moduleObj, skipped: true, defaultSortWeight: 1000, lastCommit: ""}));
+  } catch {
+    skippedModules = [];
   }
 
+  filteredModuleList = allModules.concat(skippedModules);
+  sortData(sortDropdown.value);
+  updateModuleCardContainer();
+  displayTagButtonContainer();
+  addCategoryFilter();
+
+  // Load statistics
   const statisticsFile = "data/stats.json";
   try {
     const response = await fetch(statisticsFile);
diff --git a/scripts/check_modules.py b/scripts/check_modules.py
@@ -273,7 +273,7 @@ def check_modules():
     }
 
     modules_json_file = open(
-        "./docs/data/modules.stage.4.json", encoding="utf-8")
+        "./docs/data/modules.stage.5.json", encoding="utf-8")
     modules = json.load(modules_json_file)
     stats = {
         "moduleCounter": 0,
@@ -287,7 +287,7 @@ def check_modules():
 
     markdown_output_modules = ""
 
-    for module in modules:
+    for module in modules["modules"]:
         module["defaultSortWeight"] = 0
         stats["moduleCounter"] += 1
 
@@ -528,7 +528,7 @@ def check_modules():
                 module["issues"] = False
 
             # Lift modules with many stars in the default sort order.
-            module["defaultSortWeight"] = module["defaultSortWeight"] - (module['stars'] // 20)
+            module["defaultSortWeight"] = module["defaultSortWeight"] - (module.get('stars', 0) // 20)
 
             # Modules with few stars shouldn't be too far up in the default sort order. So we give them a minimum value of one.
             if module.get('stars', 0) < 3:
diff --git a/scripts/check_modules_js.js b/scripts/check_modules_js.js
@@ -1,15 +1,15 @@
 import fs from "node:fs";
 // Disabled: import {isMinified} from "./utils.js";
 
-fs.readFile("./docs/data/modules.stage.3.json", "utf8", (err, data) => {
+fs.readFile("./docs/data/modules.stage.4.json", "utf8", (err, data) => {
   if (err) {
     console.error(err);
     return;
   }
 
   const modules = JSON.parse(data);
 
-  modules.forEach((module) => {
+  modules.modules.forEach((module) => {
     const filePath = `./modules/${module.name}-----${module.maintainer}/${module.name}.js`;
 
     if (fs.existsSync(filePath)) {
@@ -27,5 +27,5 @@ fs.readFile("./docs/data/modules.stage.3.json", "utf8", (err, data) => {
     }
   });
 
-  fs.writeFileSync("./docs/data/modules.stage.4.json", JSON.stringify(modules, null, 2));
+  fs.writeFileSync("./docs/data/modules.stage.5.json", JSON.stringify(modules, null, 2));
 });
diff --git a/scripts/create_module_list.js b/scripts/create_module_list.js
@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import {marked} from "marked";
+import process from "node:process";
 import sanitizeHtml from "sanitize-html";
 
 async function fetchMarkdownData () {
@@ -28,6 +29,7 @@ async function createModuleList () {
   const markdown = await fetchMarkdownData();
   const moduleList = [];
   let category = "";
+  const missingRepoErrors = [];
 
   for (const line of markdown.split("\n")) {
     if (line.startsWith("### ")) {
@@ -45,53 +47,60 @@ async function createModuleList () {
       if (parts.length === 5 || parts.length === 6) {
         const issues = [];
 
-        const url = parts[1].match(/\[(.*?)\]\((.*?)\)/u)[2].trim();
-        if (
-          !url.startsWith("https://github.com") &&
-          !url.startsWith("https://gitlab.com") &&
-          !url.startsWith("https://bitbucket.org")
-        ) {
-          issues.push(`URL: Neither a valid GitHub nor a valid GitLab URL: ${url}.`);
+        const repoCell = parts[1];
+        const repoMatch = repoCell.match(/\[(.*?)\]\((.*?)\)/u);
+
+        if (repoMatch === null) {
+          missingRepoErrors.push(line);
+        } else {
+          const url = repoMatch[2].trim();
+          if (
+            !url.startsWith("https://github.com") &&
+            !url.startsWith("https://gitlab.com") &&
+            !url.startsWith("https://bitbucket.org")
+          ) {
+            issues.push(`URL: Neither a valid GitHub nor a valid GitLab URL: ${url}.`);
+          }
+
+          const maintainer = url.split("/")[3];
+          const name = url.split("/")[4];
+
+          const id = `${maintainer}/${name}`;
+
+          const maintainerLinked = parts[2].match(/\[(.*?)\]\((.*?)\)/u);
+          let maintainerURL = "";
+          if (maintainerLinked !== null) {
+            maintainerURL = maintainerLinked[2];
+          }
+
+          const descriptionMarkdown = parts[3];
+          const descriptionHtml = marked.parseInline(descriptionMarkdown);
+          const descriptionHtmlATarget = descriptionHtml.replaceAll(
+            "<a href=",
+            "<a target=\"_blank\" href="
+          );
+          const description = sanitizeHtml(descriptionHtmlATarget);
+
+          const module = {
+            name,
+            category,
+            url,
+            id,
+            maintainer,
+            maintainerURL,
+            description,
+            issues
+          };
+
+          if (parts.length === 6) {
+            const outdatedMarkdown = parts[4];
+            const outdatedHtml = marked.parseInline(outdatedMarkdown);
+            const outdated = sanitizeHtml(outdatedHtml);
+            module.outdated = outdated;
+          }
+
+          moduleList.push(module);
         }
-
-        const maintainer = url.split("/")[3];
-        const name = url.split("/")[4];
-
-        const id = `${maintainer}/${name}`;
-
-        const maintainerLinked = parts[2].match(/\[(.*?)\]\((.*?)\)/u);
-        let maintainerURL = "";
-        if (maintainerLinked !== null) {
-          maintainerURL = maintainerLinked[2];
-        }
-
-        const descriptionMarkdown = parts[3];
-        const descriptionHtml = marked.parseInline(descriptionMarkdown);
-        const descriptionHtmlATarget = descriptionHtml.replaceAll(
-          "<a href=",
-          "<a target=\"_blank\" href="
-        );
-        const description = sanitizeHtml(descriptionHtmlATarget);
-
-        const module = {
-          name,
-          category,
-          url,
-          id,
-          maintainer,
-          maintainerURL,
-          description,
-          issues
-        };
-
-        if (parts.length === 6) {
-          const outdatedMarkdown = parts[4];
-          const outdatedHtml = marked.parseInline(outdatedMarkdown);
-          const outdated = sanitizeHtml(outdatedHtml);
-          module.outdated = outdated;
-        }
-
-        moduleList.push(module);
       }
     }
   }
@@ -102,6 +111,10 @@ async function createModuleList () {
     modules: sortedModuleList
   };
 
+  if (missingRepoErrors.length > 0) {
+    throw new Error(`[create_module_list] Missing repository link in ${missingRepoErrors.length} line(s):\n${missingRepoErrors.join("\n")}`);
+  }
+
 
   fs.writeFileSync(
     "./docs/data/modules.stage.1.json",
@@ -110,4 +123,7 @@ async function createModuleList () {
   );
 }
 
-createModuleList();
+createModuleList().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});
diff --git a/scripts/expand_module_list_with_repo_data.js b/scripts/expand_module_list_with_repo_data.js
diff --git a/scripts/get_modules.py b/scripts/get_modules.py
