Create llms-txt-validation.js (#166)

* Create llms-txt-validation.js

Fetches /llms.txt and returns a simple validity flag (1 or 0). It checks that the response:

- Succeeds (HTTP 2xx)
- Has a Content-Type of text/plain
- Contains balanced Markdown code fences (…)
- Has matching counts of [ vs. ] and ( vs. )

If any check fails, it returns 0; otherwise it returns 1.

* Update llms-txt-validation.js

* Update llms-txt-validation.js

* Return JSON

* JSON stringify

* Update llms-txt-validation.js

* Rename llms-txt-validation.js to dist/llms-txt-validation.js

* Update llms-txt-validation.js

* Update llms-txt-validation.js

* Rename llms-txt-validation.js to llms_txt_validation.js

* Update llms_txt_validation.js

* Change to Boolean

* Validation Changes

Making LLMs.txt scoring less strict, so present - yet malformed llms.txt files - will always be marked as "valid", rather than invalid to make it easier to count total adoption whilst understanding which files have errors or not.

---------

Co-authored-by: Barry Pollard <[email protected]>
Co-authored-by: Max Ostapenko <[email protected]>

Author: 3 people

dist/llms_txt_validation.js

@@ -0,0 +1,23 @@
+//[llms-txt-valid]
+
+const fetchWithTimeout = (url) => {
+  var controller = new AbortController();
+  setTimeout(() => {controller.abort()}, 5000);
+  return fetch(url, {signal: controller.signal});
+}
+
+return fetchWithTimeout('/llms.txt')
+  .then(response => {
+    if (!response.ok) return JSON.stringify({valid: false, message: response.status, error: "Non OK status code"});
+    const ct = response.headers.get('Content-Type')||'';
+    if (!ct.toLowerCase().includes('text/plain')) return JSON.stringify({valid: false, message: ct, error: "Invalid content type"});
+    return response.text().then(text => {
+      const m = s => (text.match(new RegExp(`\\${s}`,'g'))||[]).length;
+      if (m('[')!==m(']')||m('(')!==m(')')) return JSON.stringify({valid: true, error: "Invalid markdown: Unmatched braces"});
+      if ((text.match(/```/g)||[]).length %2) return JSON.stringify({valid: true, error: "Invalid markdown: Uneven code fences"});
+      return JSON.stringify({valid: true});
+    });
+  })
+  .catch(error => {
+    return JSON.stringify({valid: false, message: error.message, error: error});
+  });