fix: github unauthenticated fetch (#69)

Author: idosal

src/api/tools/commonTools.ts

@@ -123,12 +123,26 @@ export async function fetchDocumentation({
       // Create array of all location+branch combinations to try
       const fetchPromises = possibleLocations.flatMap((location) => [
         {
-          promise: fetchFileFromGitHub(owner, repo, "main", location, env),
+          promise: fetchFileFromGitHub(
+            owner,
+            repo,
+            "main",
+            location,
+            env,
+            false,
+          ),
           location,
           branch: "main",
         },
         {
-          promise: fetchFileFromGitHub(owner, repo, "master", location, env),
+          promise: fetchFileFromGitHub(
+            owner,
+            repo,
+            "master",
+            location,
+            env,
+            false,
+          ),
           location,
           branch: "master",
         },
@@ -186,6 +200,7 @@ export async function fetchDocumentation({
         "main",
         "README.md",
         env,
+        false,
       );
       fileUsed = "readme.md (main branch)";
 

src/api/utils/github.ts

@@ -8,6 +8,8 @@ import { fetchRawFile } from "./githubClient.js";
  * @param repo - Repository name
  * @param branch - Branch name (main, master)
  * @param path - File path within the repository
+ * @param env - Environment for GitHub token
+ * @param useAuth - Whether to use authentication
  * @returns File content or null if not found
  */
 export async function fetchFileFromGitHub(
@@ -16,8 +18,9 @@ export async function fetchFileFromGitHub(
   branch: string,
   path: string,
   env: any,
+  useAuth = false,
 ): Promise<string | null> {
-  return await fetchRawFile(owner, repo, branch, path, env);
+  return await fetchRawFile(owner, repo, branch, path, env, useAuth);
 }
 
 // Helper: search for a file in a GitHub repository using the GitHub Search API
@@ -38,6 +41,7 @@ export async function searchGitHubRepo(
         cachedPath.branch,
         cachedPath.path,
         env,
+        true,
       );
       if (content) {
         return content;

src/api/utils/githubClient.ts

@@ -117,20 +117,22 @@ async function respectRateLimits(): Promise<void> {
  * @param options - Fetch options
  * @param env - Environment containing GitHub token if available
  * @param retryCount - Current retry attempt (used internally)
+ * @param useAuth - Whether to include authorization header if token is available (default: true)
  * @returns The API response or null if failed
  */
 export async function githubApiRequest(
   url: string,
   options: RequestInit = {},
   env: any,
   retryCount = 0,
+  useAuth = true,
 ): Promise<Response | null> {
   try {
     // Extract repository context for metrics
     const repoContext = extractRepoContextFromUrl(url);
 
     // Track GitHub query count using Cloudflare analytics
-    if (env?.CLOUDFLARE_ANALYTICS && retryCount === 0) {
+    if (env.CLOUDFLARE_ANALYTICS && retryCount === 0) {
       env.CLOUDFLARE_ANALYTICS.writeDataPoint({
         blobs: [url, repoContext],
         doubles: [1],
@@ -141,15 +143,15 @@ export async function githubApiRequest(
     // Wait for rate limit if necessary
     await respectRateLimits();
 
-    // Add GitHub authentication if token is available
+    // Add GitHub authentication if token is available and useAuth is true
     const headers = new Headers(options.headers || {});
     headers.set("Accept", "application/vnd.github.v3+json");
     headers.set(
       "User-Agent",
       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
     );
 
-    if (env?.GITHUB_TOKEN) {
+    if (useAuth && env.GITHUB_TOKEN) {
       headers.set("Authorization", `token ${env.GITHUB_TOKEN}`);
     }
 
@@ -286,19 +288,21 @@ export async function searchFileByName(
  * @param branch - Branch name
  * @param path - File path
  * @param env - Environment for GitHub token
+ * @param useAuth - Whether to use authentication
  */
 export async function fetchRawFile(
   owner: string,
   repo: string,
   branch: string,
   path: string,
   env: any,
+  useAuth = false,
 ): Promise<string | null> {
   const url = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`;
 
   // Raw GitHub content doesn't need the GitHub API token
   // But we still use the client for rate limiting
-  const response = await githubApiRequest(url, {}, env);
+  const response = await githubApiRequest(url, {}, env, 0, useAuth);
 
   if (!response || !response.ok) {
     return null;