fix(scraper): resolve relative links against the post-redirect URL

fetch_links_from_url now returns the effective URL (response.geturl()) and
process_url uses it as the base for urljoin. Without this, scraping a URL that
redirects - e.g. a GitHub Pages project page where '/repo' 301-redirects to
'/repo/' - resolved relative links one directory too high, causing 404s on
every file. Add a regression test.

Author: franckferman

src/MetaDetective/MetaDetective.py

@@ -661,7 +661,7 @@ def __init__(self, extensions: List[str]):
             self.extensions = {ext.lower() for ext in EXTENSIONS}
         self.css_js_pattern = re.compile(r"\.(css|js)($|\?|#)")
 
-    def fetch_links_from_url(self, url: str, timeout: int = DEFAULT_HTTP_TIMEOUT) -> List[str]:
+    def fetch_links_from_url(self, url: str, timeout: int = DEFAULT_HTTP_TIMEOUT) -> Tuple[str, List[str]]:
         """
         Fetch all links from a given URL.
 
@@ -670,14 +670,19 @@ def fetch_links_from_url(self, url: str, timeout: int = DEFAULT_HTTP_TIMEOUT) ->
             timeout: Request timeout in seconds
 
         Returns:
-            List of links found on the page
+            Tuple of (effective URL after redirects, links found on the page).
+            The effective URL must be used as the base for resolving relative
+            links, since e.g. GitHub Pages 301-redirects '/repo' to '/repo/'.
         """
         try:
             request = urllib.request.Request(url, headers={'User-Agent': USER_AGENT})
             with urllib.request.urlopen(request, timeout=timeout) as response:
+                # Final URL after any redirects; relative links resolve against it.
+                effective_url = response.geturl() or url
+
                 content_type = response.headers.get('Content-Type', '').split(';')[0]
                 if 'text' not in content_type and 'application' not in content_type:
-                    return []
+                    return effective_url, []
 
                 raw_data = response.read()
 
@@ -688,7 +693,7 @@ def fetch_links_from_url(self, url: str, timeout: int = DEFAULT_HTTP_TIMEOUT) ->
                         data = raw_data.decode('latin-1')
                     except UnicodeDecodeError:
                         Logger.warning(f"Unable to decode content from {url}")
-                        return []
+                        return effective_url, []
 
                 parser = LinkParser()
                 parser.feed(data)
@@ -701,20 +706,20 @@ def fetch_links_from_url(self, url: str, timeout: int = DEFAULT_HTTP_TIMEOUT) ->
                     if not link or link.strip() == '':
                         continue
                     filtered_links.append(link)
-                return filtered_links
+                return effective_url, filtered_links
 
         except urllib.error.URLError as e:
             if url.startswith("mailto:"):
                 Logger.info(f"Found mailto link {url}")
             else:
                 Logger.error(f"Unable to open {url} Reason: {e}")
-            return []
+            return url, []
         except urllib.error.HTTPError as e:
             Logger.error(f"HTTP Error for URL {url} Reason: {e.code} - {e.reason}")
-            return []
+            return url, []
         except Exception as e:
             Logger.error(f"Unexpected error fetching {url} Reason: {e}")
-            return []
+            return url, []
 
     def is_valid_file_link(self, link: str) -> bool:
         """
@@ -883,12 +888,12 @@ def process_url(self, task: ScrapingTask, task_queue: queue.Queue) -> None:
 
         self.rate_limiter.wait()
 
-        links = self.scraper.fetch_links_from_url(task.url)
+        base_url, links = self.scraper.fetch_links_from_url(task.url)
 
         file_links = []
         for link in links:
             if self.scraper.is_valid_file_link(link):
-                absolute_url = urljoin(task.url, link)
+                absolute_url = urljoin(base_url, link)
                 file_links.append(absolute_url)
 
         if self.download_dir and not self.scan:
@@ -910,7 +915,7 @@ def process_url(self, task: ScrapingTask, task_queue: queue.Queue) -> None:
                 if not link.startswith(('http://', 'https://', '/')):
                     continue
 
-                absolute_link = urljoin(task.url, link)
+                absolute_link = urljoin(base_url, link)
                 parsed_link = urlparse(absolute_link)
 
                 # Validate that it's an HTTP/HTTPS URL

tests/test_MetaDetective.py

@@ -491,5 +491,27 @@ def test_paths_are_not_urls(self):
         self.assertFalse(md.looks_like_url("ftp://example.com/f"))
 
 
+class TestWebScraperRedirectBase(unittest.TestCase):
+    """Relative links must resolve against the URL *after* redirects."""
+
+    @patch("src.MetaDetective.MetaDetective.urllib.request.urlopen")
+    def test_relative_links_resolve_against_effective_url(self, mock_urlopen):
+        # Simulate GitHub Pages 301-redirecting '/repo' -> '/repo/'
+        resp = mock_urlopen.return_value.__enter__.return_value
+        resp.geturl.return_value = "https://host/repo/"
+        resp.headers.get.return_value = "text/html"
+        resp.read.return_value = b'<a href="lab/report.pdf">go</a>'
+
+        scraper = md.WebScraper(["pdf"])
+        base, links = scraper.fetch_links_from_url("https://host/repo")
+
+        self.assertEqual(base, "https://host/repo/")
+        self.assertIn("lab/report.pdf", links)
+        self.assertEqual(
+            md.urljoin(base, links[0]),
+            "https://host/repo/lab/report.pdf",
+        )
+
+
 if __name__ == "__main__":
     unittest.main()