Merge branch 'main' of https://github.com/SculptTechProject/GinioCrawler

Author: SculptTechProject

src/app_gui.py

@@ -1,8 +1,14 @@
-import os, csv, asyncio, threading, subprocess, sys
+import asyncio
+import csv
+import os
+import subprocess
+import sys
+import threading
 import tkinter as tk
-from tkinter import ttk, messagebox, filedialog, simpledialog
-from pathlib import Path
 from datetime import datetime
+from pathlib import Path
+from tkinter import filedialog, messagebox, simpledialog, ttk
+
 from dotenv import load_dotenv, set_key
 
 from src.main import run, write_excel
@@ -141,17 +147,25 @@ def build_ui():
     root.title("GinioCrawler")
     root.geometry("560x220")
     tk.Label(root, text="Fraza do wyszukania:").pack(anchor="w", padx=12, pady=(12, 0))
-    entry_query = tk.Entry(root); entry_query.pack(fill="x", padx=12, pady=6); entry_query.focus()
-    frm = tk.Frame(root); frm.pack(fill="x", padx=12, pady=(0, 6))
+    entry_query = tk.Entry(root)
+    entry_query.pack(fill="x", padx=12, pady=6)
+    entry_query.focus()
+    frm = tk.Frame(root)
+    frm.pack(fill="x", padx=12, pady=(0, 6))
     tk.Label(frm, text="Folder wyjściowy:").pack(side="left")
     out_dir_var = tk.StringVar(value=str((Path.cwd() / "wyniki")))
-    entry_dir = tk.Entry(frm, textvariable=out_dir_var); entry_dir.pack(side="left", fill="x", expand=True, padx=(8, 6))
+    entry_dir = tk.Entry(frm, textvariable=out_dir_var)
+    entry_dir.pack(side="left", fill="x", expand=True, padx=(8, 6))
     tk.Button(frm, text="Wybierz…", command=choose_dir).pack(side="left")
-    btn_start = tk.Button(root, text="Start", command=start); btn_start.pack(padx=12, pady=6)
-    prog = ttk.Progressbar(root, mode="indeterminate"); prog.pack(fill="x", padx=12, pady=(4, 8))
-    status = tk.StringVar(value="Gotowy"); tk.Label(root, textvariable=status, anchor="w").pack(fill="x", padx=12, pady=(0, 8))
+    btn_start = tk.Button(root, text="Start", command=start)
+    btn_start.pack(padx=12, pady=6)
+    prog = ttk.Progressbar(root, mode="indeterminate")
+    prog.pack(fill="x", padx=12, pady=(4, 8))
+    status = tk.StringVar(value="Gotowy")
+    tk.Label(root, textvariable=status, anchor="w").pack(fill="x", padx=12, pady=(0, 8))
     return root
 
+
 if __name__ == "__main__":
     build_ui()
     root.mainloop()

src/main.py

@@ -1,11 +1,18 @@
+import asyncio
 import csv
+import os
+import re
+import sys
+import urllib.parse
+import urllib.robotparser as rp
 from datetime import datetime
-import os, sys, re, asyncio, httpx, urllib.parse, urllib.robotparser as rp
-from bs4 import BeautifulSoup as BS
+from pathlib import Path
+
+import httpx
 import pandas as pd
-from openpyxl.styles import Font, Alignment
+from bs4 import BeautifulSoup as BS
+from openpyxl.styles import Alignment, Font
 from openpyxl.utils import get_column_letter
-from pathlib import Path
 
 EMAIL_RE = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.I)
 PL_PHONE_RE = re.compile(r"(?:\+48\s?)?(?:\d{3}[\s-]?\d{3}[\s-]?\d{3})")
@@ -35,7 +42,9 @@ def write_excel(csv_path, xlsx_path):
         ws.auto_filter.ref = ws.dimensions
         for col in ws.columns:
             length = max(len(str(c.value)) if c.value else 0 for c in col)
-            ws.column_dimensions[col[0].column_letter].width = min(max(12, int(length * 0.9)), 60)
+            ws.column_dimensions[col[0].column_letter].width = min(
+                max(12, int(length * 0.9)), 60
+            )
         cols = [c for c in ("url", "contact_url") if c in df.columns]
         for row in range(2, ws.max_row + 1):
             for name in cols:
@@ -47,7 +56,6 @@ def write_excel(csv_path, xlsx_path):
                     cell.style = "Hyperlink"
 
 
-
 def in_robots(url: str) -> bool:
     base = f"{urllib.parse.urlsplit(url).scheme}://{urllib.parse.urlsplit(url).netloc}"
     robots = rp.RobotFileParser()
@@ -144,9 +152,12 @@ async def run(query):
 
 if __name__ == "__main__":
     try:
-        import os, sys, csv, asyncio
-        from pathlib import Path
+        import asyncio
+        import csv
+        import os
+        import sys
         from datetime import datetime
+        from pathlib import Path
 
         q = input("Podaj szukane słowo: ").strip()
         if not q:

tests/conftest.py

@@ -3,4 +3,4 @@
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 if str(REPO_ROOT) not in sys.path:
-    sys.path.insert(0, str(REPO_ROOT))
+    sys.path.insert(0, str(REPO_ROOT))

tests/test_crawl_run.py

@@ -1,27 +1,51 @@
-import httpx, respx, pytest
+import httpx
+import pytest
+import respx
+
 from src.main import crawl_one, run
 
+
 @respx.mock
 @pytest.mark.anyio
 async def test_crawl_one_merges_contact(monkeypatch):
     monkeypatch.setattr("src.main.in_robots", lambda u: True)
-    respx.get("https://site.test/").mock(return_value=httpx.Response(200, text="""
-        <a href="/contact">Contact</a><p>[email protected]</p>"""))
-    respx.get("https://site.test/contact").mock(return_value=httpx.Response(200, text="""
-        <p>[email protected] 123 456 789</p>"""))
+    respx.get("https://site.test/").mock(
+        return_value=httpx.Response(
+            200,
+            text="""
+        <a href="/contact">Contact</a><p>[email protected]</p>""",
+        )
+    )
+    respx.get("https://site.test/contact").mock(
+        return_value=httpx.Response(
+            200,
+            text="""
+        <p>[email protected] 123 456 789</p>""",
+        )
+    )
     async with httpx.AsyncClient() as c:
         out = await crawl_one("https://site.test/", c)
-    assert {"[email protected]","[email protected]"} <= set(out["emails"])
+    assert {"[email protected]", "[email protected]"} <= set(out["emails"])
+
 
 @respx.mock
 @pytest.mark.anyio
 async def test_run_full(monkeypatch):
-    respx.get("https://serpapi.com/search").mock(return_value=httpx.Response(
-        200, json={"organic_results":[{"link":"https://a.pl"},{"link":"https://b.pl"}]}
-    ))
+    respx.get("https://serpapi.com/search").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "organic_results": [{"link": "https://a.pl"}, {"link": "https://b.pl"}]
+            },
+        )
+    )
     monkeypatch.setattr("src.main.in_robots", lambda u: True)
-    respx.get("https://a.pl").mock(return_value=httpx.Response(200, text="<title>A</title>"))
-    respx.get("https://b.pl").mock(return_value=httpx.Response(200, text="<title>B</title>"))
-    monkeypatch.setenv("SERPAPI_KEY","x")
+    respx.get("https://a.pl").mock(
+        return_value=httpx.Response(200, text="<title>A</title>")
+    )
+    respx.get("https://b.pl").mock(
+        return_value=httpx.Response(200, text="<title>B</title>")
+    )
+    monkeypatch.setenv("SERPAPI_KEY", "x")
     out = await run("foo")
-    assert {r["title"] for r in out} == {"A","B"}
+    assert {r["title"] for r in out} == {"A", "B"}

tests/test_excel.py

@@ -1,21 +1,34 @@
 from pathlib import Path
+
 import pandas as pd
+
 from src.app_gui import save_results
 from src.main import write_excel as real_write_excel
 
+
 def test_save_results_creates_files(tmp_path, monkeypatch):
     root_out = tmp_path / "wyniki"
-    data = [{"url":"https://x","title":"X","emails":["a@x"],"phones":["123"],"contact_url":None}]
+    data = [
+        {
+            "url": "https://x",
+            "title": "X",
+            "emails": ["a@x"],
+            "phones": ["123"],
+            "contact_url": None,
+        }
+    ]
 
     called = {}
+
     def fake_write(csv_path, xlsx_path):
-        called["csv"] = csv_path; called["xlsx"] = xlsx_path
+        called["csv"] = csv_path
+        called["xlsx"] = xlsx_path
         return real_write_excel(csv_path, xlsx_path)
 
     monkeypatch.setattr("src.app_gui.write_excel", fake_write)
 
     csv_p, xlsx_p = save_results(data, "20250101_000000", root_out)
     assert Path(csv_p).exists() and Path(xlsx_p).exists()
     df = pd.read_excel(xlsx_p)
-    assert {"url","title","emails","phones","contact_url"}.issubset(df.columns)
+    assert {"url", "title", "emails", "phones", "contact_url"}.issubset(df.columns)
     assert called["csv"] == str(csv_p) and called["xlsx"] == str(xlsx_p)

tests/test_gui.py

@@ -1,40 +1,49 @@
 from unittest.mock import patch
 
+
 def test_ensure_api_key_env(monkeypatch):
-    monkeypatch.setenv("SERPAPI_KEY","secret")
+    monkeypatch.setenv("SERPAPI_KEY", "secret")
     from src.app_gui import ensure_api_key
+
     assert ensure_api_key() == "secret"
 
+
 def test_ensure_api_key_prompt(tmp_path, monkeypatch):
     monkeypatch.delenv("SERPAPI_KEY", raising=False)
     monkeypatch.setenv("APPDATA", str(tmp_path))
     from src.app_gui import ensure_api_key
+
     with patch("src.app_gui.simpledialog.askstring", return_value="abc"):
         assert ensure_api_key() == "abc"
 
+
 def test_start_calls_run_without_threading(monkeypatch):
     import src.app_gui as g
+
     g.build_ui()
-    g.entry_query.delete(0,'end'); g.entry_query.insert(0,"kawa")
-    monkeypatch.setenv("SERPAPI_KEY","x")
+    g.entry_query.delete(0, "end")
+    g.entry_query.insert(0, "kawa")
+    monkeypatch.setenv("SERPAPI_KEY", "x")
 
     class DummyThread:
-        def __init__(self, target, daemon): self.target = target
-        def start(self): self.target()
+        def __init__(self, target, daemon):
+            self.target = target
+
+        def start(self):
+            self.target()
 
     monkeypatch.setattr("src.app_gui.threading.Thread", DummyThread)
     monkeypatch.setattr(g.root, "after", lambda ms, fn: fn(), raising=False)
 
     async def fake_run(q):
-        return [{"url":"u","title":"t","emails":[],"phones":[],"contact_url":None}]
+        return [
+            {"url": "u", "title": "t", "emails": [], "phones": [], "contact_url": None}
+        ]
+
     monkeypatch.setattr("src.app_gui.run", fake_run)
 
     with patch.object(g, "messagebox"):
         g.start()
         assert g.btn_start["state"] == "normal"
         assert "OK — zapisano" in g.status.get()
-        try:
-            g.root.destroy()
-        except Exception:
-            pass
 

tests/test_net.py

@@ -1,15 +1,22 @@
-import httpx, respx, pytest
+import httpx
+import pytest
+import respx
+
 from src.main import fetch
 
+
 @respx.mock
 @pytest.mark.anyio
 async def test_fetch_ok(monkeypatch):
     monkeypatch.setattr("src.main.in_robots", lambda u: True)
-    respx.get("https://x.test/ok").mock(return_value=httpx.Response(200, text="<h1>ok</h1>"))
+    respx.get("https://x.test/ok").mock(
+        return_value=httpx.Response(200, text="<h1>ok</h1>")
+    )
     async with httpx.AsyncClient() as c:
         html = await fetch("https://x.test/ok", c)
     assert "ok" in html
 
+
 @respx.mock
 @pytest.mark.anyio
 async def test_fetch_respects_robots(monkeypatch):

tests/test_parse.py

@@ -1,8 +1,10 @@
-from src.main import parse_info, absolutize
+from src.main import absolutize, parse_info
+
 
 def test_absolutize():
     assert absolutize("https://ex.com/dir/", "../a") == "https://ex.com/a"
 
+
 def test_parse_info_extracts_contact():
     html = """
     <html><head><title> ACME </title></head>