remove rich snippet scraper and update keyword scraper

Author: mazen-r

bing-scraper/README.md

@@ -42,5 +42,4 @@ This Bing.com scraper uses __Python 3.10__ with [scrapfly-sdk](https://pypi.org/
     # or specific scraping areas
     $ poetry run pytest test.py -k test_serp_scraping
     $ poetry run pytest test.py -k test_keyword_scraping
-    $ poetry run pytest test.py -k test_rich_snippets_scraping
     ```

bing-scraper/bing.py

@@ -22,7 +22,6 @@
     "debug":True,
     "os":"linux",
     "auto_scroll":True,
-
 }
 
 
@@ -38,7 +37,7 @@ def parse_serps(response: ScrapeApiResponse) -> List[Dict]:
         url = result.xpath(".//h2/a/@href").get()
         description = result.xpath("normalize-space(.//div/p)").extract_first()
         date = result.xpath(".//span[@class='news_dt']/text()").get()
-        if data is not None and date is not None and len(date) > 12:
+        if data is not None and len(date) > 12:
             date_pattern = re.compile(r"\b\d{2}-\d{2}-\d{4}\b")
             date_pattern.findall(description)
             dates = date_pattern.findall(date)
@@ -59,54 +58,14 @@ def parse_serps(response: ScrapeApiResponse) -> List[Dict]:
         )
     return data
 
-
 def parse_keywords(response: ScrapeApiResponse) -> Dict:
     """parse FAQs and popular keywords on bing search pages"""
     selector = response.selector
-    faqs = []
-    for faq in selector.xpath("//*[*[div[contains(@data-tag, 'RelatedQnA.Item')]]]"):
-        url = faq.xpath(".//a/@href").get()
-        faqs.append(
-            {
-                "query": faq.xpath(".//div[contains(@data-tag, 'RelatedQnA.Item')]/@data-query").get(),
-                "answer": faq.xpath(".//span[contains(@data-tag, 'QnA')]/text()").get(),
-                "title": "".join(faq.xpath(".//div[@class='b_algo']/h2/*//text()").extract()),
-                "domain": url.split("https://")[-1].split("/")[0].replace("www.", "")if url else None,
-                "url": url,
-            }
-        )
     related_keywords = []
     for keyword in selector.xpath(".//li[@class='b_ans']/div/ul/li"):
         related_keywords.append("".join(keyword.xpath(".//a/div//text()").extract()))
 
-    return {"FAQs": faqs, "related_keywords": related_keywords}
-
-
-def parse_rich_snippet(response: ScrapeApiResponse) -> Dict:
-    """parse rich snippets from Bing search"""
-    selector = response.selector
-    data = {}
-    data["title"] = " ".join(selector.xpath("//div[@class='l_ecrd_hero_ttl']//h2//text()").getall())
-    data["link"] = selector.xpath("//div[@class='l_ecrd_hero_ttl']/div/a/@href").get()
-    data["heading"] = " ".join(selector.xpath("//a[@title]/h2/span/text()").getall())
-    data["links"] = {}
-    for item in selector.xpath("//div[contains(@class, 'webicons')]/div"):
-        name = item.xpath(".//a/@title").get()
-        link = item.xpath(".//a/@href").get()
-        data["links"][name] = link
-
-    data["info"] = {}
-    for row in selector.xpath("//div[contains(@class, 'expansion')]/div[contains(@class, 'row')]"):
-        key = row.xpath(".//div/div/a[1]/text()").get().strip()
-        value = row.xpath("string(.//div[not(contains(@class, 'title'))])").get().strip().replace(key, "")
-        data["info"][key] = value
-
-    all_text = ""
-    for div_element in selector.xpath("//div[@class='lite-entcard-blk l_ecrd_bkg_hlt']"):
-        div_text = div_element.xpath("string(.)").get().strip()
-        all_text += div_text + "\n"
-    data["descrption"] = all_text
-    return data
+    return related_keywords
 
 
 async def scrape_search(query: str, max_pages: int = None):
@@ -130,24 +89,13 @@ async def scrape_search(query: str, max_pages: int = None):
     log.success(f"scraped {len(serp_data)} search results from Bing search")
     return serp_data
 
-
 async def scrape_keywords(query: str):
     """scrape bing search pages for keyword data"""
     url = f"https://www.bing.com/search?{urlencode({'q': query})}"
     log.info("scraping Bing search for keyword data")
     response = await SCRAPFLY.async_scrape(ScrapeConfig(url, **BASE_CONFIG, render_js=True))
     keyword_data = parse_keywords(response)
     log.success(
-        f"scraped {len(keyword_data['related_keywords'])} keywords and {len(keyword_data['FAQs'])} FAQs from Bing search"
+        f"scraped {len(keyword_data)} keywords from Bing search"
     )
     return keyword_data
-
-
-async def scrape_rich_snippets(query: str):
-    """scrape bing search pages for rich snippets data"""
-    url = f"https://www.bing.com/search?{urlencode({'q': query})}"
-    log.info("scraping Bing search for keyword data")
-    response = await SCRAPFLY.async_scrape(ScrapeConfig(url, asp=True, country="GB", render_js=True))
-    rich_snippet_data = parse_rich_snippet(response)
-    log.success(f"scraped {len(rich_snippet_data)} rich snippets fields from Bing search")
-    return rich_snippet_data

bing-scraper/results/keywords.json

@@ -1,72 +1,18 @@
-{
-  "FAQs": [
-    {
-      "query": "How to scrape emails from a website?",
-      "answer": "We will have to set up our scraper to click on each profile and extract their email. Next, you will need a web scraper that can scrape emails from any website. For this example, we will download and install ParseHub, a free and powerful web scraper that works with any website. Now it’s time to get scraping. Open ParseHub and click on “New Project”.",
-      "title": "How to Scrape Emails from any Website | ParseHub",
-      "domain": "bing.com",
-      "url": "https://www.bing.com/ck/a?!&&p=41cadb349b48845d81946a4548ede103e848c4600607f191c287af779dcfe29dJmltdHM9MTc1NzI4OTYwMA&ptn=3&ver=2&hsh=4&fclid=1ecd1879-483a-6ddd-344b-0e1849f86cee&u=a1aHR0cHM6Ly93d3cucGFyc2VodWIuY29tL2Jsb2cvc2NyYXBlLWVtYWlscy8&ntb=1"
-    },
-    {
-      "query": "How does email scraping work?",
-      "answer": "The key is that scraping tools — or email scrapers — are built to ",
-      "title": "How to Scrape Emails for Marketing (+ 6 Top Tools in 2025)",
-      "domain": "bing.com",
-      "url": "https://www.bing.com/ck/a?!&&p=8757a5f5e1a0f4c8e2b87e05d94f4add67393122202cac7d08398b06eaf90256JmltdHM9MTc1NzI4OTYwMA&ptn=3&ver=2&hsh=4&fclid=1ecd1879-483a-6ddd-344b-0e1849f86cee&u=a1aHR0cHM6Ly93d3cubGluZHkuYWkvYmxvZy9zY3JhcGluZy1lbWFpbHM&ntb=1"
-    },
-    {
-      "query": "What happens if you scrape emails?",
-      "answer": "Technical: Bad scraping can ",
-      "title": "How to Scrape Emails for Marketing (+ 6 Top Tools in 2025)",
-      "domain": "bing.com",
-      "url": "https://www.bing.com/ck/a?!&&p=8757a5f5e1a0f4c8e2b87e05d94f4add67393122202cac7d08398b06eaf90256JmltdHM9MTc1NzI4OTYwMA&ptn=3&ver=2&hsh=4&fclid=1ecd1879-483a-6ddd-344b-0e1849f86cee&u=a1aHR0cHM6Ly93d3cubGluZHkuYWkvYmxvZy9zY3JhcGluZy1lbWFpbHM&ntb=1"
-    },
-    {
-      "query": "Should you use a web scraper for email marketing?",
-      "answer": "With the help of a free web scraper and by carefully selecting your lead sources, you can quickly build a high-quality email list. You can then use this list for email marketing efforts or use it as a custom audience in Google or Facebook Ads. Also, many companies web scrape email lists for their prospecting or cold email outreach.",
-      "title": "How to Scrape Emails from any Website | ParseHub",
-      "domain": "bing.com",
-      "url": "https://www.bing.com/ck/a?!&&p=41cadb349b48845d81946a4548ede103e848c4600607f191c287af779dcfe29dJmltdHM9MTc1NzI4OTYwMA&ptn=3&ver=2&hsh=4&fclid=1ecd1879-483a-6ddd-344b-0e1849f86cee&u=a1aHR0cHM6Ly93d3cucGFyc2VodWIuY29tL2Jsb2cvc2NyYXBlLWVtYWlscy8&ntb=1"
-    },
-    {
-      "query": "What is the best tool for scraping email addresses?",
-      "answer": "This saves all the unique scraped email addresses to a CSV file named \"scraped_emails.csv\". While ",
-      "title": "How to Scrape Emails from Any Website in 2025: The Ultimate Guide",
-      "domain": "bing.com",
-      "url": "https://www.bing.com/ck/a?!&&p=b81d9e6a78395567b82499d19db64ba9600724423404794bfa655b0ddefe6ef9JmltdHM9MTc1NzI4OTYwMA&ptn=3&ver=2&hsh=4&fclid=1ecd1879-483a-6ddd-344b-0e1849f86cee&u=a1aHR0cHM6Ly93d3cubWFya2V0aW5nc2Nvb3AuY29tL3RlY2gvd2ViLXNjcmFwaW5nL2hvdy10by1zY3JhcGUtZW1haWxzLWZyb20tYW55LXdlYnNpdGUtZm9yLXNhbGVzLXByb3NwZWN0aW5nLw&ntb=1"
-    },
-    {
-      "query": "How to scrape emails from Google Maps?",
-      "answer": "If that’s not enough, we wrote a separate guide on scraping emails from Google Maps. To run the scraper, enter your HasData API key and either a list of websites or keywords, depending on the method you choose. Then, run the scraper: When it’s done, you can copy the data table (or part of it) or download the data as JSON or CSV.",
-      "title": "Scrape Emails from Any Websites: From Python to AI Tools",
-      "domain": "bing.com",
-      "url": "https://www.bing.com/ck/a?!&&p=834f7d6add2db48cca6a6ef4bb289361a9bb65158afb1ef5f0db1296039c7d73JmltdHM9MTc1NzI4OTYwMA&ptn=3&ver=2&hsh=4&fclid=1ecd1879-483a-6ddd-344b-0e1849f86cee&u=a1aHR0cHM6Ly9oYXNkYXRhLmNvbS9ibG9nL2VtYWlsLXNjcmFwaW5n&ntb=1"
-    }
-  ],
-  "related_keywords": [
-    "free email scraping tool",
-    "website email scraper",
-    "web email scraper",
-    "best free online email extractor",
-    "how to delete emails from website",
-    "pull email addresses from website",
-    "extract email from website free",
-    "extract emails from website",
-    "free email grabber",
-    "extract email address from website",
-    "extract email addresses from website",
-    "email extractor from website free",
-    "free email scraping tool",
-    "website email scraper",
-    "web email scraper",
-    "best free online email extractor",
-    "how to delete emails from website",
-    "pull email addresses from website",
-    "extract email from website free",
-    "extract emails from website",
-    "free email grabber",
-    "extract email address from website",
-    "extract email addresses from website",
-    "email extractor from website free"
-  ]
-}
+[
+  "email scraper",
+  "email scraping software",
+  "extract emails from web page",
+  "scrape emails from website free",
+  "scrape email addresses from websites",
+  "scrape emails from website",
+  "pull email addresses from website",
+  "how to delete emails from website",
+  "email scraper",
+  "email scraping software",
+  "extract emails from web page",
+  "scrape emails from website free",
+  "scrape email addresses from websites",
+  "scrape emails from website",
+  "pull email addresses from website",
+  "how to delete emails from website"
+]