[BAEL-9328] Add code for Web Crawler using WebMagic Library (#18712)

* [BAEL-9328] Add code for Web Crawler using WebMagic Library

* [BAEL-9328] Add Live Tests for Web Crawler using WebMagic Library

---------

Co-authored-by: rajatgarg <[email protected]>

Author: rajat-garg

libraries-4/pom.xml

@@ -99,6 +99,17 @@
             <artifactId>github-api</artifactId>
             <version>${github-api.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-core</artifactId>
+            <version>1.0.3</version>
+        </dependency>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-extension</artifactId>
+            <version>1.0.3</version>
+        </dependency>
     </dependencies>
 
     <properties>

libraries-4/src/main/java/com/baeldung/webmagic/Book.java

@@ -0,0 +1,20 @@
+package com.baeldung.webmagic;
+
+public class Book {
+    private final String title;
+    private final String price;
+
+    public Book(String title, String price) {
+        this.title = title;
+        this.price = price;
+    }
+
+    public String getTitle() {
+        return title;
+    }
+
+    public String getPrice() {
+        return price;
+    }
+}
+

libraries-4/src/main/java/com/baeldung/webmagic/BookScraper.java

@@ -0,0 +1,50 @@
+package com.baeldung.webmagic;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class BookScraper implements PageProcessor {
+
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
+    private final List<Book> books = new ArrayList<>();
+
+    @Override
+    public void process(Page page) {
+        var bookNodes = page.getHtml().css("article.product_pod");
+
+        for (int i = 0; i < Math.min(10, bookNodes.nodes().size()); i++) {
+            var book = bookNodes.nodes().get(i);
+
+            String title = book.css("h3 a", "title").get();
+            String price = book.css(".price_color", "text").get();
+
+            books.add(new Book(title, price));
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    public List<Book> getBooks() {
+        return Collections.unmodifiableList(books);
+    }
+
+    public static void main(String[] args) {
+        BookScraper bookScraper = new BookScraper();
+        Spider.create(bookScraper)
+                .addUrl("https://books.toscrape.com/")
+                .thread(1)
+                .run();
+
+        bookScraper.getBooks().forEach(book ->
+                System.out.println("Title: " + book.getTitle() + " | Price: " + book.getPrice()));
+    }
+}

libraries-4/src/test/java/com/baeldung/webmagic/BookScraperLiveTest.java

@@ -0,0 +1,94 @@
+package com.baeldung.webmagic;
+
+import us.codecraft.webmagic.Spider;
+import org.junit.jupiter.api.Test;
+import java.util.List;
+import static org.junit.jupiter.api.Assertions.*;
+
+public class BookScraperLiveTest {
+
+    @Test
+    void whenScrapeABookSite_thenShouldReturnTitleAndPrice() {
+        BookScraper scraper = new BookScraper();
+
+        Spider.create(scraper)
+                .addUrl("https://books.toscrape.com/")
+                .thread(1)
+                .run();
+
+        List<Book> books = scraper.getBooks();
+
+        assertFalse(books.isEmpty(), "Expected to scrape at least one book.");
+        assertTrue(books.size() <= 10, "Should not scrape more than 10 books.");
+
+        for (Book book : books) {
+            assertNotNull(book.getTitle(), "Book title should not be null.");
+            assertFalse(book.getTitle().isBlank(), "Book title should not be blank.");
+            assertNotNull(book.getPrice(), "Book price should not be null.");
+            assertTrue(book.getPrice().matches("£?\\d+(\\.\\d{2})?"), "Book price format seems invalid: " + book.getPrice());
+        }
+    }
+
+    @Test
+    void whenScrapeBookSite_thenParseAndSortBookPrices() {
+        BookScraper scraper = new BookScraper();
+        Spider.create(scraper)
+                .addUrl("https://books.toscrape.com/")
+                .thread(1)
+                .run();
+
+        List<Book> books = scraper.getBooks();
+        assertFalse(books.isEmpty(), "No books were scraped.");
+
+        // Extract numerical prices from string (e.g., £51.77 -> 51.77)
+        List<Double> prices = books.stream()
+                .map(Book::getPrice)
+                .map(p -> p.replace("£", ""))
+                .map(Double::parseDouble)
+                .toList();
+
+        List<Double> sorted = prices.stream().sorted((a, b) -> Double.compare(b, a)).toList();
+
+        assertEquals(sorted, prices.stream().sorted((a, b) -> Double.compare(b, a)).toList(),
+                "Prices are not in descending order after sorting.");
+    }
+
+    @Test
+    void whenScrapeBookSite_thenBookTitlesShouldContainExpectedWords() {
+        BookScraper scraper = new BookScraper();
+        Spider.create(scraper)
+                .addUrl("https://books.toscrape.com/")
+                .thread(1)
+                .run();
+
+        List<Book> books = scraper.getBooks();
+        assertFalse(books.isEmpty(), "No books were scraped.");
+
+        boolean foundKeyword = books.stream()
+                .map(Book::getTitle)
+                .anyMatch(title -> title.toLowerCase().matches(".*\\b(book|story|novel|guide|life)\\b.*"));
+
+        assertTrue(foundKeyword, "No book titles contain expected keywords.");
+    }
+
+
+    @Test
+    void whenScrapeBookSiteMultipleTimes_thenBookCountShouldStableBetweenRuns() {
+        BookScraper scraper1 = new BookScraper();
+        Spider.create(scraper1)
+                .addUrl("https://books.toscrape.com/")
+                .thread(1)
+                .run();
+
+        BookScraper scraper2 = new BookScraper();
+        Spider.create(scraper2)
+                .addUrl("https://books.toscrape.com/")
+                .thread(1)
+                .run();
+
+        int count1 = scraper1.getBooks().size();
+        int count2 = scraper2.getBooks().size();
+
+        assertEquals(count1, count2, "Book count is not stable between two runs.");
+    }
+}