Boost Scraper Test Coverage (#6)

* dftype info scrape unit test

* boost test coverage for df scraper

Author: muhrifqii

scraper/src/df/models.rs

@@ -12,6 +12,17 @@ pub struct DfTypeInfo {
     pub description: String,
 }
 
+impl DfTypeInfo {
+    pub fn new(df_type: DfType, cannon: u32, non_cannon: u32, description: String) -> Self {
+        Self {
+            df_type,
+            cannon_count: cannon,
+            non_cannon_count: non_cannon,
+            description,
+        }
+    }
+}
+
 impl std::fmt::Display for DfTypeInfo {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(
@@ -127,12 +138,6 @@ impl PartialEq for DevilFruit {
     }
 }
 
-#[derive(Debug)]
-pub struct DfUser {
-    pub id: String,
-    pub name: String,
-}
-
 #[derive(Debug)]
 pub struct Character {
     pub id: String,

scraper/src/df/parser.rs

@@ -31,7 +31,7 @@ pub struct CanonLogiaParser;
 impl DfTypeParser for CanonZoanParser {
     fn parse(&self, html: &Html) -> Result<Vec<DevilFruit>, Error> {
         let sibling_iter = html
-            .select(&Selector::parse(&DfType::Zoan.id_for_fruit_list()).unwrap())
+            .select(&Utils::parse_selector(&DfType::Zoan.id_for_fruit_list())?)
             .next()
             .and_then(|e| e.parent())
             .map(|n| n.next_siblings())
@@ -51,10 +51,7 @@ impl DfTypeParser for CanonZoanParser {
                     && el
                         .first_child()
                         .and_then(|n| ElementRef::wrap(n))
-                        .ok_or_else(|| false)
-                        .unwrap()
-                        .value()
-                        .id()
+                        .and_then(|el| el.value().id())
                         .is_some_and(|s| s != "Canon"))
             })
             .filter(|el| el.value().name() == "ul")
@@ -88,7 +85,7 @@ macro_rules! impl_canon_paramecia_logia_parser {
         impl DfTypeParser for $T {
             fn parse(&self, html: &Html) -> Result<Vec<DevilFruit>, Error> {
                 let fruits: Result<Vec<_>, _> = html
-                    .select(&Selector::parse(&$df_type.id_for_fruit_list()).unwrap())
+                    .select(&Utils::parse_selector(&$df_type.id_for_fruit_list())?)
                     .next()
                     .and_then(|e| e.parent())
                     .map(|n| n.next_siblings())
@@ -235,8 +232,7 @@ impl Utils {
         let mut sub_type_map = HashMap::new();
 
         for df_sub in DfSubType::iter() {
-            let sub_type_selector = &Selector::parse(&df_sub.id_for_fruit_list())
-                .map_err(|_| Error::InvalidStructure("Failed to parse selector".to_string()))?;
+            let sub_type_selector = &Utils::parse_selector(&df_sub.id_for_fruit_list())?;
             let res: Result<(), Error> = html_doc
                 .select(sub_type_selector)
                 .next()
@@ -266,3 +262,16 @@ impl Utils {
         Ok(sub_type_map)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::df::types::DfType;
+
+    use super::get_parser;
+
+    #[test]
+    #[should_panic]
+    fn test_get_parser() {
+        get_parser(&DfType::Undetermined, true);
+    }
+}

scraper/src/df/scraper.rs

@@ -1,7 +1,6 @@
 use itertools::Itertools;
-use lazy_static::lazy_static;
 use log::{error, info};
-use regex::Regex;
+use reqwest::Client;
 use scraper::selectable::Selectable;
 use scraper::Html;
 use std::collections::HashMap;
@@ -11,34 +10,35 @@ use strum::IntoEnumIterator;
 use super::models::{DevilFruit, DfTypeInfo};
 use crate::df::parser::{get_parser, Utils};
 use crate::df::types::DfType;
-use crate::fetcher::HtmlFetcher;
+use crate::fetcher::{FetchHtml, HtmlFetcher};
 use crate::types::{Error, UrlTyped};
 
 pub trait DfScrapable {
     async fn get_dftype_info(&self) -> Result<Vec<DfTypeInfo>, Error>;
     async fn get_df_list(&self) -> Result<Vec<DevilFruit>, Error>;
 }
 
-lazy_static! {
-    static ref REX_EN_NAME: Regex = Regex::new(r"English version: (.+)").unwrap();
-}
-
 #[derive(Debug)]
-pub struct DfScraper {
-    fetcher: HtmlFetcher,
+pub struct DfScraper<T = Client>
+where
+    T: FetchHtml + Clone,
+{
+    fetcher: HtmlFetcher<T>,
     base_url: String,
 }
 
-impl DfScraper {
-    pub fn new(fetcher: HtmlFetcher, base_url: &str) -> Self {
+impl<T: FetchHtml + Clone> DfScraper<T> {
+    pub fn new(fetcher: HtmlFetcher<T>, base_url: &str) -> Self {
         Self {
             fetcher,
             base_url: base_url.to_string(),
         }
     }
 }
 
-impl DfScrapable for DfScraper {
+impl<T: FetchHtml + Clone + std::marker::Send + std::marker::Sync + 'static> DfScrapable
+    for DfScraper<T>
+{
     async fn get_dftype_info(&self) -> Result<Vec<DfTypeInfo>, Error> {
         let url = format!("{}/wiki/Devil_Fruit", self.base_url);
         let html = self.fetcher.fetch(&url).await?;
@@ -60,11 +60,7 @@ impl DfScrapable for DfScraper {
             .map(|row| {
                 let cells = row.select(&td_selector).collect_vec();
                 if cells.len() < 3 {
-                    let msg = format!(
-                        "Expected at least 3 cells, found {}: {:?}",
-                        cells.len(),
-                        row.html()
-                    );
+                    let msg = format!("Expected at least 3 cells, found {}", cells.len());
                     return Err(Error::InvalidStructure(msg));
                 }
 
@@ -86,12 +82,7 @@ impl DfScrapable for DfScraper {
                     DfType::Logia => l_desc.trim(),
                     _ => "",
                 };
-                let obj = DfTypeInfo {
-                    df_type,
-                    cannon_count: cc,
-                    non_cannon_count: ncc,
-                    description: desc.to_string(),
-                };
+                let obj = DfTypeInfo::new(df_type, cc, ncc, desc.to_string());
                 info!("obj: {}", &obj);
                 Ok(obj)
             })
@@ -116,7 +107,6 @@ impl DfScrapable for DfScraper {
                 devil_fruits_map.insert(df_url.clone(), df);
 
                 let fetcher = self.fetcher.clone();
-
                 pic_tasks.spawn(async move {
                     let html = fetcher.fetch_only(&df_url).await?;
                     let doc = Html::parse_document(&html);
@@ -144,3 +134,133 @@ impl DfScrapable for DfScraper {
         Ok(devil_fruits_map.into_values().sorted().collect_vec())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use async_trait::async_trait;
+
+    use crate::{
+        df::scraper::{DfScrapable, DfScraper},
+        fetcher::{FetchHtml, HtmlFetcher},
+        types::Error,
+    };
+
+    #[derive(Clone)]
+    struct MockClient {
+        res_req: HashMap<String, Result<String, Error>>,
+    }
+
+    #[async_trait]
+    impl FetchHtml for MockClient {
+        async fn fetch(&self, url: &str) -> Result<String, Error> {
+            self.res_req.get(url).cloned().unwrap()
+        }
+    }
+
+    fn prepare_fetcher<const N: usize>(
+        arr: [(String, Result<String, Error>); N],
+    ) -> HtmlFetcher<MockClient> {
+        let client = MockClient {
+            res_req: HashMap::from(arr),
+        };
+        HtmlFetcher::new(client)
+    }
+
+    #[tokio::test]
+    async fn get_type_info() {
+        let fetcher = prepare_fetcher([(
+            "/wiki/Devil_Fruit".to_string(),
+            Ok(r#"<html><body>
+                <h4><span id="Paramecia">Paramecia</span></h4>
+                <p>Paramecia Text</p>
+                <h4><span id="Zoan">Zoan</span></h4>
+                <p>Zoan Text</p>
+                <h4><span id="Logia">Logia</span></h4>
+                <p>Logia Text</p>
+                <table class="wikitable">
+                    <tbody>
+                    <tr><th></th><th>Canon</th><th>Non-Canon</th><th>Total</th></tr>
+                    <tr><td>Paramecia</td><td>94 </td><td>48</td><td>142</td></tr>
+                    <tr><td>Zoan</td><td>55</td><td>7 </td><td> 62</td></tr>
+                    <tr><td>Logia</td><td>13</td><td>3</td><td>16  </td></tr>
+                    <tr><td>Undetermined</td><td>3</td><td>2</td><td>5</td></tr>
+                    <tr><td>Last</td><td></td><td></td><td></td></tr>
+                    </tbody></table>
+            </body></html>"#
+                .to_string()),
+        )]);
+        let scrape = DfScraper::new(fetcher, "");
+        let result = scrape.get_dftype_info().await;
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap().len(), 4);
+    }
+
+    #[tokio::test]
+    async fn get_df_list() {
+        let fetcher = prepare_fetcher([
+            (
+                "/wiki/Logia".to_string(),
+                Ok(r#"<html><body>
+                <h4><span id="Logia-Types">Logia</span></h4>
+                <dl></dl>
+                <h3>Some Text</h3>
+                <dl></dl>
+                <ul>
+                    <li><a href="/dfpath_logia" title="Some Logia Df">Some Logia Devil Fruit</a> (<i>English versions: Some En Logia Df</i>): Some description. Eaten by <a href="/character">Df User</a>.</li>
+                </ul>
+                </body></html>"#
+                    .to_string()),
+            ),
+            (
+                "/wiki/Paramecia".to_string(),
+                Ok(r#"<html><body>
+                <h4><span id="Paramecia-Type_Fruits">Paramecia</span></h4>
+                <dl></dl>
+                <h3>Some Text</h3>
+                <dl></dl>
+                <ul>
+                    <li><a href="/dfpath_paramecia" title="Some Paramecia Df">Some Paramecia Devil Fruit</a> (<i>English versions: Some En Paramecia Df</i>): Some description. Eaten by <a href="/character">Df User</a>.</li>
+                </ul>
+                </body></html>"#
+                    .to_string()),
+            ),
+            (
+                "/wiki/Zoan".to_string(),
+                Ok(r#"<html><body>
+                <h4><span id="Ancient_Zoan">Ancient Zoan</span></h4>
+                <dl></dl>
+                <p>wow</p>
+                <ul>
+                    <li><a href="/dfpath-ancient-zoan" title="Some Ancient Zoan">Some ancient zoan</a>: Eaten by <a href="/characterx" title="X">X</a>.</li>
+                </ul>
+                <h4><span id="Mythical_Zoan">Mythical Zoan</span></h4>
+                <dl></dl>
+                <p>wow</p>
+                <ul>
+                    <li><a href="/dfpath-mythical-zoan" title="Some Mythical Zoan">Some mythical zoan</a>: Eaten by <a href="/characterx" title="X">X</a>.</li>
+                </ul>
+                <h4><span id="List_of_Zoan-Type_Fruits">Zoan</span></h4>
+                <h3>Some Text</h3>
+                <dl></dl>
+                <ul>
+                    <li><a href="/dfpath-zoan" title="Some Zoan Df">Some Zoan Devil Fruit</a> (<i>English versions: Some En Zoan Df</i>): Some description. Eaten by <a href="/character">Df User</a>.</li>
+                    <li><a href="/dfpath-ancient-zoan" title="Some Ancient Zoan">Some ancient zoan</a> (<i>English versions: Some En Ancient Zoan Df</i>): Some description. Eaten by <a href="/characterx">X</a>.</li>
+                </ul>
+                </body></html>"#
+                    .to_string()),
+            ),
+            (
+                "/dfpath-zoan".to_string(),
+                Ok(r#"<html><body>
+                <aside><figure class="pi-image piapia"><a href="/picurl" class="image"><img src="/picurlsrc"/></a></figure></aside>
+                </body></html>"#
+                    .to_string()),
+            ),
+        ]);
+        let scrape = DfScraper::new(fetcher, "");
+        let df_list = scrape.get_df_list().await.unwrap();
+        assert_eq!(df_list.len(), 4);
+    }
+}

scraper/src/fetcher.rs

@@ -27,7 +27,7 @@ impl FetchHtml for Client {
 }
 
 #[derive(Debug, Clone)]
-pub struct HtmlFetcher<T = Client>
+pub struct HtmlFetcher<T>
 where
     T: FetchHtml,
 {