Are subdomains working as expected?

[emgardner]

I am using spider for a personal project and am not sure if i am using it wrong or if it's not working as expected (version 1.80.82)

use spider::tokio;
use spider::website::Website;
use spider::CaseInsensitiveString;
use env_logger::Env;


#[tokio::main]
async fn main() {
    // let env = Env::default()
    //     .filter_or("RUST_LOG", "info")
    //     .write_style_or("RUST_LOG_STYLE", "always");

    // env_logger::init_from_env(env);
    // https://www.adamhepner.pl
    let mut website = Website::new("https://adamhepner.pl")
        .with_depth(1000)
        .with_subdomains(true)
        .with_tld(true)
        .with_full_resources(true)
        .build()
        .unwrap();
    println!("{:#?}", website);
    website.crawl().await;
    let links = website.get_links();
    for link in links {
        println!("- {:?}", link.as_ref());
    }
}

There's a subdomain link with blog.[domain] on this page that is not picked up. Here's another site with the same behavior https://www.kylegalbraith.com/. My assumption was that enabling subdomain's would pick up those links. Using a subdomain directly Website::new("https://blog.adamhepner.pl") yields my expected behavior.

pub fn parent_host_match(
    host_name: Option<&str>,
    base_domain: &str,
    parent_host: &CompactString,
) -> bool {
    match host_name {
        Some(host) => {
            if base_domain.is_empty() {
                parent_host.eq(&host)
            } else {
                parent_host.ends_with(host)
            }
        }
        _ => false,
    }
}

It appears to stem from here. I would assume you would pass a subdomains option here and match the hostname's if it is enabled?

[j-mendez]

It seems to work as expected, make sure the link is rendered statically if you are not using the chrome feature

[emgardner]

Both of the domains I gave as an example are rendered statically i am not using the chrome feature. Any other ideas as to why it wouldn't be captured. I've added some debug log's and the href is being found but it's being marked as false for the can_process = parent_host_match it is then not being processed because the blog subdomain is being shown as false due to this statement !self.external_domains_caseless.is_empty() in:

    #[inline(always)]
    pub async fn links_stream_full_resource<A: PartialEq + Eq + std::hash::Hash + From<String>>(
        &self,
        selectors: &(&CompactString, &SmallVec<[CompactString; 2]>),
    ) -> HashSet<A> {
        let html = Box::new(crate::packages::scraper::Html::parse_document(
            &self.get_html(),
        ));
        tokio::task::yield_now().await;

        let mut stream = tokio_stream::iter(html.tree);
        let mut map = HashSet::new();

        let base_domain = &selectors.0;
        let parent_frags = &selectors.1; // todo: allow mix match tpt
        let parent_host = &parent_frags[0];
        let parent_host_scheme = &parent_frags[1];

        while let Some(node) = stream.next().await {
            if let Some(element) = node.as_element() {
                let element_name = element.name();

                let ele_attribute = if element_name == "a" || element_name == "link" {
                    "href"
                } else if element_name == "script" {
                    "src"
                } else {
                    "href"
                };

                match element.attr(ele_attribute) {
                    Some(href) => {
                        let mut abs = self.abs_path(href);
                        let host_name = abs.host_str();
                        let mut can_process =
                            parent_host_match(host_name, base_domain, parent_host);

                        let mut external_domain = false;

                        if !can_process
                            && host_name.is_some()
                            && !self.external_domains_caseless.is_empty()
                        {

                                .external_domains_caseless
                                .contains::<CaseInsensitiveString>(
                                    &host_name.unwrap_or_default().into(),
                                )
                                || self
                                    .external_domains_caseless
                                    .contains::<CaseInsensitiveString>(&CASELESS_WILD_CARD));
                            can_process = self
                                .external_domains_caseless
                                .contains::<CaseInsensitiveString>(
                                    &host_name.unwrap_or_default().into(),
                                )
                                || self
                                    .external_domains_caseless
                                    .contains::<CaseInsensitiveString>(&CASELESS_WILD_CARD);
                            external_domain = can_process;
                        }

                        if can_process {
                            if abs.scheme() != parent_host_scheme.as_str() {
                                let _ = abs.set_scheme(parent_host_scheme.as_str());
                            }

                            let h = abs.as_str();


                            if can_process
                                && (base_domain.is_empty()
                                    || external_domain
                                    || base_domain.as_str() == domain_name(&abs))
                            {
                                map.insert(h.to_string().into());
                            }
                        }
                    }
                    _ => (),
                };
            }
        }

        map
    }

[j-mendez]

Hi @emgardner wow, thanks for the reply. It looks like there was some incorrect logic for the subdomain handling beeb3be#diff-81cfc158c4f0c90631181326c6987dcb6fd074ecfed788d6593facadf3494905L164. This is now fixed in v1.80.84.

[emgardner]

Awesome thanks a lot glad you were able to find the issue!