feat(ads,tracking): add split ads and tracking list

Author: j-mendez

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "spider_firewall"
-version = "2.28.7"
+version = "2.30.2"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

Cargo.toml

@@ -1,6 +1,6 @@
+use hashbrown::HashSet;
 use reqwest::blocking::Client;
 use serde::Deserialize;
-use hashbrown::HashSet;
 use std::env;
 use std::fs;
 use std::path::PathBuf;
@@ -19,6 +19,9 @@ struct GithubContent {
 fn main() -> std::io::Result<()> {
     let client = Client::new();
     let mut unique_entries = HashSet::new();
+    let mut unique_ads_entries = HashSet::new();
+    let mut unique_tracking_entries = HashSet::new();
+    let mut unique_gambling_entries = HashSet::new();
 
     // Fetch and process GitHub directory files
     let base_url = "https://api.github.com/repos/ShadowWhisperer/BlockLists/contents/RAW";
@@ -30,7 +33,20 @@ fn main() -> std::io::Result<()> {
 
     let contents: Vec<GithubContent> = response.json().expect("Failed to parse JSON response");
 
+    let skip_list = vec![
+        "Cryptocurrency",
+        "Dating",
+        "Fonts",
+        "Microsoft",
+        "Marketing",
+        "Wild_Tracking",
+    ];
+
     for item in contents {
+        // ignore these websites.
+        if skip_list.contains(&item.name.as_str()) {
+            continue;
+        }
         if item.content_type == "file" {
             let file_url = format!(
                 "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/{}",
@@ -42,9 +58,30 @@ fn main() -> std::io::Result<()> {
                 .expect("Failed to fetch file content");
 
             let file_content = file_response.text().expect("Failed to read file content");
-            for line in file_content.lines() {
-                if !line.is_empty() {
-                    unique_entries.insert(line.to_string());
+
+            if item.name == "Wild_Tracking" || item.name == "Tracking" {
+                for line in file_content.lines() {
+                    if !line.is_empty() {
+                        unique_tracking_entries.insert(line.to_string());
+                    }
+                }
+            } else if item.name == "Wild_Ads" || item.name == "Ads" {
+                for line in file_content.lines() {
+                    if !line.is_empty() {
+                        unique_ads_entries.insert(line.to_string());
+                    }
+                }
+            }  else if item.name == "Gambling" {
+                for line in file_content.lines() {
+                    if !line.is_empty() {
+                        unique_gambling_entries.insert(line.to_string());
+                    }
+                }
+            }  else {
+                for line in file_content.lines() {
+                    if !line.is_empty() {
+                        unique_entries.insert(line.to_string());
+                    }
                 }
             }
         }
@@ -68,23 +105,50 @@ fn main() -> std::io::Result<()> {
         }
     }
 
-    // Begin building the phf set from the unique entries
     let mut set = phf_codegen::Set::new();
 
     for entry in unique_entries {
         set.entry(entry);
     }
 
+    let mut ads_set = phf_codegen::Set::new();
+
+    for entry in unique_ads_entries {
+        ads_set.entry(entry);
+    }
+
+    let mut tracking_set = phf_codegen::Set::new();
+
+    for entry in unique_tracking_entries {
+        tracking_set.entry(entry);
+    }
+
+    let mut gambling_set = phf_codegen::Set::new();
+
+    for entry in unique_gambling_entries {
+        gambling_set.entry(entry);
+    }
+
+
     // Write to destination
     let out_dir = env::var("OUT_DIR").unwrap();
     let dest_path = PathBuf::from(out_dir).join("bad_websites.rs");
 
     fs::write(
         &dest_path,
         format!(
-            "/// Bad websites that we should not crawl.\n\
-            static BAD_WEBSITES: phf::Set<&'static str> = {};",
-            set.build()
+            "/// Bad websites that we should not connect to.\n\
+            static BAD_WEBSITES: phf::Set<&'static str> = {};\n
+            /// Ads websites that we should not connect to.\n\
+            static ADS_WEBSITES: phf::Set<&'static str> = {};\n
+            /// Tracking websites that we should not connect to.\n\
+            static TRACKING_WEBSITES: phf::Set<&'static str> = {};\n
+            /// Gambling websites that we should not connect to.\n\
+            static GAMBLING_WEBSITES: phf::Set<&'static str> = {};",
+            set.build(),
+            ads_set.build(),
+            tracking_set.build(),
+            gambling_set.build()
         ),
     )?;
 

build.rs

@@ -1,10 +1,25 @@
 include!(concat!(env!("OUT_DIR"), "/bad_websites.rs"));
 
-/// The url is in the bad website.
+/// The url is in the bad list.
 pub fn is_bad_website_url(host: &str) -> bool {
     BAD_WEBSITES.contains(&host)
 }
 
+/// The url is in the ads list.
+pub fn is_ad_website_url(host: &str) -> bool {
+    ADS_WEBSITES.contains(&host)
+}
+
+/// The url is in the tracking list.
+pub fn is_tracking_website_url(host: &str) -> bool {
+    TRACKING_WEBSITES.contains(&host)
+}
+
+/// The url is in the gambling list.
+pub fn is_gambling_website_url(host: &str) -> bool {
+    GAMBLING_WEBSITES.contains(&host)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -31,4 +46,16 @@ mod tests {
         let bad_website = "10minutesto1.net";
         assert!(is_bad_website_url(bad_website.to_lowercase().as_str()));
     }
+
+    #[test]
+    fn test_is_ad_website_url() {
+        let bad_website = "admob.google.com";
+        assert!(is_ad_website_url(bad_website.as_str()));
+    }
+
+    #[test]
+    fn test_is_tracking_website_url() {
+        let bad_website = "2.atlasroofing.com";
+        assert!(is_tracking_website_url(bad_website.as_str()));
+    }
 }