finished parsing, fetch functions

Author: zphrs

src/fetch/mod.rs

@@ -0,0 +1,57 @@
+use reqwest::{Client, Error as RequestError};
+
+use crate::parse::LocationMeta;
+
+pub async fn fetch_locations_page(client: &reqwest::Client) -> Result<String, RequestError> {
+    static URL: &str = "https://nutrition.sa.ucsc.edu/";
+    let response = client.get(URL).send().await?;
+    response.text().await
+}
+
+pub fn make_client() -> reqwest::Client {
+    Client::builder()
+        .danger_accept_invalid_certs(true)
+        .build()
+        .expect("error building client")
+}
+
+pub async fn fetch_location_page(
+    client: &reqwest::Client,
+    location_meta: LocationMeta,
+) -> Result<String, RequestError> {
+    static COOKIES: &str = "WebInaCartDates=;  WebInaCartMeals=; WebInaCartQtys=; WebInaCartRecipes=; WebInaCartLocation=";
+    let id = location_meta.id();
+    let cookies = format!("{COOKIES}{id}");
+    let url = location_meta.url();
+    client
+        .get(url.clone())
+        .header("Cookie", cookies)
+        .send()
+        .await?
+        .text()
+        .await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use url::Url;
+
+    #[tokio::test]
+    async fn test_fetch_locations_page() {
+        let client = make_client();
+        let _page = fetch_locations_page(&client).await.unwrap();
+    }
+
+    async fn test_fetch_location_page() {
+        let client = make_client();
+        let url: Url = "https://nutrition.sa.ucsc.edu/shortmenu.aspx?\
+        sName=UC+Santa+Cruz+Dining&\
+        locationNum=40&\
+        locationName=College+Nine/John+R.+Lewis+Dining+Hall&naFlag=1"
+            .parse()
+            .expect("url should be valid");
+        let location_meta = LocationMeta::from_url(url).expect("location meta should be valid");
+        fetch_location_page(&client, location_meta).await.unwrap();
+    }
+}

src/main.rs

@@ -1,7 +1,7 @@
 #![allow(dead_code)] // TODO: remove this line once done with crate
 
+mod fetch;
 mod parse;
-mod scrape_menus;
 
 use std::{convert::Infallible, env, error::Error, net::SocketAddr, sync::Arc};
 

src/parse/daily_menu/allergens.rs

@@ -97,7 +97,7 @@ mod tests {
 
     use scraper::Selector;
 
-    use crate::get_or_init_selector;
+    use crate::static_selector;
 
     use super::*;
     const HTML: &str = r#"
@@ -171,9 +171,9 @@ mod tests {
 
         let doc = scraper::Html::parse_document(HTML);
         static SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let selector = get_or_init_selector!(SELECTOR, "img");
+        static_selector!(DATE_SELECTOR <- "img");
         let mut all_allergen_flags = AllergenFlags::empty();
-        for element in doc.select(&selector) {
+        for element in doc.select(&DATE_SELECTOR) {
             let img_url = element.value().attr("src").unwrap(); // all img elements should have a src attribute
             let allergen_flags = AllergenInfo::img_url_to_allergen(img_url)
                 .expect("All img urls in this example should be valid");

src/parse/daily_menu/daily_menu.rs

@@ -2,16 +2,54 @@ use chrono::NaiveDate;
 
 use super::meal::Meal;
 use crate::parse::Error;
+use crate::static_selector;
+
 #[derive(Debug)]
 pub struct DailyMenu<'a> {
     date: NaiveDate,
     meals: Vec<Meal<'a>>,
 }
 
+impl PartialEq for DailyMenu<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        self.date == other.date
+    }
+}
+
+impl PartialOrd for DailyMenu<'_> {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        self.date.partial_cmp(&other.date)
+    }
+}
+
+impl Eq for DailyMenu<'_> {}
+
+impl Ord for DailyMenu<'_> {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.date.cmp(&other.date)
+    }
+}
+
 impl<'a> DailyMenu<'a> {
     pub fn from_html_element(element: scraper::ElementRef<'a>) -> Result<Self, Error> {
-        // TODO
-        todo!("Implement the MealsOnDate::from_html_element function")
+        static_selector!(DATE_SELECTOR <- "input[name=strCurSearchDays]");
+        static_selector!(MEAL_SELECTOR <- r##"table[bordercolors="#FFFF00"]"##);
+        let date_str = element
+            .select(&DATE_SELECTOR)
+            .next()
+            .ok_or_else(|| Error::html_parse_error("Date field not found"))?
+            .attr("value")
+            .ok_or_else(|| Error::html_parse_error("No value on date field"))?;
+
+        let date = NaiveDate::parse_from_str(date_str, "%m/%d/%Y")
+            .map_err(|_x| Error::html_parse_error("Date is not in valid format."))?;
+
+        let meals = element
+            .select(&MEAL_SELECTOR)
+            .map(Meal::from_html_element)
+            .collect::<Result<_, Error>>()?;
+
+        Ok(Self { date, meals })
     }
 }
 
@@ -25,7 +63,7 @@ mod tests {
         let html =
             fs::read_to_string("./src/parse/html_examples/daily_menu/meals_on_date.html").unwrap();
         let document = scraper::Html::parse_document(&html);
-        let meals = DailyMenu::from_html_element(&document.root_element())
+        let meals = DailyMenu::from_html_element(document.root_element())
             .expect("The example html should be valid");
         assert_eq!(
             meals.date,

src/parse/daily_menu/food_item.rs

@@ -1,10 +1,8 @@
 use super::allergens::AllergenInfo;
-use crate::get_or_init_selector;
-use crate::parse::Error;
 use crate::parse::text_from_selection::{get_inner_text, text_from_selection};
+use crate::parse::Error;
+use crate::static_selector;
 use rusty_money::{iso, Money};
-use scraper::Selector;
-use std::sync::OnceLock;
 
 #[derive(Debug)]
 pub struct FoodItem<'a> {
@@ -27,19 +25,16 @@ impl<'a> FoodItem<'a> {
         // example html tr element at ./html_examples/food_item.html
 
         // get name with css selector .shortmenurecipes > span
-        static NAME_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let name_selector = get_or_init_selector!(NAME_SELECTOR, ".shortmenurecipes > span");
-        let name = text_from_selection(name_selector, element, "foodItem", "name")?.trim_end();
+        static_selector!(NAME_SELECTOR <- ".shortmenurecipes > span");
+        let name = text_from_selection(&NAME_SELECTOR, element, "foodItem", "name")?.trim_end();
         // get allergen info with css selector td > img
-        static ALLERGEN_INFO_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let allergen_info_selector = get_or_init_selector!(ALLERGEN_INFO_SELECTOR, "td > img");
+        static_selector!(ALLERGEN_INFO_SELECTOR <- "td > img");
         let allergen_info =
-            AllergenInfo::from_html_elements(element.select(&allergen_info_selector))?;
+            AllergenInfo::from_html_elements(element.select(&ALLERGEN_INFO_SELECTOR))?;
 
         // try to get price with css selector .shortmenuprices > span
-        static PRICE_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let price_selector = get_or_init_selector!(PRICE_SELECTOR, ".shortmenuprices > span");
-        let price_element = element.select(&price_selector).next();
+        static_selector!(PRICE_SELECTOR <- ".shortmenuprices > span");
+        let price_element = element.select(&PRICE_SELECTOR).next();
         let price = if let Some(price_element) = price_element {
             let price: &str = get_inner_text(price_element, "price")?; // will look like "$5.00"
                                                                        // if price is equal to &nbsp; then return None
@@ -70,7 +65,8 @@ mod tests {
     fn test_food_item_from_html_element() {
         // source: https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=40&locationName=Colleges+Nine+%26+Ten&sName=&naFlag=
         // load the html file
-        let html = std::fs::read_to_string("./src/parse/html_examples/daily_menu/food_item.html").unwrap(); // file system should be reliable
+        let html =
+            std::fs::read_to_string("./src/parse/html_examples/daily_menu/food_item.html").unwrap(); // file system should be reliable
         let doc = scraper::Html::parse_document(&html);
         let food_item = FoodItem::from_html_element(doc.root_element())
             .expect("The example html should be valid");

src/parse/daily_menu/get_or_init_selector.rs

@@ -1,14 +1,12 @@
 use std::{iter::Peekable, vec};
 
-use scraper::{element_ref::Select, selectable::Selectable, Selector};
+use scraper::{element_ref::Select, selectable::Selectable};
 
-use crate::{get_or_init_selector, parse::text_from_selection::text_from_selection};
+use crate::{parse::text_from_selection::text_from_selection, static_selector};
 
 use super::food_item::FoodItem;
 use crate::parse::Error;
 
-use std::sync::OnceLock;
-
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum MealType {
     Breakfast,
@@ -28,22 +26,17 @@ pub struct Meal<'a> {
 impl<'a> Meal<'a> {
     pub fn from_html_element(element: scraper::ElementRef<'a>) -> Result<Self, Error> {
         // example html div element at ./html_examples/meal.html
-        static ROW_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let row_selector = get_or_init_selector!(
-            ROW_SELECTOR,
-            r##"table[bordercolor="#FFFF00"] > tbody > tr"##
-        );
-        let mut top_level_row_iter = element.select(row_selector);
+        static_selector!(ROW_SELECTOR <- r##"table[bordercolor="#FFFF00"] > tbody > tr"##);
+        let mut top_level_row_iter = element.select(&ROW_SELECTOR);
         let meal_name_row = top_level_row_iter.next().ok_or_else(|| {
             Error::html_parse_error("The meal should have a row for the meal type.")
         })?;
         let meal_item_row = top_level_row_iter.next().ok_or_else(|| {
             Error::html_parse_error("The meal should have a row for the meal items.")
         })?;
-        static MEAL_TYPE_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let meal_type_selector = get_or_init_selector!(MEAL_TYPE_SELECTOR, ".shortmenumeals");
+        static_selector!(MEAL_TYPE_SELECTOR <- ".shortmenumeals");
         let meal_type =
-            text_from_selection(&meal_type_selector, meal_name_row, "meal", "meal type")?;
+            text_from_selection(&MEAL_TYPE_SELECTOR, meal_name_row, "meal", "meal type")?;
         // print out meal type
         let meal_type = match meal_type {
             "Breakfast" => MealType::Breakfast,
@@ -54,9 +47,8 @@ impl<'a> Meal<'a> {
             _ => MealType::AllDay,
         };
 
-        static SECTION_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let section_selector = get_or_init_selector!(SECTION_SELECTOR, "table > tbody > tr");
-        let section_elements = meal_item_row.select(&section_selector);
+        static_selector!(SECTION_NAME_SELECTOR <- "table > tbody > tr");
+        let section_elements = meal_item_row.select(&SECTION_NAME_SELECTOR);
         let sections = SectionIterator::new(section_elements.peekable(), meal_type);
         let mut sections_vec: Vec<MealSection> = vec![];
         for section in sections {
@@ -115,23 +107,21 @@ pub struct MealSection<'a> {
 impl<'a> MealSection<'a> {
     // takes in an iterator of tr elements of a specific meal and consumes the elements to create a MealSection
     pub fn from_html_elements(elements: &mut Peekable<Select<'a, 'a>>) -> Result<Self, Error> {
-        static MEAL_NAME_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let section_name_selector =
-            get_or_init_selector!(MEAL_NAME_SELECTOR, ".shortmenucats > span");
+        static_selector!(SECTION_NAME_SELECTOR <- ".shortmenucats > span");
 
         // if the first element does not match the section name selector, then return an error
         let first_element = elements.next().ok_or_else(|| {
             Error::html_parse_error("Every section should have a name as the first element.")
         })?;
-        let name = text_from_selection(section_name_selector, first_element, "section", "name")?;
+        let name = text_from_selection(&SECTION_NAME_SELECTOR, first_element, "section", "name")?;
 
         // trim off first and last three characters since the name looks like -- name --
         let name = &name[3..name.len() - 3];
 
         // iterate through by peeking and calling handle_element
         let mut food_items = vec![];
         while let Some(element) = elements.peek() {
-            if element.select(&section_name_selector).next().is_some() {
+            if element.select(&SECTION_NAME_SELECTOR).next().is_some() {
                 break;
             }
             if let Some(food_item) = Self::handle_element(*element)? {
@@ -143,11 +133,9 @@ impl<'a> MealSection<'a> {
     }
 
     fn handle_element(element: scraper::ElementRef<'a>) -> Result<Option<FoodItem<'a>>, Error> {
-        static SECTION_NAME_SELECTOR: OnceLock<Selector> = OnceLock::new();
-        let section_name_selector =
-            get_or_init_selector!(SECTION_NAME_SELECTOR, ".shortmenucats > span");
-        if element.select(&section_name_selector).next().is_some() {
-            return Ok(None);
+        static_selector!(SECTION_NAME_SELECTOR <- ".shortmenucats > span");
+        if element.select(&SECTION_NAME_SELECTOR).next().is_some() {
+            Ok(None)
         } else {
             let out = FoodItem::from_html_element(element)?;
             Ok(Some(out))

src/parse/daily_menu/meal.rs

@@ -1,6 +1,5 @@
 mod allergens;
 mod daily_menu;
 mod food_item;
-mod get_or_init_selector;
 mod meal;
 pub use daily_menu::DailyMenu;