feat: add vega.meta.toml file + remove locale subdir from downloaded data

Author: coko7

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "vegapull"
-version = "1.0.0"
+version = "1.1.0"
 description = "Command line tool for scraping data for the One Piece Trading Card Game"
 repository = "https://github.com/Coko7/vegapull"
 authors = ["Coko <91132775+Coko7@users.noreply.github.com>"]

Cargo.toml

@@ -1,6 +1,7 @@
 use anyhow::Result;
 use clap::{command, Parser, Subcommand, ValueEnum};
 use inquire_derive::Selectable;
+use serde::{Deserialize, Serialize};
 use std::{
     ffi::OsString,
     fmt::{self},
@@ -79,7 +80,7 @@ pub enum Commands {
     Config,
 }
 
-#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq, Selectable)]
+#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq, Selectable, Serialize, Deserialize)]
 pub enum LanguageCode {
     #[value(name = "english", alias = "en")]
     English,

src/cli.rs

@@ -2,11 +2,21 @@ use anyhow::{bail, Result};
 use inquire::{Confirm, Text};
 use log::{debug, info};
 use rayon::prelude::*;
-use std::{collections::HashMap, fs, path::PathBuf, time::Instant};
+use std::{
+    collections::{HashMap, HashSet},
+    fs,
+    path::PathBuf,
+    time::SystemTime,
+};
 use yansi::Paint;
 
 use crate::{
-    card::Card, cli::LanguageCode, localizer::Localizer, scraper::OpTcgScraper, storage::DataStore,
+    card::Card,
+    cli::LanguageCode,
+    localizer::Localizer,
+    pack::PackId,
+    scraper::OpTcgScraper,
+    storage::{DataStore, PullMode, VegaMetaStats},
     utils,
 };
 
@@ -110,14 +120,14 @@ fn pull_all_interactive(config_path: Option<PathBuf>, user_agent: Option<String>
 
     eprintln!("Fetching list of packs...");
 
-    let start = Instant::now();
+    let start = SystemTime::now();
 
     let packs = scraper.fetch_packs()?;
     store.write_packs(&packs)?;
 
     eprintln!("Found {} packs!\n", packs.len());
 
-    let pack_ids = packs.iter().map(|p| p.id.as_str()).collect::<Vec<_>>();
+    let pack_ids: HashSet<PackId> = packs.keys().cloned().collect();
 
     eprintln!("Now fetching all the cards for each pack...");
     let all_cards = scraper.fetch_all_cards(&pack_ids, true)?;
@@ -153,30 +163,22 @@ fn pull_all_interactive(config_path: Option<PathBuf>, user_agent: Option<String>
         });
     }
 
-    let duration = start.elapsed();
+    let duration = start.elapsed()?;
 
     eprintln!(
         "\nFinal data is available in: {}",
         inputs.data_dir.display()
     );
-    eprintln!("Full download completed after: {:?}", duration);
+    eprintln!("Full download took: {:?}", duration);
+
+    store.write_vega_stats(VegaMetaStats::new(
+        inputs.language,
+        start.into(),
+        duration.as_millis().try_into()?,
+        inputs.download_images,
+        PullMode::All,
+        pack_ids,
+    ))?;
+
     Ok(())
 }
-
-// fn download_images(scraper: &OpTcgScraper, cards: &HashMap<String, Card>) -> Result<()> {
-//     let card_values = cards.values().collect::<Vec<_>>();
-//     let images = scraper.fetch_all_card_images(&cards)?;
-//
-//     images.par_iter().for_each(|(card_id, image_data)| {
-//         let card = cards_by_id
-//             .get(card_id)
-//             .unwrap_or_else(|| panic!("card should exist: {card_id}"));
-//
-//         store
-//             .write_image(card, image_data.to_vec())
-//             .unwrap_or_else(|_| panic!("write_image failed for: {card_id}"));
-//         debug!("wrote image_data for: {}", card_id);
-//     });
-//
-//     Ok(())
-// }

src/commands/pull_all.rs

@@ -1,10 +1,18 @@
 use anyhow::{bail, Result};
 use log::{debug, error, info};
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
-use std::{collections::HashMap, path::Path, time::Instant};
+use std::{
+    collections::{HashMap, HashSet},
+    path::Path,
+    time::SystemTime,
+};
 
 use crate::{
-    card::Card, cli::LanguageCode, localizer::Localizer, scraper::OpTcgScraper, storage::DataStore,
+    card::Card,
+    cli::LanguageCode,
+    localizer::Localizer,
+    scraper::OpTcgScraper,
+    storage::{DataStore, PullMode, VegaMetaStats},
     utils,
 };
 
@@ -23,7 +31,7 @@ pub fn pull_cards(
     let store = DataStore::new(output_dir, language);
 
     eprintln!("fetching all cards for pack {pack_id}...");
-    let start = Instant::now();
+    let start = SystemTime::now();
 
     let cards = scraper.fetch_cards(pack_id)?;
     if cards.is_empty() {
@@ -58,79 +66,24 @@ pub fn pull_cards(
         });
     }
 
-    let duration = start.elapsed();
+    println!(
+        "downloaded cards for pack {} to: {}",
+        pack_id,
+        output_dir.display()
+    );
+
+    let duration = start.elapsed()?;
 
     info!("list_cards took: {:?}", duration);
+
+    store.write_vega_stats(VegaMetaStats::new(
+        language,
+        start.into(),
+        duration.as_millis().try_into()?,
+        with_images,
+        PullMode::SinglePack,
+        HashSet::from([pack_id.to_owned()]),
+    ))?;
+
     Ok(())
 }
-
-// pub fn download_images_fast(
-//     language: LanguageCode,
-//     cards: Vec<Card>,
-//     output_dir: &Path,
-//     user_agent: Option<String>,
-// ) -> Result<()> {
-//     let localizer = Localizer::load(language)?;
-//     let scraper = OpTcgScraper::new(localizer, user_agent);
-//
-//     if output_dir.exists() {
-//         error!("output directory already `{}` exists", output_dir.display());
-//         bail!(
-//             "cannot create directory `{}` to store images because it already exists",
-//             output_dir.display()
-//         );
-//     }
-//
-//     match fs::create_dir_all(output_dir) {
-//         Ok(_) => info!("successfully created `{}`", output_dir.display()),
-//         Err(e) => bail!("failed to create `{}`: {}", output_dir.display(), e),
-//     }
-//
-//     info!("downloading images...");
-//     let start = Instant::now();
-//
-//     let mut handles = vec![];
-//
-//     let completed_count = Arc::new(AtomicUsize::new(0));
-//     let all_cards = cards.len();
-//
-//     let scraper = Arc::new(scraper);
-//     let output_dir = output_dir.to_path_buf();
-//
-//     for card in cards.into_iter() {
-//         let scraper = Arc::clone(&scraper);
-//         let output_dir = output_dir.clone();
-//         let completed_count = Arc::clone(&completed_count);
-//
-//         let handle = thread::spawn(move || {
-//             let img_url = card.img_url.clone();
-//             let img_path = download_card_image(&output_dir, &scraper, card).unwrap();
-//             let current = completed_count.fetch_add(1, Ordering::SeqCst) + 1;
-//
-//             eprintln!(
-//                 "[{}/{}] succesfully saved image `{}` to `{}`",
-//                 current,
-//                 all_cards,
-//                 img_url,
-//                 img_path.display()
-//             );
-//
-//             debug!(
-//                 "[{}/{}] saved image `{}` to `{}`",
-//                 current,
-//                 all_cards,
-//                 img_url,
-//                 img_path.display()
-//             );
-//         });
-//         handles.push(handle);
-//     }
-//
-//     for handle in handles {
-//         handle.join().unwrap();
-//     }
-//
-//     let duration = start.elapsed();
-//     info!("downloading images took: {:?}", duration);
-//     Ok(())
-// }

src/commands/pull_cards.rs

@@ -1,9 +1,14 @@
 use anyhow::Result;
 use log::debug;
-use std::{path::Path, time::Instant};
+use std::{collections::HashSet, path::Path, time::SystemTime};
 
 use crate::{
-    cli::LanguageCode, localizer::Localizer, scraper::OpTcgScraper, storage::DataStore, utils,
+    cli::LanguageCode,
+    localizer::Localizer,
+    pack::PackId,
+    scraper::OpTcgScraper,
+    storage::{DataStore, PullMode, VegaMetaStats},
+    utils,
 };
 
 pub fn pull_packs(
@@ -19,9 +24,10 @@ pub fn pull_packs(
     let store = DataStore::new(output_dir, language);
 
     eprintln!("fetching list of packs...");
-    let start = Instant::now();
+    let start = SystemTime::now();
 
     let packs = scraper.fetch_packs()?;
+    let pack_ids: HashSet<PackId> = packs.keys().cloned().collect();
     store.write_packs(&packs)?;
 
     println!(
@@ -30,8 +36,18 @@ pub fn pull_packs(
         output_dir.display()
     );
 
-    let duration = start.elapsed();
+    let duration = start.elapsed()?;
 
     debug!("pull_packs took: {:?}", duration);
+
+    store.write_vega_stats(VegaMetaStats::new(
+        language,
+        start.into(),
+        duration.as_millis().try_into()?,
+        false,
+        PullMode::PackListOnly,
+        pack_ids,
+    ))?;
+
     Ok(())
 }

src/commands/pull_packs.rs

@@ -7,6 +7,8 @@ use std::{
     hash::{Hash, Hasher},
 };
 
+pub type PackId = String;
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct Pack {
     pub id: String,

src/pack.rs

@@ -3,15 +3,15 @@ use log::{debug, info};
 use rayon::prelude::*;
 use scraper::Html;
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     thread,
     time::{Duration, Instant},
 };
 
 use crate::{
     card::{Card, CardScraper},
     localizer::Localizer,
-    pack::Pack,
+    pack::{Pack, PackId},
 };
 
 pub struct OpTcgScraper {
@@ -53,7 +53,7 @@ impl OpTcgScraper {
         full_url
     }
 
-    pub fn fetch_packs(&self) -> Result<Vec<Pack>> {
+    pub fn fetch_packs(&self) -> Result<HashMap<PackId, Pack>> {
         let url = self.cardlist_endpoint();
         debug!("GET `{}`", url);
 
@@ -74,12 +74,12 @@ impl OpTcgScraper {
 
         let series_selector = scraper::Selector::parse(sel).unwrap();
 
-        let mut packs = Vec::new();
+        let mut packs = HashMap::new();
         for element in document.select(&series_selector) {
             match Pack::new(element) {
                 Ok(pack) => {
                     if !pack.id.is_empty() {
-                        packs.push(pack);
+                        packs.insert(pack.id.clone(), pack);
                     }
                 }
                 Err(e) => bail!("failed to scrape data about packs: {}", e),
@@ -94,12 +94,12 @@ impl OpTcgScraper {
 
     pub fn fetch_all_cards(
         &self,
-        pack_ids: &[&str],
+        pack_ids: &HashSet<PackId>,
         report_progress: bool,
     ) -> Result<HashMap<String, Vec<Card>>> {
         pack_ids
             .par_iter()
-            .map(|&pid| {
+            .map(|pid| {
                 info!("fetching all cards for pack {} via rayon", pid);
                 let pack_id = pid.to_string();
                 self.fetch_cards(&pack_id).map(|cards| {