update scraper (#684)

Author: aster-void

.cspell.json

@@ -21,12 +21,14 @@
     "pkgs",
     "psql",
     "qiita",
+    "replacen",
     "reqwest",
     "rustc",
     "safify",
     "stdenv",
     "supabase",
-    "swiper"
+    "swiper",
+    "zenki"
   ],
   "dictionaries": [
     "softwareTerms",
@@ -52,6 +54,7 @@
     "**/*.svg",
     "**/migration.sql",
     "**/data.json",
+    "**/server/src/seeds/json/**",
     "**/Cargo.*",
     "scraper/target",
     "**/rust-toolchain.toml",

.gitignore

@@ -3,6 +3,9 @@
 /.direnv
 /.husky
 
+.cache
+data.json
+
 # Logs
 logs
 *.log

biome.json

@@ -22,6 +22,7 @@
       "bun.lockb",
       "server/target",
       "data.json",
+      "server/src/seeds/json",
       "scraper/target",
       ".next",
       "next-env.d.ts",

flake.lock

@@ -32,7 +32,7 @@
       };
       unstable = nixpkgs-unstable.legacyPackages.${system};
 
-      rust-bin = pkgs.rust-bin.fromRustupToolchainFile ./scraper/rust-toolchain.toml;
+      rust-bin = pkgs.rust-bin.beta.latest.default; # pkgs.rust-bin.fromRustupToolchainFile ./scraper/rust-toolchain.toml;
       prisma = pkgs.callPackage ./server/prisma.nix {inherit prisma-utils;};
 
       common = {
@@ -62,8 +62,7 @@
       };
     in {
       packages.scraper = pkgs.callPackage ./scraper {toolchain = rust-bin;};
-      devShells.default = pkgs.mkShell common;
-      devShells.scraper = pkgs.mkShell {
+      devShells.default = pkgs.mkShell {
         inherit (common) env;
         packages =
           common.packages

flake.nix

@@ -0,0 +1,14 @@
+[
+  {
+    name: "zenki",
+    courses: [
+      {
+        name: "数理科学基礎",
+        teacher: "(人名)",
+        semester: "S1",
+        period: "月曜2限、水曜1限",
+        code: "30003 CAS-FC1871L1",
+      },
+    ],
+  },
+];

scraper/sample.ts

@@ -1,6 +1,5 @@
 use crate::types::*;
 use anyhow::ensure;
-use sha2::{Digest, Sha256};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
 
@@ -10,13 +9,20 @@ pub async fn write_to(file: &mut fs::File, content: Entry) -> anyhow::Result<()>
     Ok(())
 }
 
-use crate::CACHE_DIR;
+use crate::cache_dir;
 
 pub async fn request(url: &str) -> anyhow::Result<String> {
     println!("[request] sending request to {}", url);
 
-    let hash = Sha256::digest(url.as_bytes());
-    let path = format!("{CACHE_DIR}/{:x}", hash);
+    let cache_key = url
+        .to_string()
+        .replacen("/", "_", 1000)
+        .replacen(":", "_", 1000)
+        .replacen("?", "_", 1000)
+        .replacen("&", "_", 1000)
+        .replacen("=", "_", 1000)
+        .to_string();
+    let path = format!("{}/{cache_key}", cache_dir());
     if let Ok(bytes) = fs::read(&path).await {
         if let Ok(text) = String::from_utf8(bytes) {
             return Ok(text);

scraper/src/io.rs

@@ -16,13 +16,16 @@ use scraper::{Html, Selector};
 use urls::URLS;
 
 const RESULT_FILE: &str = "./data.json";
-const CACHE_DIR: &str = "./.cache";
+
+fn cache_dir() -> String {
+    "./.cache".to_string()
+}
 
 #[tokio::main(flavor = "multi_thread")]
 async fn main() {
     println!("[log] starting...");
 
-    let _ = fs::DirBuilder::new().create(CACHE_DIR).await;
+    let _ = fs::DirBuilder::new().create(cache_dir()).await;
 
     let mut file = fs::File::create(RESULT_FILE)
         .await
@@ -59,7 +62,8 @@ async fn get_courses_of(base_url: &str) -> Vec<Course> {
     futures::future::join_all(courses)
         .await
         .into_iter()
-        .collect::<Vec<_>>()
+        .flatten()
+        .collect()
 }
 
 lazy_static! {

scraper/src/main.rs

@@ -1,6 +1,6 @@
 use anyhow::anyhow;
 use lazy_static::lazy_static;
-use scraper::{Html, Selector};
+use scraper::{ElementRef, Html, Selector};
 
 use crate::types::*;
 
@@ -17,19 +17,24 @@ lazy_static! {
         Selector::parse(".catalog-page-detail-table-cell.code-cell").unwrap();
 }
 
-pub fn parse_course_info(html: Html) -> anyhow::Result<Course> {
-    Ok(Course {
-        name: select(&html, &NAME_SELECTOR, 1)?,
-        teacher: select(&html, &TEACHER_SELECTOR, 1)?,
-        semester: select_all(&html, &SEMESTER_SELECTOR, 1)?.join(","),
-        period: select(&html, &PERIOD_SELECTOR, 1)?,
-        code: select_all(&html, &CODE_SELECTOR, 1)?.join(" "),
-    })
+pub fn parse_course_info(html: Html) -> anyhow::Result<Vec<Course>> {
+    html.select(&Selector::parse(".catalog-page-detail-table-row").unwrap())
+        .skip(1)
+        .map(|el| {
+            Ok(Course {
+                name: select(&el, &NAME_SELECTOR)?,
+                teacher: select(&el, &TEACHER_SELECTOR)?,
+                semester: select_all(&el, &SEMESTER_SELECTOR)?.join(","),
+                period: select(&el, &PERIOD_SELECTOR)?,
+                code: select_all(&el, &CODE_SELECTOR)?.join(" "),
+            })
+        })
+        .collect()
 }
 
-fn select(html: &Html, selector: &Selector, nth: usize) -> anyhow::Result<String> {
-    html.select(selector)
-        .nth(nth)
+fn select(el: &ElementRef, selector: &Selector) -> anyhow::Result<String> {
+    el.select(selector)
+        .next()
         .ok_or(anyhow!(
             "Couldn't find matching element for selector {:?}",
             selector,
@@ -38,12 +43,12 @@ fn select(html: &Html, selector: &Selector, nth: usize) -> anyhow::Result<String
 }
 
 fn select_all<'a>(
-    html: &'a Html,
+    html: &'a ElementRef,
     selector: &'static Selector,
-    nth: usize,
+    // nth: usize,
 ) -> anyhow::Result<Vec<&'a str>> {
     html.select(selector)
-        .nth(nth)
+        .next()
         .ok_or(anyhow!(
             "Couldn't find matching element for selector {:?}",
             selector,

scraper/src/parser.rs

@@ -1,4 +1,8 @@
-pub static URLS: [(&str, &str); 10] = [
+pub static URLS: [(&str, &str); 11] = [
+    (
+      "zenki",
+      "https://catalog.he.u-tokyo.ac.jp/result?q=&type=all&faculty_id=&facet=%7B%22faculty_type%22%3A%5B%22jd%22%5D%7D&page=",
+    ),
     (
         "law",
         "https://catalog.he.u-tokyo.ac.jp/result?type=ug&faculty_id=1&page=",