update scraper

Author: aster-void

.cspell.json

@@ -25,7 +25,8 @@
     "rustc",
     "pkgs",
     "nixpkgs",
-    "libquery"
+    "libquery",
+    "replacen"
   ],
   "dictionaries": [
     "softwareTerms",

.gitignore

@@ -3,6 +3,9 @@
 /.direnv
 /.husky
 
+.cache
+data.json
+
 # Logs
 logs
 *.log

flake.lock

@@ -32,7 +32,7 @@
       };
       unstable = nixpkgs-unstable.legacyPackages.${system};
 
-      rust-bin = pkgs.rust-bin.fromRustupToolchainFile ./scraper/rust-toolchain.toml;
+      rust-bin = pkgs.rust-bin.beta.latest.default; # pkgs.rust-bin.fromRustupToolchainFile ./scraper/rust-toolchain.toml;
       prisma = pkgs.callPackage ./server/prisma.nix {inherit prisma-utils;};
 
       common = {
@@ -62,8 +62,7 @@
       };
     in {
       packages.scraper = pkgs.callPackage ./scraper {toolchain = rust-bin;};
-      devShells.default = pkgs.mkShell common;
-      devShells.scraper = pkgs.mkShell {
+      devShells.default = pkgs.mkShell {
         inherit (common) env;
         packages =
           common.packages

flake.nix

@@ -1,6 +1,5 @@
 use crate::types::*;
 use anyhow::ensure;
-use sha2::{Digest, Sha256};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
 
@@ -10,13 +9,20 @@ pub async fn write_to(file: &mut fs::File, content: Entry) -> anyhow::Result<()>
     Ok(())
 }
 
-use crate::CACHE_DIR;
+use crate::cache_dir;
 
 pub async fn request(url: &str) -> anyhow::Result<String> {
     println!("[request] sending request to {}", url);
 
-    let hash = Sha256::digest(url.as_bytes());
-    let path = format!("{CACHE_DIR}/{:x}", hash);
+    let cache_key = url
+        .to_string()
+        .replacen("/", "_", 1000)
+        .replacen(":", "_", 1000)
+        .replacen("?", "_", 1000)
+        .replacen("&", "_", 1000)
+        .replacen("=", "_", 1000)
+        .to_string();
+    let path = format!("{}/{cache_key}", cache_dir());
     if let Ok(bytes) = fs::read(&path).await {
         if let Ok(text) = String::from_utf8(bytes) {
             return Ok(text);

scraper/src/io.rs

@@ -16,13 +16,16 @@ use scraper::{Html, Selector};
 use urls::URLS;
 
 const RESULT_FILE: &str = "./data.json";
-const CACHE_DIR: &str = "./.cache";
+
+fn cache_dir() -> String {
+    "./.cache".to_string()
+}
 
 #[tokio::main(flavor = "multi_thread")]
 async fn main() {
     println!("[log] starting...");
 
-    let _ = fs::DirBuilder::new().create(CACHE_DIR).await;
+    let _ = fs::DirBuilder::new().create(cache_dir()).await;
 
     let mut file = fs::File::create(RESULT_FILE)
         .await

scraper/src/main.rs

@@ -92,7 +92,7 @@ model Course {
 // コマ。1つの講義に対して複数存在しうる。
 model Slot {
   id       Int    @id @default(autoincrement())
-  course   Course @relation(fields: [courseId], references: [id])
+  course   Course @relation(fields: [courseId], references: [id], onDelete: Cascade)
   courseId String
   period   Int // 1-6. 0 の場合はなし (集中など)
   day      Day // 曜日。other の場合は集中など
@@ -104,7 +104,7 @@ model Enrollment {
   id       Int    @id @default(autoincrement())
   user     User   @relation(fields: [userId], references: [id], onDelete: Cascade)
   userId   Int
-  course   Course @relation(fields: [courseId], references: [id])
+  course   Course @relation(fields: [courseId], references: [id], onDelete: Cascade)
   courseId String
 
   @@unique([userId, courseId])