change: replaced async-std, surf with tokio, reqwest

Author: nebocco

atcoder-problems-backend/Cargo.lock

@@ -6,12 +6,12 @@ edition = "2021"
 publish = false
 
 [dependencies]
-surf = "2.3"
+reqwest = { version = "0.11", features = ["json", "gzip"] }
 scraper = "0.12"
 chrono = "0.4"
 regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-futures = "0.3"
+tokio = { version = "1.15", features = ["macros", "rt"] }
 anyhow = "1.0"
 log = "0.4"

atcoder-problems-backend/atcoder-client/Cargo.toml

@@ -2,7 +2,6 @@ use crate::util;
 use anyhow::Result;
 
 use super::*;
-use surf::StatusCode;
 
 const ATCODER_PREFIX: &str = "https://atcoder.jp";
 
@@ -63,7 +62,7 @@ impl AtCoderClient {
                 max_page,
                 submissions,
             })
-        } else if status == StatusCode::NotFound {
+        } else if status == reqwest::StatusCode::NOT_FOUND {
             log::warn!("404: {}", url);
             Ok(AtCoderSubmissionListResponse {
                 max_page: 0,
@@ -89,44 +88,50 @@ impl AtCoderClient {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use futures::executor::block_on;
 
     #[test]
     fn test_fetch_contest_list() {
         let client = AtCoderClient::default();
-        let contests =
-            block_on(client.fetch_atcoder_contests(ContestTypeSpecifier::Normal { page: 1 }))
-                .unwrap();
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let contests = rt
+            .block_on(client.fetch_atcoder_contests(ContestTypeSpecifier::Normal { page: 1 }))
+            .unwrap();
         assert_eq!(contests.len(), 50);
     }
 
     #[test]
     fn test_fetch_hidden_contest() {
         let client = AtCoderClient::default();
-        let contests =
-            block_on(client.fetch_atcoder_contests(ContestTypeSpecifier::Hidden)).unwrap();
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let contests = rt
+            .block_on(client.fetch_atcoder_contests(ContestTypeSpecifier::Hidden))
+            .unwrap();
         assert!(!contests.is_empty());
     }
 
     #[test]
     fn test_fetch_problem_list() {
         let client = AtCoderClient::default();
-        let problems = block_on(client.fetch_problem_list("abc107")).unwrap();
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let problems = rt.block_on(client.fetch_problem_list("abc107")).unwrap();
         assert_eq!(problems.len(), 4);
     }
 
     #[test]
     fn test_fetch_submission_list() {
         let client = AtCoderClient::default();
-        let response = block_on(client.fetch_atcoder_submission_list("xmascon17", None)).unwrap();
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let response = rt
+            .block_on(client.fetch_atcoder_submission_list("xmascon17", None))
+            .unwrap();
         assert_eq!(response.submissions.len(), 20);
 
-        let response =
-            block_on(client.fetch_atcoder_submission_list("xmascon17", Some(response.max_page)))
-                .unwrap();
+        let response = rt
+            .block_on(client.fetch_atcoder_submission_list("xmascon17", Some(response.max_page)))
+            .unwrap();
         assert!(!response.submissions.is_empty());
 
-        let response = block_on(
+        let response = rt.block_on(
             client.fetch_atcoder_submission_list("xmascon17", Some(response.max_page + 1)),
         );
         assert!(response.is_err());

atcoder-problems-backend/atcoder-client/src/atcoder/client.rs

@@ -2,10 +2,10 @@ use anyhow::{anyhow, Result};
 
 use serde::de::DeserializeOwned;
 
-pub(crate) async fn get_html(url: &str) -> Result<(String, surf::StatusCode)> {
-    let mut response = surf::get(url)
+pub(crate) async fn get_html(url: &str) -> Result<(String, reqwest::StatusCode)> {
+    let response = reqwest::Client::builder().gzip(true).build()?
+        .get(url)
         .header("accept", "text/html")
-        .header("accept-encoding", "gzip")
         .send()
         .await
         .map_err(|e| anyhow!("Connection error: {:?}", e))?;
@@ -14,19 +14,22 @@ pub(crate) async fn get_html(url: &str) -> Result<(String, surf::StatusCode)> {
         log::error!("{:?}", response);
     }
     let body = response
-        .body_string()
+        .text()
         .await
         .map_err(|e| anyhow!("Failed to parse HTTP body: {:?}", e))?;
     Ok((body, status))
 }
 
 pub(crate) async fn get_json<T: DeserializeOwned>(url: &str) -> Result<T> {
-    surf::get(url)
+    reqwest::Client::builder().gzip(true).build()?
+        .get(url)
         .header("accept", "application/json")
-        .header("accept-encoding", "gzip")
-        .recv_json()
+        .send()
+        .await
+        .map_err(|_| anyhow!("Failed to get json from {}", url))?
+        .json::<T>()
         .await
-        .map_err(|_| anyhow!("Failed to get json from {}", url))
+        .map_err(|_| anyhow!("Failed to parse json from {}", url))
 }
 
 pub trait Problem {

atcoder-problems-backend/atcoder-client/src/util.rs

@@ -8,12 +8,11 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-sqlx = { version = "0.5.9", features = ["postgres", "runtime-async-std-rustls"] }
+sqlx = { version = "0.5.9", features = ["postgres", "runtime-tokio-rustls"] }
 async-trait = "0.1.52"
 serde = { version = "1.0", features = ["derive"] }
 uuid = { version = "0.8", features = ["serde", "v4"] }
 anyhow = "1.0.51"
-async-std = { version = "1.10.0", features = ["attributes"] }
+tokio = { version = "1.15", features = ["macros"] }
 regex = "1"
-futures = "0.3"
 chrono = "0.4"

atcoder-problems-backend/sql-client/Cargo.toml

@@ -1,7 +1,6 @@
 use crate::PgPool;
 use anyhow::Result;
 use async_trait::async_trait;
-use futures::try_join;
 
 #[async_trait]
 pub trait ProblemsSubmissionUpdater {
@@ -15,7 +14,7 @@ impl ProblemsSubmissionUpdater for PgPool {
         let fastest_sql = generate_query("fastest", "execution_time");
         let shortest_sql = generate_query("shortest", "length");
 
-        try_join!(
+        tokio::try_join!(
             sqlx::query(&first_sql).execute(self),
             sqlx::query(&fastest_sql).execute(self),
             sqlx::query(&shortest_sql).execute(self),

atcoder-problems-backend/sql-client/src/problems_submissions.rs

@@ -2,7 +2,6 @@ use crate::models::{ContestProblem, Submission, UserSum};
 use crate::{PgPool, FIRST_AGC_EPOCH_SECOND, MAX_INSERT_ROWS, UNRATED_STATE};
 use anyhow::Result;
 use async_trait::async_trait;
-use futures::try_join;
 use sqlx::postgres::PgRow;
 use sqlx::Row;
 use std::collections::{BTreeMap, BTreeSet};
@@ -45,7 +44,7 @@ impl RatedPointSumClient for PgPool {
                 .fetch_all(self);
 
         let (rated_contest_ids, rated_problem_ids) =
-            try_join!(rated_contest_ids_fut, rated_problem_ids_fut)?;
+            tokio::try_join!(rated_contest_ids_fut, rated_problem_ids_fut)?;
 
         let rated_contest_ids = rated_contest_ids.into_iter().collect::<BTreeSet<_>>();
         let rated_problem_ids = rated_problem_ids

atcoder-problems-backend/sql-client/src/rated_point_sum.rs

@@ -3,7 +3,7 @@ use sql_client::models::{Submission, UserProblemCount};
 
 mod utils;
 
-#[async_std::test]
+#[tokio::test]
 async fn test_accepted_count() {
     let pool = utils::initialize_and_connect_to_test_sql().await;
     let submissions = [

atcoder-problems-backend/sql-client/tests/test_accepted_count.rs

@@ -10,7 +10,7 @@ fn create_problem(id: i32) -> ContestProblem {
     }
 }
 
-#[async_std::test]
+#[tokio::test]
 async fn test_contest_problem() {
     let pool = utils::initialize_and_connect_to_test_sql().await;
     assert!(pool.load_contest_problem().await.unwrap().is_empty());

atcoder-problems-backend/sql-client/tests/test_contest_problem.rs

@@ -3,7 +3,7 @@ use sql_client::models::{Submission, UserLanguageCount, UserLanguageCountRank, U
 
 mod utils;
 
-#[async_std::test]
+#[tokio::test]
 async fn test_language_count() {
     let pool = utils::initialize_and_connect_to_test_sql().await;
     let mut submissions = vec![