[bilibili.space] Support requesting with cookies

Author: SpriteOvO

Cargo.lock

@@ -25,6 +25,7 @@ serde_json = "1.0.145"
 shadow-rs = "1.3.0"
 spdlog-rs = { version = "0.4.3", features = ["source-location"] }
 tempfile = "3.22.0"
+thiserror = "2.0.16"
 tokio = { version = "1.47.1", features = [
     "rt-multi-thread",
     "macros",

Cargo.toml

@@ -449,9 +449,10 @@ notify = ["meow", "woof", { ref = "woof", id = 123 }]
                             experimental: Default::default()
                         })),
                         twitter: Accessor::new(Some(twitter::ConfigGlobal {
-                            account: Accounts::from_iter([("MyTwitter".into(), Accessor::new(twitter::ConfigCookies::with_raw("a=b;c=d;ct0=blah")))])
+                            account: Accounts::from_iter([("MyTwitter".into(), Accessor::new(ConfigCookies::with_raw("a=b;c=d;ct0=blah")))])
                         })),
                         bilibili: Accessor::new(Some(bilibili::ConfigGlobal {
+                            cookies: Accessor::new(None),
                             playback: Accessor::new(Some(bilibili::source::playback::ConfigGlobal {
                                 bililive_recorder: Accessor::new(bilibili::source::playback::bililive_recorder::ConfigBililiveRecorder {
                                     listen_webhook: bilibili::source::playback::bililive_recorder::ConfigListen {

src/config/mod.rs

@@ -1,5 +1,7 @@
 use std::{borrow::Cow, env, error::Error as StdError, str::FromStr};
 
+use serde::Deserialize;
+
 pub trait AsSecretRef<'a, T = &'a str> {
     fn as_secret_ref(&'a self) -> SecretRef<'a, T>;
 }
@@ -129,3 +131,11 @@ macro_rules! secret_enum {
         }
     };
 }
+
+secret_enum! {
+    #[derive(Clone, Debug, PartialEq, Deserialize)]
+    #[serde(rename_all = "snake_case")]
+    pub enum ConfigCookies {
+        Cookies(String),
+    }
+}

src/config/secret.rs

@@ -4,17 +4,20 @@ use reqwest::header::{self, HeaderMap, HeaderValue};
 use serde::Deserialize;
 
 use crate::{
-    config::{Accessor, Validator},
+    config::{Accessor, ConfigCookies, Validator},
     helper, prop,
 };
 
 #[derive(Clone, Debug, PartialEq, Deserialize)]
 pub struct ConfigGlobal {
+    #[serde(flatten)]
+    pub cookies: Accessor<Option<ConfigCookies>>,
     pub playback: Accessor<Option<source::playback::ConfigGlobal>>,
 }
 
 impl Validator for ConfigGlobal {
     fn validate(&self) -> anyhow::Result<()> {
+        self.cookies.validate()?;
         self.playback.validate()?;
         Ok(())
     }

src/platform/bilibili/mod.rs

@@ -1,23 +1,26 @@
 use std::{
+    borrow::Cow,
     collections::HashSet,
     fmt::{self, Display},
     future::Future,
     ops::DerefMut,
     pin::Pin,
+    str::FromStr,
     sync::{Arc, Mutex as StdMutex},
 };
 
 use anyhow::{anyhow, bail, ensure};
 use chrono::DateTime;
+use reqwest::{header::COOKIE, Url};
 use serde::Deserialize;
 use serde_json::{self as json};
 use spdlog::prelude::*;
 use tokio::sync::Mutex;
 
 use super::super::{upgrade_to_https, Response};
 use crate::{
-    config::{Accessor, Validator},
-    platform::{PlatformMetadata, PlatformTrait},
+    config::{Accessor, AsSecretRef, Config, Validator},
+    platform::{bilibili::bilibili_request_builder, PlatformMetadata, PlatformTrait},
     prop,
     source::{
         FetcherTrait, Post, PostAttachment, PostAttachmentImage, PostContent, PostUrl, PostUrls,
@@ -486,9 +489,21 @@ impl Fetcher {
         }
     }
 
+    fn cookies(&self) -> Option<Cow<'_, str>> {
+        Config::global().platform().bilibili.as_ref().and_then(|b| {
+            b.cookies
+                .as_ref()
+                .map(|c| c.as_secret_ref().get_str().unwrap())
+        })
+    }
+
     async fn fetch_status_impl(&self) -> anyhow::Result<Status> {
-        let posts =
-            fetch_space_history(self.params.user_id, self.blocked.lock().await.deref_mut()).await?;
+        let posts = fetch_space_history(
+            self.params.user_id,
+            self.blocked.lock().await.deref_mut(),
+            self.cookies(),
+        )
+        .await?;
 
         Ok(Status::new(
             StatusKind::Posts(posts),
@@ -505,32 +520,54 @@ impl Fetcher {
 // Fans-only posts
 struct BlockedPostIds(HashSet<String>);
 
-async fn fetch_space_history(user_id: u64, blocked: &mut BlockedPostIds) -> anyhow::Result<Posts> {
-    fetch_space_history_impl(user_id, blocked).await
+async fn fetch_space_history(
+    user_id: u64,
+    blocked: &mut BlockedPostIds,
+    cookies: Option<Cow<'_, str>>,
+) -> anyhow::Result<Posts> {
+    let res = fetch_space_history_impl(user_id, blocked, cookies).await;
+    if let Err(FetchSpaceHistoryError::Auth(true)) = res {
+        warn!("bilibili space auth error with cookies used, retrying headless without cookies");
+        Ok(fetch_space_history_impl(user_id, blocked, None).await?)
+    } else {
+        Ok(res?)
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+enum FetchSpaceHistoryError {
+    #[error("auth error (cookies used: {0})")]
+    Auth(bool),
+    #[error("{0}")]
+    Anyhow(#[from] anyhow::Error),
+    #[error("response contains error, response '{0}'")]
+    Others(String),
 }
 
 fn fetch_space_history_impl<'a>(
     user_id: u64,
     blocked: &'a mut BlockedPostIds,
-) -> Pin<Box<dyn Future<Output = anyhow::Result<Posts>> + Send + 'a>> {
+    cookies: Option<Cow<'a, str>>,
+) -> Pin<Box<dyn Future<Output = Result<Posts, FetchSpaceHistoryError>> + Send + 'a>> {
     Box::pin(async move {
-        let (status, text) = fetch_space(user_id)
+        let cookies_used = cookies.is_some();
+        let (status, text) = fetch_space(user_id, cookies)
             .await
             .map_err(|err| anyhow!("failed to send request: {err}"))?;
         if status != 200 {
-            bail!("response status is not success: {text:?}");
+            return Err(anyhow!("response status is not success: {text:?}").into());
         }
 
         let resp: Response<data::SpaceHistory> = json::from_str(&text)
             .map_err(|err| anyhow!("failed to deserialize response: {err}"))?;
 
         match resp.code {
             0 => {} // Success
-            -352 => bail!("auth error"),
-            _ => bail!("response contains error, response '{text}'"),
+            -352 => return Err(FetchSpaceHistoryError::Auth(cookies_used)),
+            _ => return Err(FetchSpaceHistoryError::Others(text)),
         }
 
-        parse_response(resp.data.unwrap(), blocked)
+        Ok(parse_response(resp.data.unwrap(), blocked)?)
     })
 }
 
@@ -787,7 +824,47 @@ fn parse_response(resp: data::SpaceHistory, blocked: &mut BlockedPostIds) -> any
     Ok(Posts(items))
 }
 
-async fn fetch_space(user_id: u64) -> anyhow::Result<(u32, String)> {
+const ENDPOINT_URL: &str = "https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/space";
+
+async fn fetch_space(user_id: u64, cookies: Option<Cow<'_, str>>) -> anyhow::Result<(u32, String)> {
+    let Some(cookies) = cookies else {
+        return fetch_space_via_headless(user_id).await;
+    };
+
+    const FEATURES: &[&str] = &[
+        "itemOpusStyle",
+        "listOnlyfans",
+        "opusBigCover",
+        "onlyfansVote",
+        "forwardListHidden",
+        "decorationCard",
+        "commentsNewVersion",
+        "onlyfansAssetsV2",
+        "ugcDelete",
+        "onlyfansQaCard",
+    ];
+    let mut url = Url::from_str(ENDPOINT_URL)?;
+    {
+        let mut query = url.query_pairs_mut();
+        query.append_pair("host_mid", &user_id.to_string());
+        query.append_pair("platform", "web");
+        query.append_pair("features", &FEATURES.join(","));
+    }
+    let resp = bilibili_request_builder()?
+        .get(url.as_ref())
+        .header(COOKIE, &*cookies)
+        .send()
+        .await
+        .map_err(|err| anyhow!("failed to send request with cookies: {err}"))?;
+    let status = resp.status();
+    let text = resp
+        .text()
+        .await
+        .map_err(|err| anyhow!("failed to read response text for bilibili: {err}"))?;
+    Ok((status.as_u16() as u32, text))
+}
+
+async fn fetch_space_via_headless(user_id: u64) -> anyhow::Result<(u32, String)> {
     // Okay, I gave up on cracking the auth process
     use headless_chrome::{Browser, LaunchOptionsBuilder};
 
@@ -805,11 +882,7 @@ async fn fetch_space(user_id: u64) -> anyhow::Result<(u32, String)> {
         Box::new({
             let body_res = Arc::clone(&body_res);
             move |event, fetch_body| {
-                if event
-                    .response
-                    .url
-                    .starts_with("https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/space")
-                {
+                if event.response.url.starts_with(ENDPOINT_URL) {
                     *body_res.lock().unwrap() = Some((event.response.status, fetch_body()));
                 }
             }
@@ -846,7 +919,9 @@ mod tests {
     async fn deser() {
         let mut blocked = BlockedPostIds(HashSet::new());
 
-        let history = fetch_space_history(8047632, &mut blocked).await.unwrap();
+        let history = fetch_space_history(8047632, &mut blocked, None)
+            .await
+            .unwrap();
         assert!(history.0.iter().all(|post| !post
             .urls
             .major()
@@ -856,7 +931,9 @@ mod tests {
             .is_empty()));
         assert!(history.0.iter().all(|post| !post.content.is_empty()));
 
-        let history = fetch_space_history(178362496, &mut blocked).await.unwrap();
+        let history = fetch_space_history(178362496, &mut blocked, None)
+            .await
+            .unwrap();
         assert!(history.0.iter().all(|post| !post
             .urls
             .major()

src/platform/bilibili/source/space.rs

@@ -4,10 +4,7 @@ pub(crate) mod source;
 use request::TwitterCookies;
 use serde::Deserialize;
 
-use crate::{
-    config::{Accounts, AsSecretRef, Validator},
-    secret_enum,
-};
+use crate::config::{Accounts, AsSecretRef, ConfigCookies, Validator};
 
 // Global
 //
@@ -26,11 +23,3 @@ impl Validator for ConfigGlobal {
         Ok(())
     }
 }
-
-secret_enum! {
-    #[derive(Clone, Debug, PartialEq, Deserialize)]
-    #[serde(rename_all = "snake_case")]
-    pub enum ConfigCookies {
-        Cookies(String),
-    }
-}