feat: support delete deprecated domain files

Author: timzaak

README.md

@@ -1,7 +1,7 @@
 # SPA-SERVER
 It is to provide a static web http server with cache and hot reload.
 
-[中文 README](./README_CN.md)
+[中文说明](./README_CN.md)
 
 ## Feature
 - Built with Hyper and Warp, fast and small!

client/src/api.rs

@@ -2,7 +2,9 @@ use crate::Config;
 use anyhow::anyhow;
 use reqwest::{header, multipart, StatusCode};
 use serde_json::Value;
-use spa_server::admin_server::request::{DomainWithOptVersionOption, UpdateUploadingStatusOption};
+use spa_server::admin_server::request::{
+    DeleteDomainVersionOption, DomainWithOptVersionOption, UpdateUploadingStatusOption,
+};
 use spa_server::domain_storage::{ShortMetaData, UploadDomainPosition};
 use std::borrow::Cow;
 use std::path::PathBuf;
@@ -117,6 +119,22 @@ impl API {
         handle!(resp)
     }
 
+    pub fn remove_files(
+        &self,
+        domain: Option<String>,
+        max_reserve: Option<u32>,
+    ) -> anyhow::Result<()> {
+        let resp = self
+            .blocking_client
+            .post(self.url("files/delete"))
+            .json(&DeleteDomainVersionOption {
+                domain,
+                max_reserve,
+            })
+            .send()?;
+        handle!(resp)
+    }
+
     pub async fn upload_file<T: Into<Cow<'static, str>>>(
         &self,
         domain: T,

client/src/commands.rs

@@ -18,6 +18,7 @@ pub enum Commands {
         domain: String,
         version: Option<u32> },
     Reload,
+    Delete {domain: Option<String>, max_reserve:Option<u32>}
 }
 
 #[derive(Args, Debug)]

client/src/lib.rs

@@ -69,13 +69,20 @@ fn run_with_commands(commands: CliCommand) -> anyhow::Result<()> {
             api.reload_sap_server()?;
             success("reload success!");
         }
+        Commands::Delete {
+            domain,
+            max_reserve,
+        } => {
+            api.remove_files(domain, max_reserve)?;
+            success("delete success!");
+        }
     };
     Ok(())
 }
 
 #[cfg(test)]
 mod test {
-    use crate::{run_with_commands, CliCommand};
+    use crate::{run, run_with_commands, CliCommand};
     use clap::Parser;
     use std::env;
 
@@ -115,4 +122,15 @@ mod test {
         ]));
         result.unwrap();
     }
+    #[test]
+    fn test_delete() {
+        init_config();
+        let result = run_with_commands(CliCommand::parse_from(&[
+            "test",
+            "delete",
+            "self.noti.link",
+            "2",
+        ]));
+        result.unwrap();
+    }
 }

docs/develop/roadmap.md

@@ -7,11 +7,12 @@ There is no real roadmap for v1.3.x now, need users firstly.
 - [ ] CD: release triggered by new tag
 - [ ] TEST: test integrate, and add ci to run it
 
-### Version v1.2.6
+### Version v1.2.6(client:v0.1.4)
 - [ ] doc: add spa-server usage scenario and corresponding config File
 - [ ] doc: what's version control and file directory layout
 - [ ] chore: spa-client command help doc
 - [ ] bench: add benchmark
+- [x] feat(with client): delete deprecated version to save storage
 - [x] feat: Support S3 by docker(backward: release docker timzaak/spa-server:1.2.5-s3)
 - [x] deps: bump hocon 0.9, fix size unit config parse
 - [x] fix(build): disable generating new tag when build spa-client(js) success

docs/guide/spa-client-command-line.md

@@ -16,6 +16,9 @@ spa-client -c $CONFIG_PATH info $OPT_DOMAIN
 
 # reload spa-server, this is used to reload https cert
 spa-client -c $CONFIG_PATH reload
+
+# delete deprecated domain files
+spa-client -c $CONFIG_PATH delete $OPT_DOMAIN $OPT_MAX_RESERVE
 ```
 
 There also provides http api to interact with admin server,

docs/guide/spa-server-api.md

@@ -104,4 +104,16 @@ PATH="/upload/file/path"
 curl -X POST "http://$ADMIN_SERVER/file/upload" \
 -F "file=@$PATH" -F "domain=$DOMAIN" -F "version=$VERSION" -F "path=$PATH"  -H "Authorization: Bearer $TOKEN"
 # return status code:200 if success 
+```
+
+### Delete deprecated domain files
+```shell
+# keep 2 versions. 
+MAX_RESERVE = 1
+curl -X POST "http://$ADMIN_SERVER/files/delete" \
+--data-raw `{
+  "domain":$DOMAIN,
+  "max_reserve": $MAX_RESERVE
+}`
+# return status code:200 if success 
 ```

server/src/admin_server.rs

@@ -1,6 +1,6 @@
 use crate::admin_server::request::{
-    DomainWithOptVersionOption, DomainWithVersionOption, GetDomainOption, GetDomainPositionOption,
-    UpdateUploadingStatusOption,
+    DeleteDomainVersionOption, DomainWithOptVersionOption, DomainWithVersionOption,
+    GetDomainOption, GetDomainPositionOption, UpdateUploadingStatusOption,
 };
 use crate::config::AdminConfig;
 use crate::domain_storage::DomainStorage;
@@ -45,7 +45,8 @@ impl AdminServer {
                 self.update_domain_version()
                     .or(self.reload_server())
                     .or(self.change_upload_status())
-                    .or(self.upload_file()),
+                    .or(self.upload_file())
+                    .or(self.remove_domain_version()),
             )),
         )
     }
@@ -150,12 +151,22 @@ impl AdminServer {
             .and(warp::query::<DomainWithVersionOption>())
             .map(service::get_files_metadata)
     }
+
+    fn remove_domain_version(
+        &self,
+    ) -> impl Filter<Extract = (impl warp::Reply,), Error = Rejection> + Clone {
+        warp::path!("files" / "delete")
+            .and(with(self.domain_storage.clone()))
+            .and(warp::query::<DeleteDomainVersionOption>())
+            .map(service::remove_domain_version)
+    }
 }
 
 pub mod service {
     use crate::admin_server::request::{
-        DomainWithOptVersionOption, DomainWithVersionOption, GetDomainOption,
-        GetDomainPositionFormat, GetDomainPositionOption, UpdateUploadingStatusOption,
+        DeleteDomainVersionOption, DomainWithOptVersionOption, DomainWithVersionOption,
+        GetDomainOption, GetDomainPositionFormat, GetDomainPositionOption,
+        UpdateUploadingStatusOption,
     };
     use crate::domain_storage::{DomainStorage, URI_REGEX};
     use crate::{AdminConfig, HotReloadManager};
@@ -184,7 +195,7 @@ pub mod service {
                     return Ok(StatusCode::NOT_FOUND.into_response());
                 }
             }
-            None => Ok(warp::reply::json(&domain_info).into_response()),
+            _ => Ok(warp::reply::json(&domain_info).into_response()),
         }
     }
 
@@ -344,6 +355,46 @@ pub mod service {
             }
         }
     }
+
+    pub(super) fn remove_domain_version(
+        storage: Arc<DomainStorage>,
+        query: DeleteDomainVersionOption,
+    ) -> Response {
+        let domains_info = if let Some(domain) = query.domain {
+            storage
+                .get_domain_info_by_domain(&domain)
+                .map(|v| vec![v])
+                .unwrap_or(vec![])
+        } else {
+            storage.get_domain_info()
+        };
+        for info in domains_info {
+            let delete_versions = if let Some(max_reserve) = query.max_reserve {
+                if let Some(mut max_version) =
+                    info.current_version
+                        .or(info.versions.iter().max().map(|x| *x))
+                {
+                    max_version = max_version - max_reserve;
+                    info.versions
+                        .into_iter()
+                        .filter(|v| *v < max_version)
+                        .collect::<Vec<u32>>()
+                } else {
+                    vec![]
+                }
+            } else {
+                let current_version = info.current_version.unwrap_or(u32::MAX);
+                info.versions
+                    .into_iter()
+                    .filter(|version| *version != current_version)
+                    .collect::<Vec<u32>>()
+            };
+            for version in delete_versions {
+                let _ = storage.remove_domain_version(&info.domain, Some(version));
+            }
+        }
+        Response::default()
+    }
 }
 
 pub mod request {
@@ -385,12 +436,19 @@ pub mod request {
         pub domain: String,
         pub version: Option<u32>,
     }
+
     #[derive(Deserialize, Serialize)]
     pub struct UpdateUploadingStatusOption {
         pub domain: String,
         pub version: u32,
         pub status: UploadingStatus,
     }
+
+    #[derive(Deserialize, Serialize)]
+    pub struct DeleteDomainVersionOption {
+        pub domain: Option<String>,
+        pub max_reserve: Option<u32>,
+    }
 }
 
 // TODO: the code structure is not friendly with Unit Test, need refactor it.

server/src/domain_storage.rs

@@ -191,7 +191,7 @@ impl DomainStorage {
     }
 
     pub fn get_domain_info_by_domain(&self, domain: &str) -> Option<DomainInfo> {
-        let versions:Vec<u32> = walkdir::WalkDir::new(&self.prefix.join(domain)).max_depth(1).into_iter().filter_map(|version_entity|{
+        let versions:Vec<u32> = WalkDir::new(&self.prefix.join(domain)).max_depth(1).into_iter().filter_map(|version_entity|{
             let version_entity =version_entity.ok()?;
             let version = version_entity.file_name().to_str()?.parse::<u32>().ok()?;
             Some(version)
@@ -378,6 +378,23 @@ impl DomainStorage {
         }
         Ok(())
     }
+
+    pub fn remove_domain_version(&self, domain:&str, version:Option<u32>) ->anyhow::Result<bool> {
+        let mut path = self.prefix.join(domain);
+        if let Some(version) = version {
+            path = path.join(version.to_string());
+            if path.exists() {
+                fs::remove_dir_all(path)?;
+                return Ok(true);
+            }
+        }else {
+            if path.exists() {
+                fs::remove_dir_all(path)?;
+                return Ok(true);
+            }
+        }
+        return Ok(false)
+    }
 }
 
 pub fn md5_file(path: impl AsRef<Path>, byte_buffer: &mut Vec<u8>) -> Option<String> {