Cleanup unused chunks

Author: Taxrosdev

src/chunks/mod.rs

@@ -1,5 +1,6 @@
 mod hash;
 mod tree;
+pub mod utils;
 
 use std::path::PathBuf;
 
@@ -20,3 +21,11 @@ pub struct Chunk {
     /// Expected size in kilobytes, rounded.
     size: u64,
 }
+
+fn get_chunk_filename(hash: &str, permissions: u32) -> String {
+    let mut new_hash = hash.to_string();
+
+    new_hash.push_str(&permissions.to_string());
+
+    new_hash
+}

src/chunks/tree.rs

@@ -6,7 +6,7 @@ use std::{
 };
 use walkdir::WalkDir;
 
-use crate::chunks::{Chunk, HashKind, hash::hash};
+use crate::chunks::{Chunk, HashKind, get_chunk_filename, hash::hash};
 
 /// Turns a filesystem tree into a list of chunks
 ///
@@ -117,14 +117,6 @@ pub fn estimate_tree_size(chunks: &[Chunk]) -> u64 {
     size
 }
 
-fn get_chunk_filename(hash: &str, permissions: u32) -> String {
-    let mut new_hash = hash.to_string();
-
-    new_hash.push_str(&permissions.to_string());
-
-    new_hash
-}
-
 #[cfg(test)]
 mod tests {
     use std::os::unix::fs::MetadataExt;

src/chunks/utils.rs

@@ -0,0 +1,70 @@
+use std::{collections::HashSet, fs, path::Path};
+
+use anyhow::Result;
+
+use crate::{
+    chunks::{Chunk, get_chunk_filename},
+    repo::{get_all_installed_packages, get_all_packages},
+};
+
+/// Removes chunks that aren't actually used by any packages in the Repository
+/// This is most useful for remote Repository administrators.
+///
+/// # Errors
+///
+/// - Filesystem errors (Permissions most likely)
+/// - Repository doesn't exist
+pub fn clean_unused(repo_path: &Path) -> Result<()> {
+    let packages = get_all_packages(repo_path)?;
+    let mut chunks: Vec<Chunk> = Vec::new();
+
+    for package in packages {
+        for chunk in package.chunks.clone() {
+            chunks.push(chunk);
+        }
+    }
+
+    clean(&repo_path.join("chunks"), &chunks)
+}
+
+/// Removes chunks that are actually used by any packages in the Repository, but aren't installed
+/// This is most useful for end users.
+///
+/// # Errors
+///
+/// - Filesystem errors (Permissions most likely)
+/// - Repository doesn't exist
+pub fn clean_used(repo_path: &Path) -> Result<()> {
+    let packages = get_all_installed_packages(repo_path)?;
+    let mut chunks: Vec<Chunk> = Vec::new();
+
+    for package in packages {
+        for chunk in package.chunks.clone() {
+            chunks.push(chunk);
+        }
+    }
+
+    clean(&repo_path.join("chunks"), &chunks)
+}
+
+/// Cleans a `chunk_store` of unused chunks, using the whitelist `allowed_chunks`
+fn clean(chunk_store_path: &Path, allowed_chunks: &[Chunk]) -> Result<()> {
+    let allowed: HashSet<String> = allowed_chunks
+        .iter()
+        .map(|c| get_chunk_filename(&c.hash, c.permissions))
+        .collect();
+
+    for entry in fs::read_dir(chunk_store_path)? {
+        let entry = entry?;
+        let file_name = entry.file_name();
+        let Some(file_name_str) = file_name.to_str() else {
+            continue;
+        };
+
+        if allowed.contains(file_name_str) {
+            fs::remove_file(entry.path())?;
+        }
+    }
+
+    Ok(())
+}

src/repo/mod.rs

@@ -129,6 +129,49 @@ pub fn get_package(repo_path: &Path, id: &str) -> Result<PackageManifest> {
     bail!("No package found in Repository.");
 }
 
+/// Lists all packages from a repository.
+///
+/// # Errors
+///
+/// - Filesystem errors (Permissions most likely)
+/// - Repository doesn't exist
+pub fn get_all_packages(repo_path: &Path) -> Result<Vec<PackageManifest>> {
+    let repo_manifest = read_manifest(repo_path)?;
+    let mut packages = Vec::new();
+
+    // Check ID's and aliases
+    for package in repo_manifest.packages {
+        packages.push(package);
+    }
+
+    Ok(packages)
+}
+
+/// Lists all installed packages from a repository.
+///
+/// # Errors
+///
+/// - Filesystem errors (Permissions most likely)
+/// - Repository doesn't exist
+pub fn get_all_installed_packages(repo_path: &Path) -> Result<Vec<PackageManifest>> {
+    let mut packages = Vec::new();
+    let installed_path = &repo_path.join("installed");
+
+    if !installed_path.exists() {
+        Ok(Vec::new())
+    } else {
+        // Check ID's and aliases
+        for entry in fs::read_dir(installed_path)? {
+            let file = entry?.path();
+            let package = serde_yaml::from_str(&fs::read_to_string(file)?)?;
+
+            packages.push(package);
+        }
+
+        Ok(packages)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use temp_dir::TempDir;