Logging to directory + log file management; default to log directory for hf_xet (#502)

This PR switches the default logging to log events to a file in
'~/.cache/huggingface/xet/logs' (or 'xet/logs' under the specified cache
directory if not `~/.cache/huggingface/`).

In this directory, log files older than 2 weeks are cleaned up on
process start, and if the total size of files in the directory is larger
than 1gb, then log files are deleted by age to get the directory size
under 1gb. Log files are named with a timestamp and PID; by default,
logs newer than 1 day or logs with an active associated PID are never
deleted. All of these are user configurable constants.

Author: hoytak

Cargo.lock

@@ -16,6 +16,7 @@ members = [
     "merklehash",
     "progress_tracking",
     "utils",
+    "xet_logging",
     "xet_runtime",
 ]
 
@@ -94,6 +95,7 @@ serde_repr = "0.1"
 sha2 = "0.10"
 shellexpand = "3.1"
 static_assertions = "1.1"
+sysinfo = "0.37"
 tempfile = "3.20"
 thiserror = "2.0"
 tokio = { version = "1.47" }

Cargo.toml

@@ -44,7 +44,6 @@ async-trait = { workspace = true }
 bytes = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true }
-dirs = { workspace = true }
 jsonwebtoken = { workspace = true }
 lazy_static = { workspace = true }
 more-asserts = { workspace = true }
@@ -72,6 +71,7 @@ sha2 = { workspace = true }
 serial_test = { workspace = true }
 tracing-test = { workspace = true }
 ctor = { workspace = true }
+dirs = { workspace = true }
 
 [features]
 strict = []

data/Cargo.toml

@@ -1,5 +1,3 @@
-use std::env;
-use std::env::current_dir;
 use std::fs::File;
 use std::io::Read;
 use std::path::{Path, PathBuf};
@@ -9,15 +7,13 @@ use cas_client::remote_client::PREFIX_DEFAULT;
 use cas_client::{CHUNK_CACHE_SIZE_BYTES, CacheConfig, FileProvider, OutputProvider};
 use cas_object::CompressionScheme;
 use deduplication::DeduplicationMetrics;
-use dirs::home_dir;
 use progress_tracking::TrackingProgressUpdater;
 use progress_tracking::item_tracking::ItemProgressUpdater;
 use tracing::{Instrument, Span, info, info_span, instrument};
 use ulid::Ulid;
 use utils::auth::{AuthConfig, TokenRefresher};
-use utils::normalized_path_from_user_string;
 use xet_runtime::utils::run_constrained_with_semaphore;
-use xet_runtime::{GlobalSemaphoreHandle, XetRuntime, global_semaphore_handle};
+use xet_runtime::{GlobalSemaphoreHandle, XetRuntime, global_semaphore_handle, xet_cache_root};
 
 use crate::configurations::*;
 use crate::constants::{INGESTION_BLOCK_SIZE, MAX_CONCURRENT_DOWNLOADS};
@@ -35,34 +31,7 @@ pub fn default_config(
     token_info: Option<(String, u64)>,
     token_refresher: Option<Arc<dyn TokenRefresher>>,
 ) -> errors::Result<TranslatorConfig> {
-    // if HF_HOME is set use that instead of ~/.cache/huggingface
-    // if HF_XET_CACHE is set use that instead of ~/.cache/huggingface/xet
-    // HF_XET_CACHE takes precedence over HF_HOME
-    let cache_root_path = {
-        // If HF_XET_CACHE is set, use that directly.
-        if let Ok(cache) = env::var("HF_XET_CACHE") {
-            normalized_path_from_user_string(cache)
-
-        // If HF_HOME is set, use the $HF_HOME/xet
-        } else if let Ok(hf_home) = env::var("HF_HOME") {
-            normalized_path_from_user_string(hf_home).join("xet")
-
-        // If XDG_CACHE_HOME is set, use the $XDG_CACHE_HOME/huggingface/xet, otherwise
-        // use $HOME/.cache/huggingface/xet
-        } else if let Ok(xdg_cache_home) = env::var("XDG_CACHE_HOME") {
-            normalized_path_from_user_string(xdg_cache_home).join("huggingface").join("xet")
-
-        // Use the same default as huggingface_hub, ~/.cache/huggingface/xet (slightly nonstandard, but won't
-        // mess with it).
-        } else {
-            home_dir()
-                .unwrap_or(current_dir()?)
-                .join(".cache")
-                .join("huggingface")
-                .join("xet")
-        }
-    };
-
+    let cache_root_path = xet_cache_root();
     info!("Using cache path {cache_root_path:?}.");
 
     let (token, token_expiration) = token_info.unzip();
@@ -287,6 +256,7 @@ async fn smudge_file(
 
 #[cfg(test)]
 mod tests {
+    use dirs::home_dir;
     use serial_test::serial;
     use tempfile::tempdir;
     use utils::EnvVarGuard;

data/src/data_client.rs

@@ -16,6 +16,7 @@ error_printer = { path = "../error_printer" }
 progress_tracking = { path = "../progress_tracking" }
 utils = { path = "../utils" }
 xet_runtime = { path = "../xet_runtime" }
+xet_logging = { path = "../xet_logging" }
 
 async-trait = "0.1"
 bipbuffer = "0.1"
@@ -36,15 +37,6 @@ rand = "0.9.2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = [
-    "json",
-    "tracing-log",
-    "env-filter",
-    "registry",
-] }
-tracing-appender = "0.2"
-
-console-subscriber = { version = "0.4.1", optional = true }
 
 # Unix-specific dependencies
 [target.'cfg(unix)'.dependencies]
@@ -58,7 +50,7 @@ ctrlc = "3.4"
 native-tls = ["cas_client/native-tls-vendored"]
 native-tls-vendored = ["cas_client/native-tls-vendored"]
 profiling = ["pprof"]
-tokio-console = ["dep:console-subscriber"]
+tokio-console = ["xet_logging/tokio-console"]
 
 [profile.release]
 split-debuginfo = "packed"