aws: Add pagination, progress bars, and human time (#6861)

describe_parameters did not have proper pagination so added some with
rate limit handling that should gracefully return what we've collected
so far.

Also adds some progress bars to make it easier to observe progress over
time.

As well this also adds the ability to specify time with human readable
types of input like: `--older-than '1h'`

This was tested already on the PyTorch AWS account and has already
deleted > 20k parameters with this.

Output looks like:


![image](https://github.com/user-attachments/assets/60840598-1611-4ac2-b139-cf0c4e548a4b)

---------

Signed-off-by: Eli Uriegas <[email protected]>

Author: seemethere

aws/tools/cleanup-ssm/Cargo.toml

@@ -10,6 +10,8 @@ clap = { version = "4.5.40", features = ["derive"] }
 tokio = { version = "1.45.1", features = ["full"] }
 chrono = "0.4"
 aws-smithy-types = "1.3"
+humantime = "2.1"
+indicatif = "0.17"
 
 [dev-dependencies]
 mockall = "0.12"

aws/tools/cleanup-ssm/src/cleanup.rs

@@ -1,4 +1,5 @@
 use crate::SsmClient;
+use indicatif::{ProgressBar, ProgressStyle};
 
 const BATCH_SIZE: usize = 10;
 
@@ -9,18 +10,40 @@ pub async fn delete_parameters_in_batches<C: SsmClient>(
     let mut total_deleted = 0;
     let mut total_failed = 0;
 
-    for chunk in parameters_to_delete.chunks(BATCH_SIZE) {
+    let total_params = parameters_to_delete.len();
+    let total_batches = (total_params + BATCH_SIZE - 1).div_ceil(BATCH_SIZE);
+
+    // Create progress bar for deletion
+    let pb = ProgressBar::new(total_batches as u64);
+    pb.set_style(
+        ProgressStyle::default_bar()
+            .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} batches ({msg})")
+            .unwrap()
+            .progress_chars("#>-")
+    );
+    pb.set_message("Deleting parameters...");
+
+    for (batch_idx, chunk) in parameters_to_delete.chunks(BATCH_SIZE).enumerate() {
         let (deleted, failed) = client.delete_parameters(chunk.to_vec()).await?;
 
         total_deleted += deleted.len();
         total_failed += failed.len();
 
-        println!("Deleted {} parameters", deleted.len());
-        if !failed.is_empty() {
-            println!("Failed to delete {} parameters", failed.len());
-        }
+        pb.set_message(format!(
+            "Batch {}/{} - Deleted: {}, Failed: {}",
+            batch_idx + 1,
+            total_batches,
+            total_deleted,
+            total_failed
+        ));
+        pb.inc(1);
     }
 
+    pb.finish_with_message(format!(
+        "✓ Completed! Total deleted: {}, Total failed: {}",
+        total_deleted, total_failed
+    ));
+
     Ok((total_deleted, total_failed))
 }
 

aws/tools/cleanup-ssm/src/client.rs

@@ -1,8 +1,11 @@
 use crate::SsmClient;
 use aws_config::BehaviorVersion;
 use aws_config::Region;
+use aws_sdk_ssm::error::ProvideErrorMetadata;
 use aws_sdk_ssm::types::ParameterMetadata;
 use aws_sdk_ssm::{Client, Error};
+use indicatif::{ProgressBar, ProgressStyle};
+use tokio::time::{sleep, Duration};
 
 pub struct AwsSsmClient {
     client: Client,
@@ -22,8 +25,58 @@ impl AwsSsmClient {
 
 impl SsmClient for AwsSsmClient {
     async fn describe_parameters(&self) -> Result<Vec<ParameterMetadata>, Error> {
-        let resp = self.client.describe_parameters().send().await?;
-        Ok(resp.parameters().to_vec())
+        let mut all_parameters = Vec::new();
+        let mut next_token: Option<String> = None;
+
+        // Create a progress bar for fetching parameters
+        let pb = ProgressBar::new_spinner();
+        pb.set_style(
+            ProgressStyle::default_spinner()
+                .template("{spinner:.green} Fetching SSM parameters... [{elapsed_precise}] {msg}")
+                .unwrap(),
+        );
+        pb.set_message("Starting...");
+
+        loop {
+            let mut request = self.client.describe_parameters().max_results(50);
+
+            if let Some(token) = next_token {
+                request = request.next_token(token);
+            }
+
+            match request.send().await {
+                Ok(resp) => {
+                    let new_params = resp.parameters().to_vec();
+                    all_parameters.extend(new_params);
+                    pb.set_message(format!("Found {} parameters", all_parameters.len()));
+                    next_token = resp.next_token().map(|s| s.to_string());
+
+                    if next_token.is_none() {
+                        break;
+                    }
+
+                    // Add delay to avoid rate limiting, 250ms seems to be the sweet spot
+                    sleep(Duration::from_millis(250)).await;
+                }
+                Err(e) => {
+                    // Check if it's a throttling error
+                    let metadata = e.meta();
+                    if metadata.code() == Some("ThrottlingException") {
+                        pb.finish_with_message(format!(
+                            "Rate limit reached. Collected {} parameters",
+                            all_parameters.len()
+                        ));
+                        return Ok(all_parameters);
+                    }
+                    // For other errors, return them
+                    pb.abandon_with_message("Error fetching parameters");
+                    return Err(e.into());
+                }
+            }
+        }
+
+        pb.finish_with_message(format!("✓ Collected {} parameters", all_parameters.len()));
+        Ok(all_parameters)
     }
 
     async fn delete_parameters(

aws/tools/cleanup-ssm/src/filter.rs

@@ -5,9 +5,9 @@ use chrono::{DateTime, Duration};
 pub fn filter_old_parameters<T: TimeProvider>(
     parameters: &[ParameterMetadata],
     time_provider: &T,
-    older_than_days: u16,
+    older_than_seconds: f64,
 ) -> Vec<String> {
-    let threshold = time_provider.now() - Duration::days(older_than_days.into());
+    let threshold = time_provider.now() - Duration::seconds(older_than_seconds as i64);
     let mut parameters_to_delete = Vec::new();
 
     for parameter in parameters {
@@ -54,7 +54,7 @@ mod tests {
         let parameters = vec![];
         let time_provider = MockTimeProvider::new(Utc::now());
 
-        let result = filter_old_parameters(&parameters, &time_provider, 1);
+        let result = filter_old_parameters(&parameters, &time_provider, 86400.0); // 1 day in seconds
 
         assert_eq!(result.len(), 0);
     }
@@ -72,7 +72,7 @@ mod tests {
         let time_provider = MockTimeProvider::new(now);
         let parameters = vec![parameter];
 
-        let result = filter_old_parameters(&parameters, &time_provider, 1);
+        let result = filter_old_parameters(&parameters, &time_provider, 86400.0); // 1 day in seconds
 
         assert_eq!(result.len(), 0);
     }
@@ -90,7 +90,7 @@ mod tests {
         let time_provider = MockTimeProvider::new(now);
         let parameters = vec![parameter];
 
-        let result = filter_old_parameters(&parameters, &time_provider, 1);
+        let result = filter_old_parameters(&parameters, &time_provider, 86400.0); // 1 day in seconds
 
         assert_eq!(result.len(), 1);
         assert_eq!(result[0], "old-param");
@@ -115,7 +115,7 @@ mod tests {
         let time_provider = MockTimeProvider::new(now);
         let parameters = vec![old_parameter, recent_parameter];
 
-        let result = filter_old_parameters(&parameters, &time_provider, 2);
+        let result = filter_old_parameters(&parameters, &time_provider, 172800.0); // 2 days in seconds
 
         assert_eq!(result.len(), 1);
         assert_eq!(result[0], "old-param");
@@ -130,7 +130,7 @@ mod tests {
         let time_provider = MockTimeProvider::new(Utc::now());
         let parameters = vec![parameter];
 
-        let result = filter_old_parameters(&parameters, &time_provider, 1);
+        let result = filter_old_parameters(&parameters, &time_provider, 86400.0); // 1 day in seconds
 
         assert_eq!(result.len(), 0);
     }
@@ -147,7 +147,7 @@ mod tests {
         let time_provider = MockTimeProvider::new(now);
         let parameters = vec![parameter];
 
-        let result = filter_old_parameters(&parameters, &time_provider, 1);
+        let result = filter_old_parameters(&parameters, &time_provider, 86400.0); // 1 day in seconds
 
         assert_eq!(result.len(), 0);
     }

aws/tools/cleanup-ssm/src/lib.rs

@@ -12,7 +12,7 @@ pub mod filter;
 pub struct CleanupConfig {
     pub region: String,
     pub dry_run: bool,
-    pub older_than_days: u16,
+    pub older_than_seconds: f64,
 }
 
 #[derive(Debug)]
@@ -53,7 +53,7 @@ pub async fn cleanup_ssm_parameters<C: SsmClient, T: TimeProvider>(
     let parameters = client.describe_parameters().await?;
 
     let parameters_to_delete =
-        filter::filter_old_parameters(&parameters, time_provider, config.older_than_days);
+        filter::filter_old_parameters(&parameters, time_provider, config.older_than_seconds);
 
     println!("Found {} parameters to delete", parameters_to_delete.len());
     let parameters_found = parameters_to_delete.len();
@@ -125,7 +125,7 @@ mod tests {
         let config = CleanupConfig {
             region: "us-east-1".to_string(),
             dry_run: true,
-            older_than_days: 1,
+            older_than_seconds: 86400.0, // 1 day in seconds
         };
 
         let result = cleanup_ssm_parameters(&mock_client, &time_provider, &config)

aws/tools/cleanup-ssm/src/main.rs

@@ -10,21 +10,32 @@ struct Args {
     // if true, will not delete any parameters
     #[clap(long, default_value_t = true, action = clap::ArgAction::Set)]
     dry_run: bool,
-    // number of days older than the parameter to delete
-    #[clap(long, default_value = "1")]
-    older_than: u16,
+    // time duration older than which to delete parameters (e.g., "1d", "2h", "30m")
+    #[clap(long, default_value = "1d")]
+    older_than: String,
 }
 
 #[tokio::main]
 async fn main() -> Result<(), Box<aws_sdk_ssm::Error>> {
     let args = Args::parse();
 
+    // Parse the human-readable time string into a Duration
+    let duration = humantime::parse_duration(&args.older_than).unwrap_or_else(|e| {
+        eprintln!("Error: Invalid time format '{}': {}", args.older_than, e);
+        eprintln!("Supported formats: 30m, 2h, 1d, 2w (minutes, hours, days, weeks)");
+        eprintln!("Note: Decimal values like '1.5d' are not supported. Use '36h' instead.");
+        std::process::exit(1);
+    });
+
+    // Get duration in seconds
+    let older_than_seconds = duration.as_secs_f64();
+
     let client = AwsSsmClient::new(&args.region).await?;
     let time_provider = SystemTimeProvider;
     let config = CleanupConfig {
         region: args.region,
         dry_run: args.dry_run,
-        older_than_days: args.older_than,
+        older_than_seconds,
     };
 
     let result = cleanup_ssm_parameters(&client, &time_provider, &config).await?;