node, store: Allow finding unused deployments that were recorded a while ago

Author: lutter

node/src/bin/manager.rs

@@ -2,7 +2,7 @@ use std::{collections::HashMap, env, num::ParseIntError, sync::Arc, time::Durati
 
 use config::PoolSize;
 use git_testament::{git_testament, render_testament};
-use graph::{data::graphql::effort::LoadManager, prometheus::Registry};
+use graph::{data::graphql::effort::LoadManager, prelude::chrono, prometheus::Registry};
 use graph_core::MetricsRegistry;
 use graph_graphql::prelude::GraphQlRunner;
 use lazy_static::lazy_static;
@@ -200,6 +200,9 @@ pub enum UnusedCommand {
         /// Remove a specific deployment
         #[structopt(short, long, conflicts_with = "count")]
         deployment: Option<String>,
+        /// Remove unused deployments that were recorded at least this many minutes ago
+        #[structopt(short, long)]
+        older: Option<u32>,
     },
 }
 
@@ -531,9 +534,14 @@ async fn main() {
             match cmd {
                 List { existing } => commands::unused_deployments::list(store, existing),
                 Record => commands::unused_deployments::record(store),
-                Remove { count, deployment } => {
+                Remove {
+                    count,
+                    deployment,
+                    older,
+                } => {
                     let count = count.unwrap_or(1_000_000);
-                    commands::unused_deployments::remove(store, count, deployment)
+                    let older = older.map(|older| chrono::Duration::minutes(older as i64));
+                    commands::unused_deployments::remove(store, count, deployment, older)
                 }
             }
         }

node/src/manager/commands/unused_deployments.rs

@@ -1,6 +1,6 @@
 use std::{sync::Arc, time::Instant};
 
-use graph::prelude::anyhow::Error;
+use graph::prelude::{anyhow::Error, chrono};
 use graph_store_postgres::{unused, SubgraphStore, UnusedDeployment};
 
 use crate::manager::display::List;
@@ -77,8 +77,13 @@ pub fn remove(
     store: Arc<SubgraphStore>,
     count: usize,
     deployment: Option<String>,
+    older: Option<chrono::Duration>,
 ) -> Result<(), Error> {
-    let unused = store.list_unused_deployments(unused::Filter::New)?;
+    let filter = match older {
+        Some(duration) => unused::Filter::UnusedLongerThan(duration),
+        None => unused::Filter::New,
+    };
+    let unused = store.list_unused_deployments(filter)?;
     let unused = match &deployment {
         None => unused,
         Some(deployment) => unused

store/postgres/src/primary.rs

@@ -24,7 +24,7 @@ use diesel::{
 };
 use graph::{
     components::store::DeploymentId as GraphDeploymentId,
-    prelude::{CancelHandle, CancelToken},
+    prelude::{chrono, CancelHandle, CancelToken},
 };
 use graph::{
     components::store::DeploymentLocator,
@@ -1392,6 +1392,21 @@ impl<'a> Connection<'a> {
                 .filter(u::removed_at.is_null())
                 .order_by(u::entity_count)
                 .load(self.conn.as_ref())?),
+            UnusedLongerThan(duration) => {
+                let ts = chrono::offset::Local::now()
+                    .checked_sub_signed(duration)
+                    .ok_or_else(|| {
+                        StoreError::ConstraintViolation(format!(
+                            "duration {} is too large",
+                            duration
+                        ))
+                    })?;
+                Ok(u::table
+                    .filter(u::removed_at.is_null())
+                    .filter(u::unused_at.lt(ts))
+                    .order_by(u::entity_count)
+                    .load(self.conn.as_ref())?)
+            }
         }
     }
 

store/postgres/src/subgraph_store.rs

@@ -125,11 +125,16 @@ pub trait DeploymentPlacer {
 
 /// Tools for managing unused deployments
 pub mod unused {
+    use graph::prelude::chrono::Duration;
+
     pub enum Filter {
         /// List all unused deployments
         All,
         /// List only deployments that are unused but have not been removed yet
         New,
+        /// List only deployments that were recorded as unused at least this
+        /// long ago but have not been removed at
+        UnusedLongerThan(Duration),
     }
 }