Batch Job: Add prune operation to clean cluster state. (#4193)

* Added batch_job_prune to interface and vine implementation.

* Connect makeflow garbage collection to batch_job_prune for cluster cleanup.

* Add stub prune option to remainder of batch systems

* format

---------

Co-authored-by: Benjamin Tovar <[email protected]>

Author: dthain

batch_job/src/batch_queue.c

@@ -44,6 +44,7 @@ static struct batch_queue_module batch_queue_unknown = {
 		NULL,
 		NULL,
 		NULL,
+		NULL,
 };
 
 #define BATCH_JOB_SYSTEMS "local, vine, wq, condor, uge (sge), pbs, lsf, torque, moab, slurm, amazon, k8s, flux, dryrun"
@@ -249,4 +250,9 @@ int batch_queue_remove(struct batch_queue *q, batch_queue_id_t jobid, batch_queu
 	return q->module->remove(q, jobid, mode);
 }
 
+int batch_queue_prune(struct batch_queue *q, const char *filename)
+{
+	return q->module->prune(q, filename);
+}
+
 /* vim: set noexpandtab tabstop=8: */

batch_job/src/batch_queue.h

@@ -141,6 +141,16 @@ You must still call @ref batch_queue_wait to wait for the removal to complete.
 */
 int batch_queue_remove(struct batch_queue *q, batch_queue_id_t jobid, batch_queue_remove_mode_t mode );
 
+/** Remove in-cluster cached data associated with filename.
+This call may be used to indicate that cached data associated with the given
+filename will no longer be used in the future, and may be freed.
+@param q The queue to prune from.
+@param filename The name of the file to be pruned.
+@return Greater than zero if the file exists and was pruned, zero otherwise.
+*/
+
+int batch_queue_prune(struct batch_queue *q, const char *filename );
+
 /** Converts a string into a batch queue type.
 @param str A string listing all of the known batch queue types (which changes over time.)
 @return The batch queue type corresponding to the string, or BATCH_QUEUE_TYPE_UNKNOWN if the string is invalid.

batch_job/src/batch_queue_amazon.c

@@ -827,6 +827,7 @@ static int batch_queue_amazon_create(struct batch_queue *q)
 batch_queue_stub_free(amazon);
 batch_queue_stub_port(amazon);
 batch_queue_stub_option_update(amazon);
+batch_queue_stub_prune(amazon);
 
 const struct batch_queue_module batch_queue_amazon = {
 		BATCH_QUEUE_TYPE_AMAZON,
@@ -840,4 +841,5 @@ const struct batch_queue_module batch_queue_amazon = {
 		batch_queue_amazon_submit,
 		batch_queue_amazon_wait,
 		batch_queue_amazon_remove,
+		batch_queue_amazon_prune,
 };

batch_job/src/batch_queue_cluster.c

@@ -515,6 +515,7 @@ static int batch_queue_cluster_create(struct batch_queue *q)
 batch_queue_stub_free(cluster);
 batch_queue_stub_port(cluster);
 batch_queue_stub_option_update(cluster);
+batch_queue_stub_prune(cluster);
 
 const struct batch_queue_module batch_queue_cluster = {
 		BATCH_QUEUE_TYPE_CLUSTER,
@@ -528,6 +529,7 @@ const struct batch_queue_module batch_queue_cluster = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 const struct batch_queue_module batch_queue_moab = {
@@ -542,6 +544,7 @@ const struct batch_queue_module batch_queue_moab = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 const struct batch_queue_module batch_queue_uge = {
@@ -556,6 +559,7 @@ const struct batch_queue_module batch_queue_uge = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 /* retained sge keyword for backwards compatibility after sge->uge name change. */
@@ -571,6 +575,7 @@ const struct batch_queue_module batch_queue_sge = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 const struct batch_queue_module batch_queue_pbs = {
@@ -585,6 +590,7 @@ const struct batch_queue_module batch_queue_pbs = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 const struct batch_queue_module batch_queue_lsf = {
@@ -599,6 +605,7 @@ const struct batch_queue_module batch_queue_lsf = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 const struct batch_queue_module batch_queue_torque = {
@@ -613,6 +620,7 @@ const struct batch_queue_module batch_queue_torque = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 const struct batch_queue_module batch_queue_slurm = {
@@ -627,6 +635,7 @@ const struct batch_queue_module batch_queue_slurm = {
 		batch_queue_cluster_submit,
 		batch_queue_cluster_wait,
 		batch_queue_cluster_remove,
+		batch_queue_cluster_prune,
 };
 
 /* vim: set noexpandtab tabstop=8: */

batch_job/src/batch_queue_condor.c

@@ -390,6 +390,7 @@ static int batch_queue_condor_create(struct batch_queue *q)
 batch_queue_stub_free(condor);
 batch_queue_stub_port(condor);
 batch_queue_stub_option_update(condor);
+batch_queue_stub_prune(condor);
 
 const struct batch_queue_module batch_queue_condor = {
 		BATCH_QUEUE_TYPE_CONDOR,
@@ -403,6 +404,7 @@ const struct batch_queue_module batch_queue_condor = {
 		batch_queue_condor_submit,
 		batch_queue_condor_wait,
 		batch_queue_condor_remove,
+		batch_queue_condor_prune,
 };
 
 /* vim: set noexpandtab tabstop=8: */

batch_job/src/batch_queue_dryrun.c

@@ -102,6 +102,7 @@ static int batch_queue_dryrun_create(struct batch_queue *q)
 batch_queue_stub_free(dryrun);
 batch_queue_stub_port(dryrun);
 batch_queue_stub_option_update(dryrun);
+batch_queue_stub_prune(dryrun);
 
 const struct batch_queue_module batch_queue_dryrun = {
 		BATCH_QUEUE_TYPE_DRYRUN,
@@ -115,6 +116,7 @@ const struct batch_queue_module batch_queue_dryrun = {
 		batch_queue_dryrun_submit,
 		batch_queue_dryrun_wait,
 		batch_queue_dryrun_remove,
+		batch_queue_dryrun_prune,
 };
 
 /* vim: set noexpandtab tabstop=8: */

batch_job/src/batch_queue_flux.c

@@ -360,6 +360,7 @@ static int batch_queue_flux_free(struct batch_queue *q)
 
 batch_queue_stub_port(flux);
 batch_queue_stub_option_update(flux);
+batch_queue_stub_prune(flux);
 
 const struct batch_queue_module batch_queue_flux = {
 		BATCH_QUEUE_TYPE_FLUX,
@@ -373,6 +374,7 @@ const struct batch_queue_module batch_queue_flux = {
 		batch_queue_flux_submit,
 		batch_queue_flux_wait,
 		batch_queue_flux_remove,
+		batch_queue_flux_prune,
 };
 
 /* vim: set noexpandtab tabstop=8: */

batch_job/src/batch_queue_internal.h

@@ -32,8 +32,8 @@ struct batch_queue_module {
 
 	batch_queue_id_t (*submit) (struct batch_queue *Q, struct batch_job *bt );
 	batch_queue_id_t (*wait) (struct batch_queue *Q, struct batch_job_info *info, time_t stoptime);
-
 	int (*remove) (struct batch_queue *Q, batch_queue_id_t id, batch_queue_remove_mode_t mode );
+	int (*prune) (struct batch_queue *Q, const char *filename);
 };
 
 struct batch_queue {
@@ -53,7 +53,7 @@ struct batch_queue {
 #define batch_queue_stub_free(name)  static int batch_queue_##name##_free (struct batch_queue *Q) { return 0; }
 #define batch_queue_stub_port(name)  static int batch_queue_##name##_port (struct batch_queue *Q) { return 0; }
 #define batch_queue_stub_option_update(name)  static void batch_queue_##name##_option_update (struct batch_queue *Q, const char *what, const char *value) { return; }
-
+#define batch_queue_stub_prune(name) static int batch_queue_##name##_prune (struct batch_queue *Q, const char *filename ) { return 0; }
 #endif
 
 /* vim: set noexpandtab tabstop=8: */

batch_job/src/batch_queue_k8s.c

@@ -694,6 +694,7 @@ static int batch_queue_k8s_free(struct batch_queue *q)
 
 batch_queue_stub_port(k8s);
 batch_queue_stub_option_update(k8s);
+batch_queue_stub_prune(k8s);
 
 const struct batch_queue_module batch_queue_k8s = {
 		BATCH_QUEUE_TYPE_K8S,
@@ -707,6 +708,7 @@ const struct batch_queue_module batch_queue_k8s = {
 		batch_queue_k8s_submit,
 		batch_queue_k8s_wait,
 		batch_queue_k8s_remove,
+		batch_queue_k8s_prune,
 };
 
 /* vim: set noexpandtab tabstop=8: */

batch_job/src/batch_queue_local.c

@@ -129,6 +129,7 @@ static int batch_queue_local_create(struct batch_queue *q)
 batch_queue_stub_free(local);
 batch_queue_stub_port(local);
 batch_queue_stub_option_update(local);
+batch_queue_stub_prune(local);
 
 const struct batch_queue_module batch_queue_local = {
 		BATCH_QUEUE_TYPE_LOCAL,
@@ -142,6 +143,7 @@ const struct batch_queue_module batch_queue_local = {
 		batch_queue_local_submit,
 		batch_queue_local_wait,
 		batch_queue_local_remove,
+		batch_queue_local_prune,
 };
 
 /* vim: set noexpandtab tabstop=8: */