diff --git a/src/bindings/python/flux/job/list.py b/src/bindings/python/flux/job/list.py index 4df18e1b09df..02d7083a698f 100644 --- a/src/bindings/python/flux/job/list.py +++ b/src/bindings/python/flux/job/list.py @@ -46,18 +46,31 @@ def job_list( name=None, queue=None, ): + # N.B. an "and" operation with no values returns everything + constraint = {"and": []} + if userid != flux.constants.FLUX_USERID_UNKNOWN: + constraint["and"].append({"userid": [userid]}) + if name: + constraint["and"].append({"name": [name]}) + if queue: + constraint["and"].append({"queue": [queue]}) + if states and results: + if states & flux.constants.FLUX_JOB_STATE_INACTIVE: + states &= ~flux.constants.FLUX_JOB_STATE_INACTIVE + tmp = {"or": []} + tmp["or"].append({"states": [states]}) + tmp["or"].append({"results": [results]}) + constraint["and"].append(tmp) + elif states: + constraint["and"].append({"states": [states]}) + elif results: + constraint["and"].append({"results": [results]}) payload = { "max_entries": int(max_entries), "attrs": attrs, - "userid": int(userid), - "states": states, - "results": results, "since": since, + "constraint": constraint, } - if name: - payload["name"] = name - if queue: - payload["queue"] = queue return JobListRPC(flux_handle, "job-list.list", payload) @@ -240,8 +253,6 @@ def add_filter(self, fname): if fname in self.STATES: self.states |= self.STATES[fname] elif fname in self.RESULTS: - # Must specify "inactive" to get results: - self.states |= self.STATES["inactive"] self.results |= self.RESULTS[fname] else: raise ValueError(f"Invalid filter specified: {fname}") diff --git a/src/cmd/flux-job.c b/src/cmd/flux-job.c index 8234d478788d..5fe73ca48a8d 100644 --- a/src/cmd/flux-job.c +++ b/src/cmd/flux-job.c @@ -1262,6 +1262,7 @@ int cmd_list (optparse_t *p, int argc, char **argv) json_t *value; uint32_t userid; int states = 0; + json_t *c; if (isatty (STDOUT_FILENO)) { fprintf (stderr, @@ -1290,16 +1291,20 @@ int cmd_list (optparse_t *p, int argc, char **argv) else userid = getuid (); + if (!(c = json_pack ("{ s:[ {s:[i]}, {s:[i]} ] }", + "and", + "userid", userid, + "states", states))) + log_msg_exit ("failed to construct constraint object"); + if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0, - "{s:i s:[s] s:i s:i s:i}", + "{s:i s:[s] s:o}", "max_entries", max_entries, "attrs", "all", - "userid", userid, - "states", states, - "results", 0))) + "constraint", c))) log_err_exit ("flux_rpc_pack"); if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0) log_err_exit ("flux job-list.list"); @@ -1327,6 +1332,7 @@ int cmd_list_inactive (optparse_t *p, int argc, char **argv) json_t *jobs; size_t index; json_t *value; + json_t *c; if (isatty (STDOUT_FILENO)) { fprintf (stderr, @@ -1341,17 +1347,18 @@ int cmd_list_inactive (optparse_t *p, int argc, char **argv) if (!(h = flux_open (NULL, 0))) log_err_exit ("flux_open"); + if (!(c = json_pack ("{s:[i]}", "states", FLUX_JOB_STATE_INACTIVE))) + log_msg_exit ("failed to construct constraint object"); + if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0, - "{s:i s:f s:i s:i s:i s:[s]}", + "{s:i s:f s:[s] s:o}", "max_entries", max_entries, "since", since, - "userid", FLUX_USERID_UNKNOWN, - "states", FLUX_JOB_STATE_INACTIVE, - "results", 0, - "attrs", "all"))) + "attrs", "all", + "constraint", c))) log_err_exit ("flux_rpc_pack"); if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0) log_err_exit ("flux job-list.list"); diff --git a/src/cmd/top/joblist_pane.c b/src/cmd/top/joblist_pane.c index 3a0642a74c09..937e71896ef1 100644 --- a/src/cmd/top/joblist_pane.c +++ b/src/cmd/top/joblist_pane.c @@ -357,11 +357,10 @@ void joblist_pane_query (struct joblist_pane *joblist) "job-list.list", 0, 0, - "{s:i s:i s:i s:i s:[s,s,s,s,s,s,s,s]}", + "{s:i s:{s:[i]} s:[s,s,s,s,s,s,s,s]}", "max_entries", win_dim.y_length - 1, - "userid", FLUX_USERID_UNKNOWN, + "constraint", "states", FLUX_JOB_STATE_RUNNING, - "results", 0, "attrs", "annotations", "userid", diff --git a/src/common/libjob/list.c b/src/common/libjob/list.c index 30eb118f8927..b33b8fb8b67b 100644 --- a/src/common/libjob/list.c +++ b/src/common/libjob/list.c @@ -25,6 +25,7 @@ flux_future_t *flux_job_list (flux_t *h, { flux_future_t *f; json_t *o = NULL; + json_t *c = NULL; int valid_states = (FLUX_JOB_STATE_PENDING | FLUX_JOB_STATE_RUNNING | FLUX_JOB_STATE_INACTIVE); @@ -37,15 +38,22 @@ flux_future_t *flux_job_list (flux_t *h, errno = EINVAL; return NULL; } + if (!(c = json_pack ("{ s:[ {s:[i]}, {s:[i]} ] }", + "and", + "userid", userid, + "states", states ? states : valid_states))) { + json_decref (o); + errno = ENOMEM; + return NULL; + } if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0, - "{s:i s:o s:i s:i s:i}", + "{s:i s:o s:o}", "max_entries", max_entries, "attrs", o, - "userid", userid, - "states", states, - "results", 0))) { + "constraint", c))) { saved_errno = errno; json_decref (o); + json_decref (c); errno = saved_errno; return NULL; } @@ -59,6 +67,7 @@ flux_future_t *flux_job_list_inactive (flux_t *h, { flux_future_t *f; json_t *o = NULL; + json_t *c = NULL; int saved_errno; if (!h || max_entries < 0 || since < 0. || !attrs_json_str @@ -66,16 +75,20 @@ flux_future_t *flux_job_list_inactive (flux_t *h, errno = EINVAL; return NULL; } + if (!(c = json_pack ("{s:[i]}", "states", FLUX_JOB_STATE_INACTIVE))) { + json_decref (o); + errno = ENOMEM; + return NULL; + } if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0, - "{s:i s:f s:i s:i s:i s:o}", + "{s:i s:f s:o s:o}", "max_entries", max_entries, "since", since, - "userid", FLUX_USERID_UNKNOWN, - "states", FLUX_JOB_STATE_INACTIVE, - "results", 0, - "attrs", o))) { + "attrs", o, + "constraint", c))) { saved_errno = errno; json_decref (o); + json_decref (c); errno = saved_errno; return NULL; } diff --git a/src/common/libjob/state.c b/src/common/libjob/state.c index 28bbad838332..c27858a2243f 100644 --- a/src/common/libjob/state.c +++ b/src/common/libjob/state.c @@ -26,6 +26,9 @@ static struct strtab states[] = { { FLUX_JOB_STATE_RUN, "RUN", "run", "R", "r" }, { FLUX_JOB_STATE_CLEANUP, "CLEANUP", "cleanup", "C", "c" }, { FLUX_JOB_STATE_INACTIVE, "INACTIVE", "inactive", "I", "i" }, + { FLUX_JOB_STATE_PENDING, "PENDING", "pending", "PD", "pd" }, + { FLUX_JOB_STATE_RUNNING, "RUNNING", "running", "RU", "ru" }, + { FLUX_JOB_STATE_ACTIVE, "ACTIVE", "active", "A", "a" }, }; diff --git a/src/common/libjob/test/job.c b/src/common/libjob/test/job.c index 29f0d07f2ee2..db07c296d137 100644 --- a/src/common/libjob/test/job.c +++ b/src/common/libjob/test/job.c @@ -241,6 +241,9 @@ struct ss sstab[] = { { FLUX_JOB_STATE_RUN, "R", "RUN", "r", "run" }, { FLUX_JOB_STATE_CLEANUP, "C", "CLEANUP", "c", "cleanup" }, { FLUX_JOB_STATE_INACTIVE, "I", "INACTIVE", "i", "inactive" }, + { FLUX_JOB_STATE_PENDING, "PD", "PENDING", "pd", "pending" }, + { FLUX_JOB_STATE_RUNNING, "RU", "RUNNING", "ru", "running" }, + { FLUX_JOB_STATE_ACTIVE, "A", "ACTIVE", "a", "active" }, { -1, NULL, NULL, NULL, NULL }, }; diff --git a/src/modules/job-archive/job-archive.c b/src/modules/job-archive/job-archive.c index 3d3d18437f91..22d71c7d9f65 100644 --- a/src/modules/job-archive/job-archive.c +++ b/src/modules/job-archive/job-archive.c @@ -506,19 +506,18 @@ void job_archive_cb (flux_reactor_t *r, "job-list.list", FLUX_NODEID_ANY, 0, - "{s:i s:f s:i s:i s:i s:[ssssss]}", + "{s:i s:f s:[ssssss] s:{s:[i]}}", "max_entries", 0, "since", ctx->since, - "userid", FLUX_USERID_UNKNOWN, - "states", FLUX_JOB_STATE_INACTIVE, - "results", 0, "attrs", "userid", "ranks", "t_submit", "t_run", "t_cleanup", - "t_inactive"))) { + "t_inactive", + "constraint", + "states", FLUX_JOB_STATE_INACTIVE))) { flux_log_error (ctx->h, "%s: flux_rpc_pack", __FUNCTION__); return; } diff --git a/src/modules/job-list/Makefile.am b/src/modules/job-list/Makefile.am index a4dc11524ea9..6e01d49f5c24 100644 --- a/src/modules/job-list/Makefile.am +++ b/src/modules/job-list/Makefile.am @@ -29,10 +29,18 @@ libjob_list_la_SOURCES = \ idsync.h \ idsync.c \ stats.h \ - stats.c + stats.c \ + match.h \ + match.c \ + state_match.h \ + state_match.c \ + match_util.h \ + match_util.c TESTS = \ - test_job_data.t + test_job_data.t \ + test_match.t \ + test_state_match.t test_ldadd = \ $(builddir)/libjob-list.la \ @@ -65,6 +73,22 @@ test_job_data_t_LDADD = \ test_job_data_t_LDFLAGS = \ $(test_ldflags) +test_match_t_SOURCES = test/match.c +test_match_t_CPPFLAGS = \ + $(test_cppflags) +test_match_t_LDADD = \ + $(test_ldadd) +test_match_t_LDFLAGS = \ + $(test_ldflags) + +test_state_match_t_SOURCES = test/state_match.c +test_state_match_t_CPPFLAGS = \ + $(test_cppflags) +test_state_match_t_LDADD = \ + $(test_ldadd) +test_state_match_t_LDFLAGS = \ + $(test_ldflags) + EXTRA_DIST = \ test/R/1node_1core.R \ test/R/1node_4core.R \ diff --git a/src/modules/job-list/list.c b/src/modules/job-list/list.c index 01809fa55b01..97d495f3fc40 100644 --- a/src/modules/job-list/list.c +++ b/src/modules/job-list/list.c @@ -26,6 +26,8 @@ #include "list.h" #include "job_util.h" #include "job_data.h" +#include "match.h" +#include "state_match.h" json_t *get_job_by_id (struct job_state_ctx *jsctx, job_list_error_t *errp, @@ -35,28 +37,6 @@ json_t *get_job_by_id (struct job_state_ctx *jsctx, flux_job_state_t state, bool *stall); -/* Filter test to determine if job desired by caller */ -bool job_filter (struct job *job, - uint32_t userid, - int states, - int results, - const char *name, - const char *queue) -{ - if (name && (!job->name || !streq (job->name, name))) - return false; - if (queue && (!job->queue || !streq (job->queue, queue))) - return false; - if (!(job->state & states)) - return false; - if (userid != FLUX_USERID_UNKNOWN && job->userid != userid) - return false; - if (job->state & FLUX_JOB_STATE_INACTIVE - && !(job->result & results)) - return false; - return true; -} - /* Put jobs from list onto jobs array, breaking if max_entries has * been reached. Returns 1 if jobs array is full, 0 if continue, -1 * one error with errno set: @@ -68,12 +48,8 @@ int get_jobs_from_list (json_t *jobs, zlistx_t *list, int max_entries, json_t *attrs, - uint32_t userid, - int states, - int results, double since, - const char *name, - const char *queue) + struct list_constraint *c) { struct job *job; @@ -91,7 +67,7 @@ int get_jobs_from_list (json_t *jobs, if (job->t_inactive > 0. && job->t_inactive <= since) break; - if (job_filter (job, userid, states, results, name, queue)) { + if (job_match (job, c)) { json_t *o; if (!(o = job_to_json (job, attrs, errp))) return -1; @@ -122,11 +98,8 @@ json_t *get_jobs (struct job_state_ctx *jsctx, int max_entries, double since, json_t *attrs, - uint32_t userid, - int states, - int results, - const char *name, - const char *queue) + struct list_constraint *c, + struct state_constraint *statec) { json_t *jobs = NULL; int saved_errno; @@ -138,51 +111,39 @@ json_t *get_jobs (struct job_state_ctx *jsctx, /* We return jobs in the following order, pending, running, * inactive */ - if (states & FLUX_JOB_STATE_PENDING) { + if (state_match (FLUX_JOB_STATE_PENDING, statec)) { if ((ret = get_jobs_from_list (jobs, errp, jsctx->pending, max_entries, attrs, - userid, - states, - results, 0., - name, - queue)) < 0) + c)) < 0) goto error; } - if (states & FLUX_JOB_STATE_RUNNING) { + if (state_match (FLUX_JOB_STATE_RUNNING, statec)) { if (!ret) { if ((ret = get_jobs_from_list (jobs, errp, jsctx->running, max_entries, attrs, - userid, - states, - results, 0., - name, - queue)) < 0) + c)) < 0) goto error; } } - if (states & FLUX_JOB_STATE_INACTIVE) { + if (state_match (FLUX_JOB_STATE_INACTIVE, statec)) { if (!ret) { if ((ret = get_jobs_from_list (jobs, errp, jsctx->inactive, max_entries, attrs, - userid, - states, - results, since, - name, - queue)) < 0) + c)) < 0) goto error; } } @@ -207,21 +168,16 @@ void list_cb (flux_t *h, flux_msg_handler_t *mh, json_t *attrs; int max_entries; double since = 0.; - uint32_t userid; - int states; - int results; - const char *name = NULL; - const char *queue = NULL; + json_t *constraint = NULL; + struct list_constraint *c = NULL; + struct state_constraint *statec = NULL; + flux_error_t error; - if (flux_request_unpack (msg, NULL, "{s:i s:o s:i s:i s:i s?F s?s s?s}", + if (flux_request_unpack (msg, NULL, "{s:i s:o s?F s?o}", "max_entries", &max_entries, "attrs", &attrs, - "userid", &userid, - "states", &states, - "results", &results, "since", &since, - "name", &name, - "queue", &queue) < 0) { + "constraint", &constraint) < 0) { seterror (&err, "invalid payload: %s", flux_msg_last_error (msg)); errno = EPROTO; goto error; @@ -241,32 +197,38 @@ void list_cb (flux_t *h, flux_msg_handler_t *mh, errno = EPROTO; goto error; } - /* If user sets no states, assume they want all information */ - if (!states) - states = (FLUX_JOB_STATE_PENDING - | FLUX_JOB_STATE_RUNNING - | FLUX_JOB_STATE_INACTIVE); - - /* If user sets no results, assume they want all information */ - if (!results) - results = (FLUX_JOB_RESULT_COMPLETED - | FLUX_JOB_RESULT_FAILED - | FLUX_JOB_RESULT_CANCELED - | FLUX_JOB_RESULT_TIMEOUT); + if (!(c = list_constraint_create (constraint, &error))) { + seterror (&err, + "invalid payload: constraint object invalid: %s", + error.text); + errno = EPROTO; + goto error; + } + if (!(statec = state_constraint_create (constraint, &error))) { + seterror (&err, + "invalid payload: constraint object invalid: %s", + error.text); + errno = EPROTO; + goto error; + } if (!(jobs = get_jobs (ctx->jsctx, &err, max_entries, since, - attrs, userid, states, results, name, queue))) + attrs, c, statec))) goto error; if (flux_respond_pack (h, msg, "{s:O}", "jobs", jobs) < 0) flux_log_error (h, "%s: flux_respond_pack", __FUNCTION__); json_decref (jobs); + list_constraint_destroy (c); + state_constraint_destroy (statec); return; error: if (flux_respond_error (h, msg, errno, err.text) < 0) flux_log_error (h, "%s: flux_respond_error", __FUNCTION__); + list_constraint_destroy (c); + state_constraint_destroy (statec); } void check_id_valid_continuation (flux_future_t *f, void *arg) diff --git a/src/modules/job-list/match.c b/src/modules/job-list/match.c new file mode 100644 index 000000000000..95a502639c9e --- /dev/null +++ b/src/modules/job-list/match.c @@ -0,0 +1,598 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "src/common/libutil/errprintf.h" +#include "ccan/str/str.h" + +#include "match.h" +#include "match_util.h" + +typedef bool (*match_f) (struct list_constraint *, const struct job *); + +struct list_constraint { + zlistx_t *values; + match_f match; +}; + +typedef enum { + MATCH_T_SUBMIT = 1, + MATCH_T_DEPEND = 2, + MATCH_T_RUN = 3, + MATCH_T_CLEANUP = 4, + MATCH_T_INACTIVE = 5, +} match_timestamp_type_t; + +typedef enum { + MATCH_GREATER_THAN_EQUAL = 1, + MATCH_LESS_THAN_EQUAL = 2, + MATCH_GREATER_THAN = 3, + MATCH_LESS_THAN = 4, +} match_comparison_t; + +struct timestamp_value { + double t_value; + match_timestamp_type_t t_type; + match_comparison_t t_comp; +}; + +static void timestamp_value_destroy (void *data) +{ + if (data) { + int save_errno = errno; + free (data); + errno = save_errno; + } +} + +/* zlistx_set_destructor */ +static void wrap_timestamp_value_destroy (void **item) +{ + if (item) { + struct timestamp_value *tv = *item; + timestamp_value_destroy (tv); + (*item) = NULL; + } +} + +static struct timestamp_value *timestamp_value_create ( + double t_value, + const char *type, + match_comparison_t comp) +{ + struct timestamp_value *tv; + + if (!(tv = calloc (1, sizeof (*tv)))) + return NULL; + tv->t_value = t_value; + + if (streq (type, "t_submit")) + tv->t_type = MATCH_T_SUBMIT; + else if (streq (type, "t_depend")) + tv->t_type = MATCH_T_DEPEND; + else if (streq (type, "t_run")) + tv->t_type = MATCH_T_RUN; + else if (streq (type, "t_cleanup")) + tv->t_type = MATCH_T_CLEANUP; + else if (streq (type, "t_inactive")) + tv->t_type = MATCH_T_INACTIVE; + else + goto cleanup; + + tv->t_comp = comp; + return tv; + +cleanup: + timestamp_value_destroy (tv); + return NULL; +} + +static struct timestamp_value *timestamp_value_create_str ( + const char *t_value, + const char *type, + match_comparison_t comp, + flux_error_t *errp) +{ + double t; + char *endptr; + errno = 0; + t = strtod (t_value, &endptr); + if (errno != 0 || *endptr != '\0') { + errprintf (errp, "Invalid timestamp value specified"); + return NULL; + } + if (t < 0.0) { + errprintf (errp, "timestamp value must be >= 0.0"); + return NULL; + } + return timestamp_value_create (t, type, comp); +} + +static bool match_true (struct list_constraint *c, const struct job *job) +{ + return true; +} + +static struct list_constraint *list_constraint_new (match_f match_cb, + destructor_f destructor_cb, + flux_error_t *errp) +{ + struct list_constraint *c; + if (!(c = calloc (1, sizeof (*c))) + || !(c->values = zlistx_new ())) { + list_constraint_destroy (c); + errprintf (errp, "Out of memory"); + return NULL; + } + c->match = match_cb; + if (destructor_cb) + zlistx_set_destructor (c->values, destructor_cb); + return c; +} + +static void list_constraint_destructor (void **item) +{ + if (item) { + list_constraint_destroy (*item); + *item = NULL; + } +} + +/* zlistx_set_destructor */ +static void wrap_free (void **item) +{ + if (item) { + free (*item); + (*item) = NULL; + } +} + +static bool match_userid (struct list_constraint *c, + const struct job *job) +{ + uint32_t *userid = zlistx_first (c->values); + while (userid) { + if ((*userid) == FLUX_USERID_UNKNOWN) + return true; + if ((*userid) == job->userid) + return true; + userid = zlistx_next (c->values); + } + return false; +} + +static struct list_constraint *create_userid_constraint (json_t *values, + flux_error_t *errp) +{ + struct list_constraint *c; + json_t *entry; + size_t index; + + if (!(c = list_constraint_new (match_userid, wrap_free, errp))) + return NULL; + json_array_foreach (values, index, entry) { + uint32_t *userid; + if (!json_is_integer (entry)) { + errprintf (errp, "userid value must be an integer"); + goto error; + } + if (!(userid = malloc (sizeof (*userid)))) + goto error; + (*userid) = json_integer_value (entry); + if (!zlistx_add_end (c->values, userid)) { + free (userid); + goto error; + } + } + return c; + error: + list_constraint_destroy (c); + return NULL; +} + +static struct list_constraint *create_string_constraint (const char *op, + json_t *values, + match_f match_cb, + flux_error_t *errp) +{ + struct list_constraint *c; + json_t *entry; + size_t index; + + if (!(c = list_constraint_new (match_cb, wrap_free, errp))) + return NULL; + json_array_foreach (values, index, entry) { + char *s = NULL; + if (!json_is_string (entry)) { + errprintf (errp, "%s value must be a string", op); + goto error; + } + if (!(s = strdup (json_string_value (entry)))) + return NULL; + if (!zlistx_add_end (c->values, s)) { + free (s); + goto error; + } + } + return c; + error: + list_constraint_destroy (c); + return NULL; +} + +static bool match_name (struct list_constraint *c, const struct job *job) +{ + const char *name = zlistx_first (c->values); + while (name) { + if (job->name && streq (name, job->name)) + return true; + name = zlistx_next (c->values); + } + return false; +} + +static struct list_constraint *create_name_constraint (json_t *values, + flux_error_t *errp) +{ + return create_string_constraint ("name", values, match_name, errp); +} + +static bool match_queue (struct list_constraint *c, const struct job *job) +{ + const char *queue = zlistx_first (c->values); + while (queue) { + if (job->queue && streq (queue, job->queue)) + return true; + queue = zlistx_next (c->values); + } + return false; +} + +static struct list_constraint *create_queue_constraint (json_t *values, + flux_error_t *errp) +{ + return create_string_constraint ("queue", values, match_queue, errp); +} + +static struct list_constraint *create_bitmask_constraint ( + const char *op, + json_t *values, + match_f match_cb, + array_to_bitmask_f array_to_bitmask_cb, + flux_error_t *errp) +{ + struct list_constraint *c; + int *bitmask = NULL; + int tmp; + if ((tmp = array_to_bitmask_cb (values, errp)) < 0) + return NULL; + if (!(bitmask = malloc (sizeof (*bitmask)))) + return NULL; + (*bitmask) = tmp; + if (!(c = list_constraint_new (match_cb, wrap_free, errp)) + || !zlistx_add_end (c->values, bitmask)) { + list_constraint_destroy (c); + free (bitmask); + return NULL; + } + return c; +} + +static bool match_states (struct list_constraint *c, const struct job *job) +{ + int *states = zlistx_first (c->values); + return ((*states) & job->state); +} + +static struct list_constraint *create_states_constraint (json_t *values, + flux_error_t *errp) +{ + return create_bitmask_constraint ("states", + values, + match_states, + array_to_states_bitmask, + errp); +} + +static bool match_results (struct list_constraint *c, + const struct job *job) +{ + int *results = zlistx_first (c->values); + if (job->state != FLUX_JOB_STATE_INACTIVE) + return false; + return ((*results) & job->result); +} + +static int array_to_results_bitmask (json_t *values, flux_error_t *errp) +{ + int results = 0; + json_t *entry; + size_t index; + int valid_results = (FLUX_JOB_RESULT_COMPLETED + | FLUX_JOB_RESULT_FAILED + | FLUX_JOB_RESULT_CANCELED + | FLUX_JOB_RESULT_TIMEOUT); + + json_array_foreach (values, index, entry) { + flux_job_result_t result; + if (json_is_string (entry)) { + const char *resultstr = json_string_value (entry); + if (flux_job_strtoresult (resultstr, &result) < 0) { + errprintf (errp, + "invalid results value '%s' specified", + resultstr); + return -1; + } + } + else if (json_is_integer (entry)) { + result = json_integer_value (entry); + if (result & ~valid_results) { + errprintf (errp, + "invalid results value '%Xh' specified", + result); + return -1; + } + } + else { + errprintf (errp, "results value invalid type"); + return -1; + } + results |= result; + } + return results; +} + +static struct list_constraint *create_results_constraint (json_t *values, + flux_error_t *errp) +{ + return create_bitmask_constraint ("results", + values, + match_results, + array_to_results_bitmask, + errp); +} + +static bool match_timestamp (struct list_constraint *c, + const struct job *job) +{ + struct timestamp_value *tv = zlistx_first (c->values); + double t; + + if (tv->t_type == MATCH_T_SUBMIT) + t = job->t_submit; + else if (tv->t_type == MATCH_T_DEPEND) { + /* if submit_version < 1, it means it was not set. This is + * before the introduction of event `validate` after 0.41.1. + * Before the introduction of this event, t_submit and + * t_depend are the same. + */ + if (job->submit_version < 1) + t = job->t_submit; + else if (job->states_mask & FLUX_JOB_STATE_DEPEND) + t = job->t_depend; + else + return false; + } + else if (tv->t_type == MATCH_T_RUN + && (job->states_mask & FLUX_JOB_STATE_RUN)) + t = job->t_run; + else if (tv->t_type == MATCH_T_CLEANUP + && (job->states_mask & FLUX_JOB_STATE_CLEANUP)) + t = job->t_cleanup; + else if (tv->t_type == MATCH_T_INACTIVE + && (job->states_mask & FLUX_JOB_STATE_INACTIVE)) + t = job->t_inactive; + else + return false; + + if (tv->t_comp == MATCH_GREATER_THAN_EQUAL) + return t >= tv->t_value; + else if (tv->t_comp == MATCH_LESS_THAN_EQUAL) + return t <= tv->t_value; + else if (tv->t_comp == MATCH_GREATER_THAN) + return t > tv->t_value; + else /* tv->t_comp == MATCH_LESS_THAN */ + return t < tv->t_value; +} + +static struct list_constraint *create_timestamp_constraint (const char *type, + json_t *values, + flux_error_t *errp) +{ + struct timestamp_value *tv = NULL; + struct list_constraint *c; + const char *str; + json_t *v = json_array_get (values, 0); + + if (!v) { + errprintf (errp, "timestamp value not specified"); + return NULL; + } + if (!json_is_string (v)) { + errprintf (errp, "%s value must be a string", type); + return NULL; + } + str = json_string_value (v); + if (strstarts (str, ">=")) + tv = timestamp_value_create_str (str + 2, + type, + MATCH_GREATER_THAN_EQUAL, + errp); + else if (strstarts (str, "<=")) + tv = timestamp_value_create_str (str + 2, + type, + MATCH_LESS_THAN_EQUAL, + errp); + else if (strstarts (str, ">")) + tv = timestamp_value_create_str (str + 1, + type, + MATCH_GREATER_THAN, + errp); + else if (strstarts (str, "<")) + tv = timestamp_value_create_str (str + 1, + type, + MATCH_LESS_THAN, + errp); + else + errprintf (errp, "timestamp comparison operator not specified"); + + if (!tv) + return NULL; + + if (!(c = list_constraint_new (match_timestamp, + wrap_timestamp_value_destroy, + errp)) + || !zlistx_add_end (c->values, tv)) { + list_constraint_destroy (c); + timestamp_value_destroy (tv); + return NULL; + } + return c; +} + +static bool match_and (struct list_constraint *c, const struct job *job) +{ + struct list_constraint *cp = zlistx_first (c->values); + while (cp) { + if (!cp->match (cp, job)) + return false; + cp = zlistx_next (c->values); + } + return true; +} + +static bool match_or (struct list_constraint *c, const struct job *job) +{ + struct list_constraint *cp = zlistx_first (c->values); + /* no values in "or" defined as true per RFC31 */ + if (!cp) + return true; + while (cp) { + if (cp->match (cp, job)) + return true; + cp = zlistx_next (c->values); + } + return false; +} + +static bool match_not (struct list_constraint *c, const struct job *job) +{ + return !match_and (c, job); +} + +static struct list_constraint *conditional_constraint (const char *type, + json_t *values, + flux_error_t *errp) +{ + json_t *entry; + size_t index; + struct list_constraint *c; + match_f match_cb; + + if (streq (type, "and")) + match_cb = match_and; + else if (streq (type, "or")) + match_cb = match_or; + else /* streq (type, "not") */ + match_cb = match_not; + + if (!(c = list_constraint_new (match_cb, list_constraint_destructor, errp))) + return NULL; + + json_array_foreach (values, index, entry) { + struct list_constraint *cp = list_constraint_create (entry, errp); + if (!cp) + goto error; + if (!zlistx_add_end (c->values, cp)) { + errprintf (errp, "Out of memory"); + list_constraint_destroy (cp); + goto error; + } + } + return c; + + error: + list_constraint_destroy (c); + return NULL; +} + +void list_constraint_destroy (struct list_constraint *constraint) +{ + if (constraint) { + int saved_errno = errno; + zlistx_destroy (&constraint->values); + free (constraint); + errno = saved_errno; + } +} + +struct list_constraint *list_constraint_create (json_t *constraint, flux_error_t *errp) +{ + const char *op; + json_t *values; + + if (constraint) { + if (!json_is_object (constraint)) { + errprintf (errp, "constraint must be JSON object"); + return NULL; + } + if (json_object_size (constraint) > 1) { + errprintf (errp, "constraint must only contain 1 element"); + return NULL; + } + json_object_foreach (constraint, op, values) { + if (!json_is_array (values)) { + errprintf (errp, "operator %s values not an array", op); + return NULL; + } + if (streq (op, "userid")) + return create_userid_constraint (values, errp); + else if (streq (op, "name")) + return create_name_constraint (values, errp); + else if (streq (op, "queue")) + return create_queue_constraint (values, errp); + else if (streq (op, "states")) + return create_states_constraint (values, errp); + else if (streq (op, "results")) + return create_results_constraint (values, errp); + else if (streq (op, "t_submit") + || streq (op, "t_depend") + || streq (op, "t_run") + || streq (op, "t_cleanup") + || streq (op, "t_inactive")) + return create_timestamp_constraint (op, values, errp); + else if (streq (op, "or") || streq (op, "and") || streq (op, "not")) + return conditional_constraint (op, values, errp); + else { + errprintf (errp, "unknown constraint operator: %s", op); + return NULL; + } + } + } + return list_constraint_new (match_true, NULL, errp); +} + +bool job_match (const struct job *job, struct list_constraint *constraint) +{ + if (!job || !constraint) + return false; + return constraint->match (constraint, job); +} + +/* vi: ts=4 sw=4 expandtab + */ diff --git a/src/modules/job-list/match.h b/src/modules/job-list/match.h new file mode 100644 index 000000000000..f6f9a552702c --- /dev/null +++ b/src/modules/job-list/match.h @@ -0,0 +1,39 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#ifndef HAVE_JOB_LIST_MATCH_H +#define HAVE_JOB_LIST_MATCH_H 1 + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include /* flux_error_t */ +#include + +#include "job_data.h" + +/* Load and validate RFC 31 constraint spec 'constraint'. 'constraint' + * can be NULL to indicate a constraint that matches everything. + * + * Returns a list constraint object if constraint is valid spec, + * Returns NULL with error in errp if errp != NULL. + */ +struct list_constraint *list_constraint_create (json_t *constraint, + flux_error_t *errp); + +void list_constraint_destroy (struct list_constraint *constraint); + +/* Return true if job matches constraints in RFC 31 constraint + * specification 'constraint'. + */ +bool job_match (const struct job *job, struct list_constraint *constraint); + +#endif /* !HAVE_JOB_LIST_MATCH_H */ diff --git a/src/modules/job-list/match_util.c b/src/modules/job-list/match_util.c new file mode 100644 index 000000000000..e7a0a6f36b2d --- /dev/null +++ b/src/modules/job-list/match_util.c @@ -0,0 +1,64 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "src/common/libutil/errprintf.h" + +#include "match_util.h" + +int array_to_states_bitmask (json_t *values, flux_error_t *errp) +{ + int states = 0; + json_t *entry; + size_t index; + int valid_states = (FLUX_JOB_STATE_NEW + | FLUX_JOB_STATE_PENDING + | FLUX_JOB_STATE_RUNNING + | FLUX_JOB_STATE_INACTIVE); + + json_array_foreach (values, index, entry) { + flux_job_state_t state; + if (json_is_string (entry)) { + const char *statestr = json_string_value (entry); + if (flux_job_strtostate (statestr, &state) < 0) { + errprintf (errp, + "invalid states value '%s' specified", + statestr); + return -1; + } + } + else if (json_is_integer (entry)) { + state = json_integer_value (entry); + if (state & ~valid_states) { + errprintf (errp, + "invalid states value '%Xh' specified", + state); + return -1; + } + } + else { + errprintf (errp, "states value invalid type"); + return -1; + } + states |= state; + } + return states; +} + +/* vi: ts=4 sw=4 expandtab + */ diff --git a/src/modules/job-list/match_util.h b/src/modules/job-list/match_util.h new file mode 100644 index 000000000000..59dbcd9c500c --- /dev/null +++ b/src/modules/job-list/match_util.h @@ -0,0 +1,29 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#ifndef HAVE_JOB_LIST_MATCH_UTIL_H +#define HAVE_JOB_LIST_MATCH_UTIL_H 1 + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include /* flux_error_t */ +#include + +/* identical to czmq_destructor, czmq.h happens to define a + * conflicting symbol we use */ +typedef void (destructor_f) (void **item); + +typedef int (*array_to_bitmask_f) (json_t *, flux_error_t *); + +int array_to_states_bitmask (json_t *values, flux_error_t *errp); + +#endif /* !HAVE_JOB_LIST_MATCH_UTIL_H */ diff --git a/src/modules/job-list/state_match.c b/src/modules/job-list/state_match.c new file mode 100644 index 000000000000..ed3b7b922bed --- /dev/null +++ b/src/modules/job-list/state_match.c @@ -0,0 +1,427 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "src/common/libutil/errprintf.h" +#include "ccan/str/str.h" + +#include "state_match.h" +#include "match_util.h" + +/* MATCH_ALWAYS - constraint always matches job in state X + * MATCH_MAYBE - constraint maybe matches job in state X + * MATCH_NEVER - constraint never matches job in state X + * + * examples: + * + * states=depend + * + * This constraint ALWAYS matches a job in state depend and NEVER matches + * a job in any other job state. + * + * userid=42 + * + * This constraint MAYBE matches a job in job state X because + * the job state does not matter, it depends on the userid. + * + * NOT (userid=42) + * + * This constraint MAYBE matches a job in job state X because again, + * it depends on the userid. The NOT of a MAYBE is still MAYBE. + * + * (states=depend OR userid=42) + * + * This constraint ALWAYS matches a job in state depend, but MAYBE matches + * a job in any other job state, since it depends on the userid. + * + * (states=depend AND userid=42) + * + * This constraint MAYBE matches a job state in state depend, because + * it depends on the userid. It NEVER matches a job in any other + * state. + * + */ +typedef enum { + MATCH_NOTSET, + MATCH_ALWAYS, + MATCH_MAYBE, + MATCH_NEVER, +} state_match_t; + +typedef state_match_t (*match_f) (struct state_constraint *, + flux_job_state_t state); + +struct state_constraint { + zlistx_t *values; + match_f match; +}; + +static state_match_t match_always (struct state_constraint *c, flux_job_state_t state) +{ + return MATCH_ALWAYS; +} + +static state_match_t match_maybe (struct state_constraint *c, flux_job_state_t state) +{ + return MATCH_MAYBE; +} + +static struct state_constraint *state_constraint_new (match_f match_cb, + destructor_f destructor_cb, + flux_error_t *errp) +{ + struct state_constraint *c; + if (!(c = calloc (1, sizeof (*c))) + || !(c->values = zlistx_new ())) { + state_constraint_destroy (c); + errprintf (errp, "Out of memory"); + return NULL; + } + c->match = match_cb; + if (destructor_cb) + zlistx_set_destructor (c->values, destructor_cb); + return c; +} + +static void state_constraint_destructor (void **item) +{ + if (item) { + state_constraint_destroy (*item); + *item = NULL; + } +} + +/* zlistx_set_destructor */ +static void wrap_free (void **item) +{ + if (item) { + free (*item); + (*item) = NULL; + } +} + +static state_match_t match_states (struct state_constraint *c, + flux_job_state_t state) +{ + int *states = zlistx_first (c->values); + if ((*states) & state) + return MATCH_ALWAYS; + return MATCH_NEVER; +} + +static struct state_constraint *create_states_constraint (json_t *values, + flux_error_t *errp) +{ + struct state_constraint *c; + int *bitmask = NULL; + int tmp; + if ((tmp = array_to_states_bitmask (values, errp)) < 0) + return NULL; + /* if no states specified, returns MATCH_ALWAYS */ + if (!tmp) + return state_constraint_new (match_always, NULL, errp); + if (!(bitmask = malloc (sizeof (*bitmask)))) + return NULL; + (*bitmask) = tmp; + if (!(c = state_constraint_new (match_states, wrap_free, errp)) + || !zlistx_add_end (c->values, bitmask)) { + state_constraint_destroy (c); + free (bitmask); + return NULL; + } + return c; +} + +static state_match_t match_result (struct state_constraint *c, flux_job_state_t state) +{ + if (state != FLUX_JOB_STATE_INACTIVE) + return MATCH_NEVER; + return MATCH_MAYBE; +} + +/* N.B. Not all job states can be reached, e.g. a pending job is + * canceled, so it never reaches the RUN state. That is still handled + * here in this logic. e.g. a constraint on `t_run` can MAYBE pass if + * the job state is INACTIVE. We don't know if `t_run` was ever set, + * but since it can MAYBE be set, we must check. + */ +static state_match_t match_t_submit (struct state_constraint *c, + flux_job_state_t state) +{ + return MATCH_MAYBE; +} + +static state_match_t match_t_depend (struct state_constraint *c, + flux_job_state_t state) +{ + if (state >= FLUX_JOB_STATE_DEPEND) + return MATCH_MAYBE; + return MATCH_NEVER; +} + +static state_match_t match_t_run (struct state_constraint *c, + flux_job_state_t state) +{ + if (state >= FLUX_JOB_STATE_RUN) + return MATCH_MAYBE; + return MATCH_NEVER; +} + +static state_match_t match_t_cleanup (struct state_constraint *c, + flux_job_state_t state) +{ + if (state >= FLUX_JOB_STATE_CLEANUP) + return MATCH_MAYBE; + return MATCH_NEVER; +} + +static state_match_t match_t_inactive (struct state_constraint *c, + flux_job_state_t state) +{ + if (state == FLUX_JOB_STATE_INACTIVE) + return MATCH_MAYBE; + return MATCH_NEVER; +} + +static struct state_constraint *create_timestamp_constraint (const char *type, + flux_error_t *errp) +{ + struct state_constraint *c; + match_f cb; + + if (streq (type, "t_submit")) + cb = match_t_submit; + else if (streq (type, "t_depend")) + cb = match_t_depend; + else if (streq (type, "t_run")) + cb = match_t_run; + else if (streq (type, "t_cleanup")) + cb = match_t_cleanup; + else /* streq (type, "t_inactive") */ + cb = match_t_inactive; + + if (!(c = state_constraint_new (cb, NULL, errp))) + return NULL; + return c; +} + +static state_match_t match_and (struct state_constraint *c, + flux_job_state_t state) +{ + state_match_t rv = MATCH_NOTSET; + struct state_constraint *cp = zlistx_first (c->values); + while (cp) { + /* This is an and statement, so if it a match is NEVER, we + * know that this constraint will return NEVER all the time. + * + * An ALWAYS can be demoted to a MAYBE and a MAYBE can be + * demoted to NEVER, so we keep iterating the match callbacks. + */ + state_match_t m = cp->match (cp, state); + if (m == MATCH_NEVER) + return MATCH_NEVER; + else if (rv == MATCH_NOTSET) + rv = m; + else if (rv == MATCH_ALWAYS + && m == MATCH_MAYBE) { + rv = MATCH_MAYBE; + } + /* else if rv == MATCH_MAYBE, + * m == MATCH_MAYBE or m == MAYBE_ALWAYS, + * rv stays MATCH_MAYBE + */ + cp = zlistx_next (c->values); + } + /* empty op return MATCH_ALWAYS */ + if (rv == MATCH_NOTSET) + return MATCH_ALWAYS; + return rv; +} + +static state_match_t match_or (struct state_constraint *c, + flux_job_state_t state) +{ + state_match_t rv = MATCH_NOTSET; + struct state_constraint *cp = zlistx_first (c->values); + while (cp) { + /* This is an or statement, so if it a match is ALWAYS, we + * know that this constraint will return ALWAYS all the time. + * + * A NEVER can be promoted to to a MAYBE and a MAYBE can be + * promoted to an ALWAYS, so we keep on iterating the match + * callbacks. + */ + state_match_t m = cp->match (cp, state); + if (m == MATCH_ALWAYS) + return MATCH_ALWAYS; + else if (rv == MATCH_NOTSET) + rv = m; + else if (rv == MATCH_NEVER + && m == MATCH_MAYBE) + rv = MATCH_MAYBE; + /* else if rv == MATCH_MAYBE, + * m == MATCH_NEVER or m == MAYBE_MAYBE, + * rv stays MATCH_MAYBE + */ + cp = zlistx_next (c->values); + } + /* empty op return MATCH_ALWAYS */ + if (rv == MATCH_NOTSET) + return MATCH_ALWAYS; + return rv; +} + +static state_match_t match_not (struct state_constraint *c, + flux_job_state_t state) +{ + state_match_t m = match_and (c, state); + if (m == MATCH_ALWAYS) + return MATCH_NEVER; + else if (m == MATCH_NEVER) + return MATCH_ALWAYS; + return MATCH_MAYBE; +} + +static struct state_constraint *conditional_constraint (const char *type, + json_t *values, + flux_error_t *errp) +{ + json_t *entry; + size_t index; + struct state_constraint *c; + match_f match_cb; + + if (streq (type, "and")) + match_cb = match_and; + else if (streq (type, "or")) + match_cb = match_or; + else /* streq (type, "not") */ + match_cb = match_not; + + if (!(c = state_constraint_new (match_cb, + state_constraint_destructor, + errp))) + return NULL; + + json_array_foreach (values, index, entry) { + struct state_constraint *cp = state_constraint_create (entry, errp); + if (!cp) + goto error; + if (!zlistx_add_end (c->values, cp)) { + errprintf (errp, "Out of memory"); + state_constraint_destroy (cp); + goto error; + } + } + return c; + + error: + state_constraint_destroy (c); + return NULL; +} + +void state_constraint_destroy (struct state_constraint *constraint) +{ + if (constraint) { + int saved_errno = errno; + zlistx_destroy (&constraint->values); + free (constraint); + errno = saved_errno; + } +} + +struct state_constraint *state_constraint_create (json_t *constraint, flux_error_t *errp) +{ + const char *op; + json_t *values; + + if (constraint) { + if (!json_is_object (constraint)) { + errprintf (errp, "constraint must be JSON object"); + return NULL; + } + if (json_object_size (constraint) > 1) { + errprintf (errp, "constraint must only contain 1 element"); + return NULL; + } + json_object_foreach (constraint, op, values) { + if (!json_is_array (values)) { + errprintf (errp, "operator %s values not an array", op); + return NULL; + } + if (streq (op, "userid") + || streq (op, "name") + || streq (op, "queue")) + return state_constraint_new (match_maybe, NULL, errp); + else if (streq (op, "results")) + return state_constraint_new (match_result, NULL, errp); + else if (streq (op, "states")) + return create_states_constraint (values, errp); + else if (streq (op, "t_submit") + || streq (op, "t_depend") + || streq (op, "t_run") + || streq (op, "t_cleanup") + || streq (op, "t_inactive")) + return create_timestamp_constraint (op, errp); + else if (streq (op, "or") || streq (op, "and") || streq (op, "not")) + return conditional_constraint (op, values, errp); + else { + errprintf (errp, "unknown constraint operator: %s", op); + return NULL; + } + } + } + return state_constraint_new (match_always, NULL, errp); +} + +bool state_match (int state, struct state_constraint *constraint) +{ + int valid_states = (FLUX_JOB_STATE_ACTIVE | FLUX_JOB_STATE_INACTIVE); + + if (!state + || (state & ~valid_states) + || ((state & (state - 1)) != 0 /* classic is more than 1 bit set trick */ + && state != FLUX_JOB_STATE_PENDING + && state != FLUX_JOB_STATE_RUNNING + && state != FLUX_JOB_STATE_ACTIVE) + || !constraint) + return false; + + if ((state & (state - 1)) != 0) { + if (state == FLUX_JOB_STATE_PENDING) + return (state_match (FLUX_JOB_STATE_DEPEND, constraint) + || state_match (FLUX_JOB_STATE_PRIORITY, constraint) + || state_match (FLUX_JOB_STATE_SCHED, constraint)); + else if (state == FLUX_JOB_STATE_RUNNING) + return (state_match (FLUX_JOB_STATE_RUN, constraint) + || state_match (FLUX_JOB_STATE_CLEANUP, constraint)); + else /* state == FLUX_JOB_STATE_ACTIVE */ + return (state_match (FLUX_JOB_STATE_PENDING, constraint) + || state_match (FLUX_JOB_STATE_RUNNING, constraint)); + } + else { + state_match_t m; + m = constraint->match (constraint, state); + if (m == MATCH_ALWAYS || m == MATCH_MAYBE) + return true; + return false; + } +} + +/* vi: ts=4 sw=4 expandtab + */ diff --git a/src/modules/job-list/state_match.h b/src/modules/job-list/state_match.h new file mode 100644 index 000000000000..56422011ae13 --- /dev/null +++ b/src/modules/job-list/state_match.h @@ -0,0 +1,37 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#ifndef HAVE_JOB_LIST_STATE_MATCH_H +#define HAVE_JOB_LIST_STATE_MATCH_H 1 + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include /* flux_error_t */ +#include + +#include "job_data.h" + +/* Similar to list_constraint_create() but only cares about + * "states" operation and the potential for a consraint to + * return true given a job state.. + */ +struct state_constraint *state_constraint_create (json_t *constraint, + flux_error_t *errp); + +void state_constraint_destroy (struct state_constraint *constraint); + +/* determines if a job in 'state' could potentially return true with + * the given constraint. 'state' can be job state or virtual job state. + */ +bool state_match (int state, struct state_constraint *constraint); + +#endif /* !HAVE_JOB_LIST_STATE_MATCH_H */ diff --git a/src/modules/job-list/test/match.c b/src/modules/job-list/test/match.c new file mode 100644 index 000000000000..2859e4c2d17e --- /dev/null +++ b/src/modules/job-list/test/match.c @@ -0,0 +1,1492 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include + +#include "src/common/libtap/tap.h" +#include "src/modules/job-list/job_data.h" +#include "src/modules/job-list/match.h" +#include "ccan/str/str.h" + +static void list_constraint_create_corner_case (const char *str, + const char *fmt, + ...) +{ + struct list_constraint *c; + char buf[1024]; + flux_error_t error; + json_error_t jerror; + json_t *jc; + va_list ap; + + if (!(jc = json_loads (str, 0, &jerror))) + BAIL_OUT ("json constraint invalid: %s", jerror.text); + + va_start (ap, fmt); + vsnprintf(buf, sizeof (buf), fmt, ap); + va_end (ap); + + c = list_constraint_create (jc, &error); + + ok (c == NULL, "list_constraint_create fails on %s", buf); + diag ("error: %s", error.text); + json_decref (jc); +} + +static void test_corner_case (void) +{ + ok (job_match (NULL, NULL) == false, + "job_match returns false on NULL inputs"); + + list_constraint_create_corner_case ("{\"userid\":[1], \"name\":[\"foo\"] }", + "object with too many keys"); + list_constraint_create_corner_case ("{\"userid\":1}", + "object with values not array"); + list_constraint_create_corner_case ("{\"foo\":[1]}", + "object with invalid operation"); + list_constraint_create_corner_case ("{\"userid\":[\"foo\"]}", + "userid value not integer"); + list_constraint_create_corner_case ("{\"name\":[1]}", + "name value not string"); + list_constraint_create_corner_case ("{\"queue\":[1]}", + "queue value not string"); + list_constraint_create_corner_case ("{\"states\":[0.0]}", + "states value not integer or string"); + list_constraint_create_corner_case ("{\"states\":[\"foo\"]}", + "states value not valid string"); + list_constraint_create_corner_case ("{\"states\":[8192]}", + "states value not valid integer"); + list_constraint_create_corner_case ("{\"results\":[0.0]}", + "results value not integer or string"); + list_constraint_create_corner_case ("{\"results\":[\"foo\"]}", + "results value not valid string"); + list_constraint_create_corner_case ("{\"results\":[8192]}", + "results value not valid integer"); + list_constraint_create_corner_case ("{\"t_depend\":[]}", + "t_depend value not specified"); + list_constraint_create_corner_case ("{\"t_depend\":[1.0]}", + "t_depend value in invalid format (int)"); + list_constraint_create_corner_case ("{\"t_depend\":[\"0.0\"]}", + "t_depend no comparison operator"); + list_constraint_create_corner_case ("{\"t_depend\":[\">=foof\"]}", + "t_depend value invalid (str)"); + list_constraint_create_corner_case ("{\"t_depend\":[\">=-1.0\"]}", + "t_depend value < 0.0 (str)"); + list_constraint_create_corner_case ("{\"not\":[1]}", + "sub constraint not a constraint"); +} + +static struct job *setup_job (uint32_t userid, + const char *name, + const char *queue, + flux_job_state_t state, + flux_job_result_t result, + double t_submit, + double t_depend, + double t_run, + double t_cleanup, + double t_inactive) +{ + struct job *job; + int bitmask = 0x1; + if (!(job = job_create (NULL, FLUX_JOBID_ANY))) + BAIL_OUT ("failed to create job"); + job->userid = userid; + if (name) + job->name = name; + if (queue) + job->queue = queue; + job->state = state; + if (state) { + /* Assume all jobs run, we don't skip any states, so add bitmask + * for all states lower than configured one + */ + job->states_mask = job->state; + while (!(job->states_mask & bitmask)) { + job->states_mask |= bitmask; + bitmask <<= 1; + } + } + job->result = result; + job->t_submit = t_submit; + job->t_depend = t_depend; + job->t_run = t_run; + job->t_cleanup = t_cleanup; + job->t_inactive = t_inactive; + /* assume for all tests */ + job->submit_version = 1; + return job; +} + +static struct list_constraint *create_list_constraint (const char *constraint) +{ + struct list_constraint *c; + flux_error_t error; + json_error_t jerror; + json_t *jc = NULL; + + if (constraint) { + if (!(jc = json_loads (constraint, 0, &jerror))) + BAIL_OUT ("json constraint invalid: %s", jerror.text); + } + + if (!(c = list_constraint_create (jc, &error))) + BAIL_OUT ("list constraint create fail: %s", error.text); + + json_decref (jc); + return c; +} + +static void test_basic_special_cases (void) +{ + struct job *job = setup_job (0, NULL, NULL, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0); + struct list_constraint *c; + bool rv; + + c = create_list_constraint ("{}"); + rv = job_match (job, c); + ok (rv == true, "empty object works as expected"); + list_constraint_destroy (c); + + c = create_list_constraint (NULL); + rv = job_match (job, c); + ok (rv == true, "NULL constraint works as expected"); + list_constraint_destroy (c); + + job_destroy (job); +} + +struct basic_userid_test { + uint32_t userid; + bool expected; +}; + +struct basic_userid_constraint_test { + const char *constraint; + struct basic_userid_test tests[4]; +} basic_userid_tests[] = { + { + "{ \"userid\": [ ] }", + { + { 42, false, }, + { 0, false, }, + }, + }, + { + "{ \"userid\": [ 42 ] }", + { + { 42, true, }, + { 43, false, }, + { 0, false, }, + }, + }, + { + "{ \"userid\": [ 42, 43 ] }", + { + { 42, true, }, + { 43, true, }, + { 44, false, }, + { 0, false, }, + }, + }, + /* FLUX_USERID_UNKNOWN = 0xFFFFFFFF */ + { + "{ \"userid\": [ -1 ] }", + { + { 42, true, }, + { 43, true, }, + { 0, false, }, + }, + }, + { + NULL, + { + { 0, false, }, + }, + }, +}; + +static void test_basic_userid (void) +{ + struct basic_userid_constraint_test *ctests = basic_userid_tests; + int index = 0; + + while (ctests->constraint) { + struct basic_userid_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (tests->userid) { + struct job *job; + bool rv; + job = setup_job (tests->userid, NULL, NULL, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0); + rv = job_match (job, c); + ok (rv == tests->expected, + "basic userid job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +struct basic_name_test { + const char *name; + bool expected; + bool end; /* name can be NULL */ +}; + +struct basic_name_constraint_test { + const char *constraint; + struct basic_name_test tests[5]; +} basic_name_tests[] = { + { + "{ \"name\": [ ] }", + { + /* N.B. name can potentially be NULL */ + { NULL, false, false, }, + { NULL, false, true, }, + }, + }, + { + "{ \"name\": [ \"foo\" ] }", + { + /* N.B. name can potentially be NULL */ + { NULL, false, false, }, + { "foo", true, false, }, + { "bar", false, false, }, + { NULL, false, true, }, + }, + }, + { + "{ \"name\": [ \"foo\", \"bar\" ] }", + { + /* N.B. name can potentially be NULL */ + { NULL, false, false, }, + { "foo", true, false, }, + { "bar", true, false, }, + { "baz", false, false, }, + { NULL, false, true, }, + }, + }, + { + NULL, + { + { NULL, false, true, }, + }, + }, +}; + +static void test_basic_name (void) +{ + struct basic_name_constraint_test *ctests = basic_name_tests; + int index = 0; + + while (ctests->constraint) { + struct basic_name_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (!tests->end) { + struct job *job; + bool rv; + job = setup_job (0, tests->name, NULL, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0); + rv = job_match (job, c); + ok (rv == tests->expected, + "basic name job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +struct basic_queue_test { + const char *queue; + bool expected; + bool end; /* queue can be NULL */ +}; + +struct basic_queue_constraint_test { + const char *constraint; + struct basic_queue_test tests[5]; +} basic_queue_tests[] = { + { + "{ \"queue\": [ ] }", + { + /* N.B. queue can potentially be NULL */ + { NULL, false, false, }, + { NULL, false, true, }, + }, + }, + { + "{ \"queue\": [ \"foo\" ] }", + { + /* N.B. queue can potentially be NULL */ + { NULL, false, false, }, + { "foo", true, false, }, + { "bar", false, false, }, + { NULL, false, true, }, + }, + }, + { + "{ \"queue\": [ \"foo\", \"bar\" ] }", + { + /* N.B. queue can potentially be NULL */ + { NULL, false, false, }, + { "foo", true, false, }, + { "bar", true, false, }, + { "baz", false, false, }, + { NULL, false, true, }, + }, + }, + { + NULL, + { + { NULL, false, true, }, + }, + }, +}; + +static void test_basic_queue (void) +{ + struct basic_queue_constraint_test *ctests = basic_queue_tests; + int index = 0; + + while (ctests->constraint) { + struct basic_queue_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (!tests->end) { + struct job *job; + bool rv; + job = setup_job (0, NULL, tests->queue, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0); + rv = job_match (job, c); + ok (rv == tests->expected, + "basic queue job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +struct basic_states_test { + flux_job_state_t state; + bool expected; +}; + +struct basic_states_constraint_test { + const char *constraint; + struct basic_states_test tests[4]; +} basic_states_tests[] = { + { + "{ \"states\": [ ] }", + { + { FLUX_JOB_STATE_NEW, false, }, + { 0, false, }, + }, + }, + { + /* sanity check integer inputs work, we assume FLUX_JOB_STATE_NEW + * will always be 1, use strings everywhere else + */ + "{ \"states\": [ 1 ] }", + { + { FLUX_JOB_STATE_NEW, true, }, + { 0, false, }, + }, + }, + { + "{ \"states\": [ \"sched\" ] }", + { + { FLUX_JOB_STATE_SCHED, true, }, + { FLUX_JOB_STATE_RUN, false, }, + { 0, false, }, + }, + }, + { + "{ \"states\": [ \"sched\", \"RUN\" ] }", + { + { FLUX_JOB_STATE_SCHED, true, }, + { FLUX_JOB_STATE_RUN, true, }, + { FLUX_JOB_STATE_INACTIVE, false, }, + { 0, false, }, + }, + }, + { + NULL, + { + { 0, false, }, + }, + }, +}; + +static void test_basic_states (void) +{ + struct basic_states_constraint_test *ctests = basic_states_tests; + int index = 0; + + while (ctests->constraint) { + struct basic_states_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (tests->state) { + struct job *job; + bool rv; + job = setup_job (0, NULL, NULL, tests->state, 0, 0.0, 0.0, 0.0, 0.0, 0.0); + rv = job_match (job, c); + ok (rv == tests->expected, + "basic states job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +struct basic_results_test { + flux_job_state_t state; + flux_job_result_t result; + bool expected; +}; + +struct basic_results_constraint_test { + const char *constraint; + struct basic_results_test tests[4]; +} basic_results_tests[] = { + { + "{ \"results\": [ ] }", + { + { FLUX_JOB_STATE_NEW, FLUX_JOB_RESULT_COMPLETED, false, }, + { 0, 0, false, }, + }, + }, + { + /* sanity check integer inputs work, we assume + * FLUX_JOB_RESULT_COMPLETED will always be 1, use strings + * everywhere else + */ + "{ \"results\": [ 1 ] }", + { + { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_COMPLETED, true, }, + { 0, 0, false, }, + }, + }, + { + "{ \"results\": [ \"completed\" ] }", + { + { FLUX_JOB_STATE_RUN, 0, false, }, + { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_COMPLETED, true, }, + { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_FAILED, false, }, + { 0, 0, false, }, + }, + }, + { + "{ \"results\": [ \"completed\", \"FAILED\" ] }", + { + { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_COMPLETED, true, }, + { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_FAILED, true, }, + { FLUX_JOB_STATE_INACTIVE, FLUX_JOB_RESULT_CANCELED, false, }, + { 0, 0, false, }, + }, + }, + { + NULL, + { + { 0, 0, false, }, + }, + }, +}; + +static void test_basic_results (void) +{ + struct basic_results_constraint_test *ctests = basic_results_tests; + int index = 0; + + while (ctests->constraint) { + struct basic_results_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (tests->state) { /* result can be 0, iterate on state > 0 */ + struct job *job; + bool rv; + job = setup_job (0, NULL, NULL, tests->state, tests->result, 0.0, 0.0, 0.0, 0.0, 0.0); + rv = job_match (job, c); + ok (rv == tests->expected, + "basic results job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +struct basic_timestamp_test { + flux_job_state_t state; + int submit_version; + double t_submit; + double t_depend; + double t_run; + double t_cleanup; + double t_inactive; + bool expected; + bool end; /* timestamps can be 0 */ +}; + +struct basic_timestamp_constraint_test { + const char *constraint; + struct basic_timestamp_test tests[7]; +} basic_timestamp_tests[] = { + { + "{ \"t_submit\": [ \">=0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, true, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_depend\": [ \">=0.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, true, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + /* N.B. t_run >= 0 is false if state RUN not yet reached */ + { + "{ \"t_run\": [ \">=0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, true, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + /* N.B. t_cleanup >= 0 is false if state CLEANUP not yet reached */ + { + "{ \"t_cleanup\": [ \">=0.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, true, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + /* N.B. t_inactive >= 0 is false if state INACTIVE not yet reached */ + { + "{ \"t_inactive\": [ \">=0.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \"<100.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \"<=100.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \"<50.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \"<=50.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \"<25.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \"<=25.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \">100.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \">=100.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \">50.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \">=50.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \">25.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_inactive\": [ \">=25.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + /* + * Need to test special legacy case, submit_version == 0 where + * `t_depend` means `t_submit`. So all tests fail for <15.0 when + * submit version == 1, but should all pass for submit version == 0. + */ + { + "{ \"t_depend\": [ \"<15.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_PRIORITY, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_SCHED, 1, 10.0, 20.0, 0.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_RUN, 1, 10.0, 20.0, 30.0, 0.0, 0.0, false, false, }, + { FLUX_JOB_STATE_CLEANUP, 1, 10.0, 20.0, 30.0, 40.0, 0.0, false, false, }, + { FLUX_JOB_STATE_INACTIVE, 1, 10.0, 20.0, 30.0, 40.0, 50.0, false, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + "{ \"t_depend\": [ \"<15.0\" ] }", + { + { FLUX_JOB_STATE_DEPEND, 0, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_PRIORITY, 0, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_SCHED, 0, 10.0, 20.0, 0.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_RUN, 0, 10.0, 20.0, 30.0, 0.0, 0.0, true, false, }, + { FLUX_JOB_STATE_CLEANUP, 0, 10.0, 20.0, 30.0, 40.0, 0.0, true, false, }, + { FLUX_JOB_STATE_INACTIVE, 0, 10.0, 20.0, 30.0, 40.0, 50.0, true, false, }, + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, + { + NULL, + { + { 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, false, true, }, + }, + }, +}; + +static void test_basic_timestamp (void) +{ + struct basic_timestamp_constraint_test *ctests = basic_timestamp_tests; + int index = 0; + + while (ctests->constraint) { + struct basic_timestamp_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (!tests->end) { + struct job *job; + bool rv; + job = setup_job (0, + NULL, + NULL, + tests->state, + 0, + tests->t_submit, + tests->t_depend, + tests->t_run, + tests->t_cleanup, + tests->t_inactive); + /* special for legacy corner case */ + job->submit_version = tests->submit_version; + rv = job_match (job, c); + ok (rv == tests->expected, + "basic timestamp job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +struct basic_conditionals_test { + uint32_t userid; + const char *name; + bool expected; +}; + +struct basic_conditionals_constraint_test { + const char *constraint; + struct basic_conditionals_test tests[5]; +} basic_conditionals_tests[] = { + { + "{ \"or\": [] }", + { + { 42, "foo", true, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"and\": [] }", + { + { 42, "foo", true, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"not\": [] }", + { + { 42, "foo", false, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"not\": [ { \"userid\": [ 42 ] } ] }", + { + { 42, "foo", false, }, + { 43, "foo", true, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"or\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + { 43, "bar", false, }, + { 42, "bar", true, }, + { 43, "foo", true, }, + { 42, "foo", true, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"or\": \ + [ \ + { \"not\": [ { \"userid\": [ 42 ] } ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + { 43, "bar", true, }, + { 42, "bar", false, }, + { 43, "foo", true, }, + { 42, "foo", true, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"not\": \ + [ \ + { \"or\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + } \ + ] \ + }", + { + { 43, "bar", true, }, + { 42, "bar", false, }, + { 43, "foo", false, }, + { 42, "foo", false, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"and\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + { 43, "bar", false, }, + { 42, "bar", false, }, + { 43, "foo", false, }, + { 42, "foo", true, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"and\": \ + [ \ + { \"not\": [ { \"userid\": [ 42 ] } ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + { 43, "bar", false, }, + { 42, "bar", false, }, + { 43, "foo", true, }, + { 42, "foo", false, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"not\": \ + [ \ + { \"and\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + } \ + ] \ + }", + { + { 43, "bar", true, }, + { 42, "bar", true, }, + { 43, "foo", true, }, + { 42, "foo", false, }, + { 0, NULL, false, }, + }, + }, + { + "{ \"and\": \ + [ \ + { \"or\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"userid\": [ 43 ] } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + { 43, "bar", false, }, + { 42, "bar", false, }, + { 43, "foo", true, }, + { 42, "foo", true, }, + { 0, NULL, false, }, + }, + }, + { + NULL, + { + { 0, NULL, false, }, + }, + }, +}; + +static void test_basic_conditionals (void) +{ + struct basic_conditionals_constraint_test *ctests = basic_conditionals_tests; + int index = 0; + + while (ctests->constraint) { + struct basic_conditionals_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (tests->userid) { + struct job *job; + bool rv; + job = setup_job (tests->userid, tests->name, NULL, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0); + rv = job_match (job, c); + ok (rv == tests->expected, + "basic conditionals job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +/* following tests emulate some "realworld"-ish matching */ +struct realworld_test { + uint32_t userid; + const char *name; + const char *queue; + flux_job_state_t state; + flux_job_result_t result; + double t_inactive; + bool expected; +}; + +struct realworld_constraint_test { + const char *constraint; + struct realworld_test tests[8]; +} realworld_tests[] = { + { + /* all the jobs in all states for a specific user */ + "{ \"and\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"states\": [ \"pending\", \"running\", \"inactive\" ] } \ + ] \ + }", + { + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_DEPEND, + 0, + 0.0, + true, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_RUN, + 0, + 0.0, + true, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_COMPLETED, + 2000.0, + true, + }, + { + 43, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_COMPLETED, + 2000.0, + false, + }, + { + 0, + NULL, + NULL, + 0, + 0, + 0.0, + false + }, + }, + }, + { + /* all the unsuccessful jobs for a specific user */ + "{ \"and\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"results\": [ \"failed\", \"canceled\", \"timeout\" ] } \ + ] \ + }", + { + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_FAILED, + 2000.0, + true, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_CANCELED, + 2000.0, + true, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_TIMEOUT, + 2000.0, + true, + }, + { + 43, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_FAILED, + 2000.0, + false, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_DEPEND, + 0, + 0.0, + false, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_RUN, + 0, + 0.0, + false, + }, + { + 0, + NULL, + NULL, + 0, + 0, + 0.0, + false + }, + }, + }, + { + /* all the pending and running jobs for a user, in two specific queues */ + "{ \"and\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"states\" : [ \"pending\", \"running\" ] }, \ + { \"queue\": [ \"batch\", \"debug\" ] } \ + ] \ + }", + { + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_DEPEND, + 0, + 0.0, + true, + }, + { + 42, + "foo", + "debug", + FLUX_JOB_STATE_DEPEND, + 0, + 0.0, + true, + }, + { + 42, + "foo", + "debug", + FLUX_JOB_STATE_RUN, + 0, + 0.0, + true, + }, + { + 43, + "foo", + "batch", + FLUX_JOB_STATE_DEPEND, + 0, + 0.0, + false, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_COMPLETED, + 2000.0, + false, + }, + { + 42, + "foo", + "gpu", + FLUX_JOB_STATE_DEPEND, + 0, + 0.0, + false, + }, + { + 0, + NULL, + NULL, + 0, + 0, + 0.0, + false + }, + }, + }, + { + /* jobs for a user, in queue batch, with specific job name, are running */ + "{ \"and\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"queue\": [ \"batch\" ] }, \ + { \"name\": [ \"foo\" ] }, \ + { \"states\": [ \"running\" ] } \ + ] \ + }", + { + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_RUN, + 0, + 0.0, + true, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_CLEANUP, + 0, + 0.0, + true, + }, + { + 43, + "foo", + "batch", + FLUX_JOB_STATE_RUN, + 0, + 0.0, + false, + }, + { + 42, + "foo", + "debug", + FLUX_JOB_STATE_RUN, + 0, + 0.0, + false, + }, + { + 42, + "bar", + "batch", + FLUX_JOB_STATE_RUN, + 0, + 0.0, + false, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_COMPLETED, + 2000.0, + false, + }, + { + 0, + NULL, + NULL, + 0, + 0, + 0.0, + false + }, + }, + }, + { + /* all the inactive jobs since a specific time (via t_inactve) */ + "{ \"and\": \ + [ \ + { \"states\": [ \"inactive\" ] }, \ + { \"t_inactive\": [ \">=500.0\" ] } \ + ] \ + }", + { + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_SCHED, + 0, + 0, + false, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_RUN, + 0, + 0, + false, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_COMPLETED, + 100.0, + false, + }, + { + 42, + "foo", + "batch", + FLUX_JOB_STATE_INACTIVE, + FLUX_JOB_RESULT_COMPLETED, + 1000.0, + true, + }, + { + 0, + NULL, + NULL, + 0, + 0, + 0.0, + false + }, + }, + }, + { + NULL, + { + { + 0, + NULL, + NULL, + 0, + 0, + 0.0, + false + }, + }, + }, +}; + +static void test_realworld (void) +{ + struct realworld_constraint_test *ctests = realworld_tests; + int index = 0; + + while (ctests->constraint) { + struct realworld_test *tests = ctests->tests; + struct list_constraint *c; + int index2 = 0; + + c = create_list_constraint (ctests->constraint); + while (tests->userid) { + struct job *job; + bool rv; + job = setup_job (tests->userid, + tests->name, + tests->queue, + tests->state, + tests->result, + 0.0, + 0.0, + 0.0, + 0.0, + tests->t_inactive); + rv = job_match (job, c); + ok (rv == tests->expected, + "realworld job match test #%d/#%d", + index, index2); + job_destroy (job); + index2++; + tests++; + } + + index++; + list_constraint_destroy (c); + ctests++; + } +} + +int main (int argc, char *argv[]) +{ + plan (NO_PLAN); + + test_corner_case (); + test_basic_special_cases (); + test_basic_userid (); + test_basic_name (); + test_basic_queue (); + test_basic_states (); + test_basic_results (); + test_basic_timestamp (); + test_basic_conditionals (); + test_realworld (); + + done_testing (); +} + +/* + * vi:ts=4 sw=4 expandtab + */ diff --git a/src/modules/job-list/test/state_match.c b/src/modules/job-list/test/state_match.c new file mode 100644 index 000000000000..d05a8c173ff9 --- /dev/null +++ b/src/modules/job-list/test/state_match.c @@ -0,0 +1,1204 @@ +/************************************************************\ + * Copyright 2023 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include + +#include "src/common/libtap/tap.h" +#include "src/modules/job-list/job_data.h" +#include "src/modules/job-list/state_match.h" +#include "ccan/str/str.h" + +static void state_constraint_create_corner_case (const char *str, + const char *fmt, + ...) +{ + struct state_constraint *c; + char buf[1024]; + flux_error_t error; + json_error_t jerror; + json_t *jc; + va_list ap; + + if (!(jc = json_loads (str, 0, &jerror))) + BAIL_OUT ("json constraint invalid: %s", jerror.text); + + va_start (ap, fmt); + vsnprintf(buf, sizeof (buf), fmt, ap); + va_end (ap); + + c = state_constraint_create (jc, &error); + ok (c == NULL, "state_constraint_create fails on %s", buf); + diag ("error: %s", error.text); + json_decref (jc); +} + +static void test_corner_case (void) +{ + ok (state_match (0, NULL) == false, + "state_match returns false on NULL inputs"); + + state_constraint_create_corner_case ("{\"userid\":[1], \"name\":[\"foo\"] }", + "object with too many keys"); + state_constraint_create_corner_case ("{\"userid\":1}", + "object with values not array"); + state_constraint_create_corner_case ("{\"foo\":[1]}", + "object with invalid operation"); + state_constraint_create_corner_case ("{\"not\":[1]}", + "sub constraint not a constraint"); +} + +/* expected array - expected values for + * FLUX_JOB_STATE_DEPEND + * FLUX_JOB_STATE_PRIORITY + * FLUX_JOB_STATE_SCHED + * FLUX_JOB_STATE_RUN + * FLUX_JOB_STATE_CLEANUP + * FLUX_JOB_STATE_INACTIVE + * FLUX_JOB_STATE_PENDING + * FLUX_JOB_STATE_RUNNING + * FLUX_JOB_STATE_ACTIVE + */ +struct state_match_constraint_test { + const char *constraint; + bool expected[9]; +} state_match_tests[] = { + /* + * Empty values tests + */ + { + "{ \"states\": [ ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"and\": [ ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"or\": [ ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"not\": [ ] }", + { + false, + false, + false, + false, + false, + false, + false, + false, + false, + } + }, + /* + * Simple states tests + */ + { + "{ \"states\": [ \"pending\" ] }", + { + true, + true, + true, + false, + false, + false, + true, + false, + true, + } + }, + { + "{ \"and\": [ { \"states\": [ \"pending\" ] } ] }", + { + true, + true, + true, + false, + false, + false, + true, + false, + true, + } + }, + { + "{ \"or\": [ { \"states\": [ \"pending\" ] } ] }", + { + true, + true, + true, + false, + false, + false, + true, + false, + true, + } + }, + { + "{ \"not\": [ { \"states\": [ \"pending\" ] } ] }", + { + false, + false, + false, + true, + true, + true, + false, + true, + true, + } + }, + /* + * Simple results tests + */ + /* N.B. "results" assumes job state == INACTIVE */ + { + "{ \"results\": [ \"completed\" ] }", + { + false, + false, + false, + false, + false, + true, + false, + false, + false, + } + }, + /* N.B. Returning 'true' for FLUX_JOB_STATE_INACTIVE may be + * surprising here. If the job state is FLUX_JOB_STATE_INACTIVE, + * the result of "results=COMPLETED" is "maybe true", b/c it + * depends on the actual result. So the "not" of a "maybe true" + * is still "maybe true". + */ + { + "{ \"not\": [ { \"results\": [ \"completed\" ] } ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + /* + * Simple timestamp tests + */ + { + "{ \"t_submit\": [ 100.0 ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"t_run\": [ \">100.0\" ] }", + { + false, + false, + false, + true, + true, + true, + false, + true, + true, + } + }, + /* N.B. For state depend, priority, sched, is always false, so not + * makes it always true. For states run, cleanup, and inactive is + * maybe true, so not maybe true = true. So all would return + * true. + */ + { + "{ \"not\": [ { \"t_run\": [ \"<=500\" ] } ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + /* + * AND tests w/ states + */ + { + "{ \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"states\": [ \"priority\" ] } \ + ] \ + }", + { + false, + false, + false, + false, + false, + false, + false, + false, + false, + } + }, + { + "{ \"not\": \ + [ \ + { \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"states\": [ \"priority\" ] } \ + ] \ + } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"and\": \ + [ \ + { \"not\": [ { \"states\": [ \"depend\" ] } ] }, \ + { \"states\": [ \"priority\" ] } \ + ] \ + }", + { + false, + true, + false, + false, + false, + false, + true, + false, + true, + } + }, + /* + * OR tests w/ states + */ + { + "{ \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"states\": [ \"priority\" ] } \ + ] \ + }", + { + true, + true, + false, + false, + false, + false, + true, + false, + true, + } + }, + { + "{ \"not\": \ + [ \ + { \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"states\": [ \"priority\" ] } \ + ] \ + } \ + ] \ + }", + { + false, + false, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"or\": \ + [ \ + { \"not\": [ { \"states\": [ \"depend\" ] } ] }, \ + { \"states\": [ \"priority\" ] } \ + ] \ + }", + { + false, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + /* + * AND tests w/ states & results + */ + { + "{ \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"results\": [ \"completed\" ] } \ + ] \ + }", + { + false, + false, + false, + false, + false, + false, + false, + false, + false, + } + }, + { + "{ \"not\": \ + [ \ + { \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"results\": [ \"completed\" ] } \ + ] \ + } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"not\": [ { \"results\": [ \"completed\" ] } ] } \ + ] \ + }", + { + true, + false, + false, + false, + false, + false, + true, + false, + true, + } + }, + /* + * OR tests w/ states & results + */ + { + "{ \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"results\": [ \"completed\" ] } \ + ] \ + }", + { + true, + false, + false, + false, + false, + true, + true, + false, + true, + } + }, + { + "{ \"not\": \ + [ \ + { \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"results\": [ \"completed\" ] } \ + ] \ + } \ + ] \ + }", + { + false, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"not\": [ { \"results\": [ \"completed\" ] } ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + /* + * AND tests w/ states & t_inactive + */ + { + "{ \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"t_inactive\": [ \">=100.0\" ] } \ + ] \ + }", + { + false, + false, + false, + false, + false, + false, + false, + false, + false, + } + }, + { + "{ \"not\": \ + [ \ + { \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"t_inactive\": [ \">=100.0\" ] } \ + ] \ + } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"not\": [ { \"t_inactive\": [ \">=100.0\" ] } ] } \ + ] \ + }", + { + true, + false, + false, + false, + false, + false, + true, + false, + true, + } + }, + /* + * OR tests w/ states & t_inactive + */ + { + "{ \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"t_inactive\": [ \">=100.0\" ] } \ + ] \ + }", + { + true, + false, + false, + false, + false, + true, + true, + false, + true, + } + }, + { + "{ \"not\": \ + [ \ + { \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"t_inactive\": [ \">=100.0\" ] } \ + ] \ + } \ + ] \ + }", + { + false, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + { + "{ \"or\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"not\": [ { \"t_inactive\": [ \">=100.0\" ] } ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + } + }, + /* + * Simple non-states tests + */ + { + "{ \"userid\": [ 42 ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"not\": [ { \"userid\": [ 42 ] } ] }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + /* + * non-states AND tests + */ + { + "{ \"and\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"and\": \ + [ \ + { \"not\": [ { \"userid\": [ 42 ] } ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + /* + * non-states OR tests + */ + { + "{ \"or\": \ + [ \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"or\": \ + [ \ + { \"not\": [ { \"userid\": [ 42 ] } ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + /* + * states and non-states AND tests + */ + { + "{ \"and\": \ + [ \ + { \"states\": [ \"running\" ] }, \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + false, + false, + false, + true, + true, + false, + false, + true, + true, + }, + }, + { + "{ \"and\": \ + [ \ + { \"not\": [ { \"states\": [ \"running\" ] } ] }, \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + false, + false, + true, + true, + false, + true, + }, + }, + /* N.B. All returning true may be difficult to understand here. + * The states check is effectively irrelevant. The userid or name + * could could be false, leading to the "and" constraint + * potentially being false for any job state. So the full + * constraint could be true for any job state. + */ + { + "{ \"not\": \ + [ \ + { \"and\": \ + [ \ + { \"states\": [ \"running\" ] }, \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + /* + * states and non-states OR tests + */ + /* N.B. All states return true here, b/c the states check is sort + * of irrelevant, the userid or name checks could always return + * true, leading to the or statement to be true that any state + * could be matched with this constraint. + */ + { + "{ \"or\": \ + [ \ + { \"states\": [ \"running\" ] }, \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"or\": \ + [ \ + { \"not\": [ { \"states\": [ \"running\" ] } ] }, \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"not\": \ + [ \ + { \"or\": \ + [ \ + { \"states\": [ \"running\" ] }, \ + { \"userid\": [ 42 ] }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + } \ + ] \ + }", + { + true, + true, + true, + false, + false, + true, + true, + false, + true, + }, + }, + /* + * complex tests, conditionals inside conditionals + */ + { + "{ \"and\": \ + [ \ + { \"and\": \ + [ \ + { \"states\": [ \"priority\" ] }, \ + { \"userid\": [ 42 ] } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + false, + true, + false, + false, + false, + false, + true, + false, + true, + }, + }, + { + "{ \"and\": \ + [ \ + { \"or\": \ + [ \ + { \"states\": [ \"priority\" ] }, \ + { \"userid\": [ 42 ] } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"and\": \ + [ \ + { \"and\": \ + [ \ + { \"results\": [ \"completed\" ] }, \ + { \"userid\": [ 42 ] } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + false, + false, + false, + false, + false, + true, + false, + false, + false, + }, + }, + { + "{ \"and\": \ + [ \ + { \"or\": \ + [ \ + { \"results\": [ \"completed\" ] }, \ + { \"userid\": [ 42 ] } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"or\": \ + [ \ + { \"states\": [ \"priority\" ] }, \ + { \"userid\": [ 42 ] } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + false, + false, + false, + false, + false, + true, + false, + true, + }, + }, + { + "{ \"and\": \ + [ \ + { \"not\": [ { \"states\": [ \"depend\" ] } ] }, \ + { \"or\": \ + [ \ + { \"states\": [ \"priority\" ] }, \ + { \"userid\": [ 42 ] } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + false, + true, + true, + true, + true, + true, + true, + true, + true, + }, + }, + { + "{ \"and\": \ + [ \ + { \"states\": [ \"depend\" ] }, \ + { \"not\": \ + [ \ + { \"or\": \ + [ \ + { \"states\": [ \"priority\" ] }, \ + { \"userid\": [ 42 ] } \ + ] \ + } \ + ] \ + }, \ + { \"name\": [ \"foo\" ] } \ + ] \ + }", + { + true, + false, + false, + false, + false, + false, + true, + false, + true, + }, + }, + { + NULL, + { + false, + false, + false, + false, + false, + false, + false, + false, + false, + }, + }, +}; + +static struct state_constraint *create_state_constraint (const char *constraint) +{ + struct state_constraint *c; + flux_error_t error; + json_error_t jerror; + json_t *jc = NULL; + + if (constraint) { + if (!(jc = json_loads (constraint, 0, &jerror))) + BAIL_OUT ("json constraint invalid: %s", jerror.text); + } + + if (!(c = state_constraint_create (jc, &error))) + BAIL_OUT ("constraint create fail: %s", error.text); + + json_decref (jc); + return c; +} + +static void test_state_match (void) +{ + struct state_match_constraint_test *ctests = state_match_tests; + struct state_constraint *c; + bool rv; + int index = 0; + + /* First test special case */ + c = create_state_constraint (NULL); + rv = state_match (FLUX_JOB_STATE_DEPEND, c); + ok (rv == ctests->expected[0], "state match test NULL DEPEND"); + rv = state_match (FLUX_JOB_STATE_PRIORITY, c); + ok (rv == ctests->expected[1], "state match test NULL PRIORITY"); + rv = state_match (FLUX_JOB_STATE_SCHED, c); + ok (rv == ctests->expected[2], "state match test NULL SCHED"); + rv = state_match (FLUX_JOB_STATE_RUN, c); + ok (rv == ctests->expected[3], "state match test NULL RUN"); + rv = state_match (FLUX_JOB_STATE_CLEANUP, c); + ok (rv == ctests->expected[4], "state match test NULL CLEANUP"); + rv = state_match (FLUX_JOB_STATE_INACTIVE, c); + ok (rv == ctests->expected[5], "state match test NULL INACTIVE"); + rv = state_match (FLUX_JOB_STATE_PENDING, c); + ok (rv == ctests->expected[6], "state match test NULL PENDING"); + rv = state_match (FLUX_JOB_STATE_RUNNING, c); + ok (rv == ctests->expected[7], "state match test NULL RUNNING"); + rv = state_match (FLUX_JOB_STATE_ACTIVE, c); + ok (rv == ctests->expected[8], "state match test NULL ACTIVE"); + + while (ctests->constraint) { + c = create_state_constraint (ctests->constraint); + rv = state_match (FLUX_JOB_STATE_DEPEND, c); + ok (rv == ctests->expected[0], "state match test #%d DEPEND", index); + rv = state_match (FLUX_JOB_STATE_PRIORITY, c); + ok (rv == ctests->expected[1], "state match test #%d PRIORITY", index); + rv = state_match (FLUX_JOB_STATE_SCHED, c); + ok (rv == ctests->expected[2], "state match test #%d SCHED", index); + rv = state_match (FLUX_JOB_STATE_RUN, c); + ok (rv == ctests->expected[3], "state match test #%d RUN", index); + rv = state_match (FLUX_JOB_STATE_CLEANUP, c); + ok (rv == ctests->expected[4], "state match test #%d CLEANUP", index); + rv = state_match (FLUX_JOB_STATE_INACTIVE, c); + ok (rv == ctests->expected[5], "state match test #%d INACTIVE", index); + rv = state_match (FLUX_JOB_STATE_PENDING, c); + ok (rv == ctests->expected[6], "state match test #%d PENDING", index); + rv = state_match (FLUX_JOB_STATE_RUNNING, c); + ok (rv == ctests->expected[7], "state match test #%d RUNNING", index); + rv = state_match (FLUX_JOB_STATE_ACTIVE, c); + ok (rv == ctests->expected[8], "state match test #%d ACTIVE", index); + + index++; + state_constraint_destroy (c); + ctests++; + } +} + +int main (int argc, char *argv[]) +{ + plan (NO_PLAN); + + test_corner_case (); + test_state_match (); + + done_testing (); +} + +/* + * vi:ts=4 sw=4 expandtab + */ diff --git a/t/t2260-job-list.t b/t/t2260-job-list.t index f841fff1c199..351574a01d9b 100755 --- a/t/t2260-job-list.t +++ b/t/t2260-job-list.t @@ -206,36 +206,36 @@ test_expect_success 'flux job list inactive jobs results are correct' ' test_expect_success 'flux job list only canceled jobs' ' id=$(id -u) && - state=`${JOB_CONV} strtostate INACTIVE` && result=`${JOB_CONV} strtoresult CANCELED` && - $jq -j -c -n "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \ + constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_canceled.out && test_cmp canceled.ids list_result_canceled.out ' test_expect_success 'flux job list only failed jobs' ' id=$(id -u) && - state=`${JOB_CONV} strtostate INACTIVE` && result=`${JOB_CONV} strtoresult FAILED` && - $jq -j -c -n "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \ + constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_failed.out && test_cmp failed.ids list_result_failed.out ' test_expect_success 'flux job list only timeout jobs' ' id=$(id -u) && - state=`${JOB_CONV} strtostate INACTIVE` && result=`${JOB_CONV} strtoresult TIMEOUT` && - $jq -j -c -n "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \ + constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_timeout.out && test_cmp timeout.ids list_result_timeout.out ' test_expect_success 'flux job list only completed jobs' ' id=$(id -u) && - state=`${JOB_CONV} strtostate INACTIVE` && result=`${JOB_CONV} strtoresult COMPLETED` && - $jq -j -c -n "{max_entries:1000, userid:${id}, states:${state}, results:${result}, attrs:[]}" \ + constraint="{ and: [ {userid:[${id}]}, {results:[${result}]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_result_completed.out && test_cmp completed.ids list_result_completed.out ' @@ -363,6 +363,229 @@ test_expect_success 'flux module stats job-list is open to guests' ' flux module stats job-list >/dev/null ' +# do some more advanced constraint queries + +test_expect_success 'flux job list hostname jobs' ' + id=$(id -u) && + constraint="{ and: [ {userid:[${id}]}, {name:[\"hostname\"]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_hostname_jobs.out && + numlines=$(cat pending.ids completed.ids | wc -l) && + test $(cat list_constraint_hostname_jobs.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list active hostname jobs' ' + id=$(id -u) && + constraint="{ and: [ {userid:[${id}]}, {states:[\"active\"]}, {name:[\"hostname\"]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_hostname.out && + test_cmp list_constraint_pending_hostname.out pending.ids +' + +test_expect_success 'flux job list inactive hostname jobs' ' + id=$(id -u) && + constraint="{ and: [ {userid:[${id}]}, {states:[\"inactive\"]}, {name:[\"hostname\"]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_inactive_hostname.out && + test_cmp list_constraint_inactive_hostname.out completed.ids +' + +test_expect_success 'flux job list invalid queue' ' + id=$(id -u) && + constraint="{ and: [ {userid:[${id}]}, {queue:[\"blarg\"]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_invalid_queue.out && + test $(cat list_constraint_invalid_queue.out | wc -l) -eq 0 +' + +test_expect_success 'flux job list active (1)' ' + state1=`${JOB_CONV} strtostate SCHED` && + state2=`${JOB_CONV} strtostate RUN` && + constraint="{ or: [ {states:[${state1}]}, {states:[${state2}]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_active1.out && + numlines=$(cat active.ids | wc -l) && + test $(cat list_constraint_active1.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list active (2)' ' + state1=`${JOB_CONV} strtostate INACTIVE` && + constraint="{ not: [ {states:[${state1}]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_active2.out && + numlines=$(cat active.ids | wc -l) && + test $(cat list_constraint_active2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list pending jobs or inactive jobs (1)' ' + state1=`${JOB_CONV} strtostate SCHED` && + state2=`${JOB_CONV} strtostate INACTIVE` && + constraint="{ or: [ {states:[${state1}]}, {states:[${state2}]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_inactive1.out && + numlines=$(cat pending.ids inactive.ids | wc -l) && + test $(cat list_constraint_pending_inactive1.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list pending jobs or inactive jobs (2)' ' + state1=`${JOB_CONV} strtostate SCHED` && + state2=`${JOB_CONV} strtostate INACTIVE` && + constraint="{ or: [ {states:[${state1}, ${state2}]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_inactive2.out && + numlines=$(cat pending.ids inactive.ids | wc -l) && + test $(cat list_constraint_pending_inactive2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list failed and canceled jobs (1)' ' + result1=`${JOB_CONV} strtoresult FAILED` && + result2=`${JOB_CONV} strtoresult CANCELED` && + constraint="{ or: [ {results:[${result1}]}, {results:[${result2}]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_failed_canceled1.out && + numlines=$(cat canceled.ids failed.ids | wc -l) && + test $(cat list_constraint_failed_canceled1.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list failed and canceled jobs (2)' ' + result1=`${JOB_CONV} strtoresult FAILED` && + result2=`${JOB_CONV} strtoresult CANCELED` && + constraint="{ and: [ {userid:[${id}]}, {results:[${result1}, ${result2}]}] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_failed_canceled2.out && + numlines=$(cat canceled.ids failed.ids | wc -l) && + test $(cat list_constraint_failed_canceled2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list pending jobs or failed jobs (1)' ' + state1=`${JOB_CONV} strtostate SCHED` && + state2=`${JOB_CONV} strtostate INACTIVE` && + result1=`${JOB_CONV} strtoresult FAILED` && + constraint="{ or: [ {states:[${state1}]}, {and: [ {states:[${state2}]}, {results:[${result1}]} ] } ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_failed1.out && + numlines=$(cat pending.ids failed.ids | wc -l) && + test $(cat list_constraint_pending_failed1.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list pending jobs or failed jobs (2)' ' + state1=`${JOB_CONV} strtostate SCHED` && + result1=`${JOB_CONV} strtoresult FAILED` && + constraint="{ or: [ {states:[${state1}]}, {results:[${result1}]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_pending_failed2.out && + numlines=$(cat pending.ids failed.ids | wc -l) && + test $(cat list_constraint_pending_failed2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list inactive (1)' ' + state1=`${JOB_CONV} strtostate INACTIVE` && + constraint="{ or: [ {states:[${state1}]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_inactive1.out && + numlines=$(cat inactive.ids | wc -l) && + test $(cat list_constraint_inactive1.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list inactive (2)' ' + state1=`${JOB_CONV} strtostate SCHED` && + state2=`${JOB_CONV} strtostate RUN` && + constraint="{ not: [ { or: [ {states:[${state1}]}, {states:[${state2}]} ] } ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_inactive2.out && + numlines=$(cat inactive.ids | wc -l) && + test $(cat list_constraint_inactive2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list have run via t_run (1)' ' + constraint="{ or: [ {t_run:[\">=0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_run1.out && + numlines=$(cat running.ids failed.ids timeout.ids completed.ids | wc -l) && + test $(cat list_constraint_t_run1.out | wc -l) -eq ${numlines} +' + +# use a floating point in this one +test_expect_success 'flux job list have run via t_run (2)' ' + constraint="{ or: [ {t_run:[\">=1.0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_run2.out && + numlines=$(cat running.ids failed.ids timeout.ids completed.ids | wc -l) && + test $(cat list_constraint_t_run2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list have run via t_run (3)' ' + constraint="{ or: [ {t_run:[\">1.1\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_run3.out && + numlines=$(cat running.ids failed.ids timeout.ids completed.ids | wc -l) && + test $(cat list_constraint_t_run3.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list inactive via t_inactive (1)' ' + constraint="{ or: [ {t_inactive:[\">=0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_inactive1.out && + numlines=$(cat inactive.ids | wc -l) && + test $(cat list_constraint_t_inactive1.out | wc -l) -eq ${numlines} +' + +# use a floating point in this one +test_expect_success 'flux job list inactive via t_inactive (2)' ' + constraint="{ or: [ {t_inactive:[\">=1.0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_inactive2.out && + numlines=$(cat inactive.ids | wc -l) && + test $(cat list_constraint_t_inactive2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list inactive via t_inactive (3)' ' + constraint="{ or: [ {t_inactive:[\">1.1\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_t_inactive3.out && + numlines=$(cat inactive.ids | wc -l) && + test $(cat list_constraint_t_inactive3.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list none via t_inactive (1)' ' + constraint="{ or: [ {t_inactive:[\"<0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_none1.out && + test $(cat list_constraint_none1.out | wc -l) -eq 0 +' + +test_expect_success 'flux job list none via t_inactive (2)' ' + constraint="{ or: [ {t_inactive:[\"<=0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_none1.out && + test $(cat list_constraint_none1.out | wc -l) -eq 0 +' + +test_expect_success 'flux job list all via t_depend (1)' ' + constraint="{ or: [ {t_depend:[\">=0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_all1.out && + numlines=$(cat all.ids | wc -l) && + test $(cat list_constraint_all1.out | wc -l) -eq ${numlines} +' + +# use a floating point in this one +test_expect_success 'flux job list all via t_depend (2)' ' + constraint="{ or: [ {t_depend:[\">=1.0\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_all2.out && + numlines=$(cat all.ids | wc -l) && + test $(cat list_constraint_all2.out | wc -l) -eq ${numlines} +' + +test_expect_success 'flux job list all via t_depend (3)' ' + constraint="{ or: [ {t_depend:[\">1.1\"]} ] }" && + $jq -j -c -n "{max_entries:1000, attrs:[], constraint:${constraint}}" \ + | $RPC job-list.list | $jq .jobs | $jq -c '.[]' | $jq .id > list_constraint_all3.out && + numlines=$(cat all.ids | wc -l) && + test $(cat list_constraint_all3.out | wc -l) -eq ${numlines} +' + # with single anonymous queue, queues arrays should be zero length test_expect_success 'job stats lists jobs in correct state (mix)' ' flux job stats | jq -e ".job_states.depend == 0" && @@ -1652,9 +1875,8 @@ test_expect_success 'verify task count preserved across restart' ' # so we check for all the core / expected attributes for the situation test_expect_success 'list request with all attr works (job success)' ' - id=$(id -u) && flux run hostname && - $jq -j -c -n "{max_entries:1, userid:${id}, states:0, results:0, attrs:[\"all\"]}" \ + $jq -j -c -n "{max_entries:1, attrs:[\"all\"]}" \ | $RPC job-list.list | jq ".jobs[0]" > all_success.out && cat all_success.out | jq -e ".id" && cat all_success.out | jq -e ".userid" && @@ -1679,9 +1901,8 @@ test_expect_success 'list request with all attr works (job success)' ' ' test_expect_success 'list request with all attr works (job fail)' ' - id=$(id -u) && ! flux run -N1000 -n1000 hostname && - $jq -j -c -n "{max_entries:1, userid:${id}, states:0, results:0, attrs:[\"all\"]}" \ + $jq -j -c -n "{max_entries:1, attrs:[\"all\"]}" \ | $RPC job-list.list | jq ".jobs[0]" > all_fail.out && cat all_fail.out | jq -e ".id" && cat all_fail.out | jq -e ".userid" && @@ -1830,8 +2051,7 @@ test_expect_success 'list request with empty payload fails with EPROTO(71)' ' ' test_expect_success 'list request with invalid input fails with EPROTO(71) (attrs not an array)' ' name="attrs-not-array" && - id=$(id -u) && - $jq -j -c -n "{max_entries:5, userid:${id}, states:0, results:0, attrs:5}" \ + $jq -j -c -n "{max_entries:5, attrs:5}" \ | $listRPC >${name}.out && cat <<-EOF >${name}.expected && errno 71: invalid payload: attrs must be an array @@ -1840,8 +2060,7 @@ test_expect_success 'list request with invalid input fails with EPROTO(71) (attr ' test_expect_success 'list request with invalid input fails with EINVAL(22) (attrs non-string)' ' name="attr-not-string" && - id=$(id -u) && - $jq -j -c -n "{max_entries:5, userid:${id}, states:0, results:0, attrs:[5]}" \ + $jq -j -c -n "{max_entries:5, attrs:[5]}" \ | $listRPC > ${name}.out && cat <<-EOF >${name}.expected && errno 22: attr has no string value @@ -1850,8 +2069,7 @@ test_expect_success 'list request with invalid input fails with EINVAL(22) (attr ' test_expect_success 'list request with invalid input fails with EINVAL(22) (attrs illegal field)' ' name="field-not-valid" && - id=$(id -u) && - $jq -j -c -n "{max_entries:5, userid:${id}, states:0, results:0, attrs:[\"foo\"]}" \ + $jq -j -c -n "{max_entries:5, attrs:[\"foo\"]}" \ | $listRPC > ${name}.out && cat <<-EOF >${name}.expected && errno 22: foo is not a valid attribute