Skip to content

job-list: support job list constraints #5126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mergify merged 5 commits into from
Jul 25, 2023
Merged

job-list: support job list constraints #5126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
93c7a61
libjob: support virtual states in conversions
chu11 Apr 25, 2023
0bf85d7
job-list: add job-list matching library
chu11 Apr 21, 2023
bce7253
job-list: add state matching library
chu11 Apr 29, 2023
4ffc8b3
job-list: use constraints to filter jobs
chu11 Apr 28, 2023
589a8f2
testsuite: add more job-list constraint tests
chu11 May 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions src/bindings/python/flux/job/list.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,31 @@ def job_list(
name=None,
queue=None,
):
# N.B. an "and" operation with no values returns everything
constraint = {"and": []}
if userid != flux.constants.FLUX_USERID_UNKNOWN:
constraint["and"].append({"userid": [userid]})
if name:
constraint["and"].append({"name": [name]})
if queue:
constraint["and"].append({"queue": [queue]})
if states and results:
if states & flux.constants.FLUX_JOB_STATE_INACTIVE:
states &= ~flux.constants.FLUX_JOB_STATE_INACTIVE
tmp = {"or": []}
tmp["or"].append({"states": [states]})
tmp["or"].append({"results": [results]})
constraint["and"].append(tmp)
elif states:
constraint["and"].append({"states": [states]})
elif results:
constraint["and"].append({"results": [results]})
payload = {
"max_entries": int(max_entries),
"attrs": attrs,
"userid": int(userid),
"states": states,
"results": results,
"since": since,
"constraint": constraint,
}
if name:
payload["name"] = name
if queue:
payload["queue"] = queue
return JobListRPC(flux_handle, "job-list.list", payload)


Expand Down Expand Up @@ -240,8 +253,6 @@ def add_filter(self, fname):
if fname in self.STATES:
self.states |= self.STATES[fname]
elif fname in self.RESULTS:
# Must specify "inactive" to get results:
self.states |= self.STATES["inactive"]
self.results |= self.RESULTS[fname]
else:
raise ValueError(f"Invalid filter specified: {fname}")
Expand Down
25 changes: 16 additions & 9 deletions src/cmd/flux-job.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1262,6 +1262,7 @@ int cmd_list (optparse_t *p, int argc, char **argv)
json_t *value;
uint32_t userid;
int states = 0;
json_t *c;

if (isatty (STDOUT_FILENO)) {
fprintf (stderr,
Expand Down Expand Up @@ -1290,16 +1291,20 @@ int cmd_list (optparse_t *p, int argc, char **argv)
else
userid = getuid ();

if (!(c = json_pack ("{ s:[ {s:[i]}, {s:[i]} ] }",
"and",
"userid", userid,
"states", states)))
log_msg_exit ("failed to construct constraint object");

if (!(f = flux_rpc_pack (h,
"job-list.list",
FLUX_NODEID_ANY,
0,
"{s:i s:[s] s:i s:i s:i}",
"{s:i s:[s] s:o}",
"max_entries", max_entries,
"attrs", "all",
"userid", userid,
"states", states,
"results", 0)))
"constraint", c)))
log_err_exit ("flux_rpc_pack");
if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0)
log_err_exit ("flux job-list.list");
Expand Down Expand Up @@ -1327,6 +1332,7 @@ int cmd_list_inactive (optparse_t *p, int argc, char **argv)
json_t *jobs;
size_t index;
json_t *value;
json_t *c;

if (isatty (STDOUT_FILENO)) {
fprintf (stderr,
Expand All @@ -1341,17 +1347,18 @@ int cmd_list_inactive (optparse_t *p, int argc, char **argv)
if (!(h = flux_open (NULL, 0)))
log_err_exit ("flux_open");

if (!(c = json_pack ("{s:[i]}", "states", FLUX_JOB_STATE_INACTIVE)))
log_msg_exit ("failed to construct constraint object");

if (!(f = flux_rpc_pack (h,
"job-list.list",
FLUX_NODEID_ANY,
0,
"{s:i s:f s:i s:i s:i s:[s]}",
"{s:i s:f s:[s] s:o}",
"max_entries", max_entries,
"since", since,
"userid", FLUX_USERID_UNKNOWN,
"states", FLUX_JOB_STATE_INACTIVE,
"results", 0,
"attrs", "all")))
"attrs", "all",
"constraint", c)))
log_err_exit ("flux_rpc_pack");
if (flux_rpc_get_unpack (f, "{s:o}", "jobs", &jobs) < 0)
log_err_exit ("flux job-list.list");
Expand Down
5 changes: 2 additions & 3 deletions src/cmd/top/joblist_pane.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,11 +357,10 @@ void joblist_pane_query (struct joblist_pane *joblist)
"job-list.list",
0,
0,
"{s:i s:i s:i s:i s:[s,s,s,s,s,s,s,s]}",
"{s:i s:{s:[i]} s:[s,s,s,s,s,s,s,s]}",
"max_entries", win_dim.y_length - 1,
"userid", FLUX_USERID_UNKNOWN,
"constraint",
"states", FLUX_JOB_STATE_RUNNING,
"results", 0,
"attrs",
"annotations",
"userid",
Expand Down
31 changes: 22 additions & 9 deletions src/common/libjob/list.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ flux_future_t *flux_job_list (flux_t *h,
{
flux_future_t *f;
json_t *o = NULL;
json_t *c = NULL;
int valid_states = (FLUX_JOB_STATE_PENDING
| FLUX_JOB_STATE_RUNNING
| FLUX_JOB_STATE_INACTIVE);
Expand All @@ -37,15 +38,22 @@ flux_future_t *flux_job_list (flux_t *h,
errno = EINVAL;
return NULL;
}
if (!(c = json_pack ("{ s:[ {s:[i]}, {s:[i]} ] }",
"and",
"userid", userid,
"states", states ? states : valid_states))) {
json_decref (o);
errno = ENOMEM;
return NULL;
}
if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0,
"{s:i s:o s:i s:i s:i}",
"{s:i s:o s:o}",
"max_entries", max_entries,
"attrs", o,
"userid", userid,
"states", states,
"results", 0))) {
"constraint", c))) {
saved_errno = errno;
json_decref (o);
json_decref (c);
errno = saved_errno;
return NULL;
}
Expand All @@ -59,23 +67,28 @@ flux_future_t *flux_job_list_inactive (flux_t *h,
{
flux_future_t *f;
json_t *o = NULL;
json_t *c = NULL;
int saved_errno;

if (!h || max_entries < 0 || since < 0. || !attrs_json_str
|| !(o = json_loads (attrs_json_str, 0, NULL))) {
errno = EINVAL;
return NULL;
}
if (!(c = json_pack ("{s:[i]}", "states", FLUX_JOB_STATE_INACTIVE))) {
json_decref (o);
errno = ENOMEM;
return NULL;
}
if (!(f = flux_rpc_pack (h, "job-list.list", FLUX_NODEID_ANY, 0,
"{s:i s:f s:i s:i s:i s:o}",
"{s:i s:f s:o s:o}",
"max_entries", max_entries,
"since", since,
"userid", FLUX_USERID_UNKNOWN,
"states", FLUX_JOB_STATE_INACTIVE,
"results", 0,
"attrs", o))) {
"attrs", o,
"constraint", c))) {
saved_errno = errno;
json_decref (o);
json_decref (c);
errno = saved_errno;
return NULL;
}
Expand Down
3 changes: 3 additions & 0 deletions src/common/libjob/state.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ static struct strtab states[] = {
{ FLUX_JOB_STATE_RUN, "RUN", "run", "R", "r" },
{ FLUX_JOB_STATE_CLEANUP, "CLEANUP", "cleanup", "C", "c" },
{ FLUX_JOB_STATE_INACTIVE, "INACTIVE", "inactive", "I", "i" },
{ FLUX_JOB_STATE_PENDING, "PENDING", "pending", "PD", "pd" },
{ FLUX_JOB_STATE_RUNNING, "RUNNING", "running", "RU", "ru" },
{ FLUX_JOB_STATE_ACTIVE, "ACTIVE", "active", "A", "a" },
};


Expand Down
3 changes: 3 additions & 0 deletions src/common/libjob/test/job.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,9 @@ struct ss sstab[] = {
{ FLUX_JOB_STATE_RUN, "R", "RUN", "r", "run" },
{ FLUX_JOB_STATE_CLEANUP, "C", "CLEANUP", "c", "cleanup" },
{ FLUX_JOB_STATE_INACTIVE, "I", "INACTIVE", "i", "inactive" },
{ FLUX_JOB_STATE_PENDING, "PD", "PENDING", "pd", "pending" },
{ FLUX_JOB_STATE_RUNNING, "RU", "RUNNING", "ru", "running" },
{ FLUX_JOB_STATE_ACTIVE, "A", "ACTIVE", "a", "active" },
{ -1, NULL, NULL, NULL, NULL },
};

Expand Down
9 changes: 4 additions & 5 deletions src/modules/job-archive/job-archive.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,19 +506,18 @@ void job_archive_cb (flux_reactor_t *r,
"job-list.list",
FLUX_NODEID_ANY,
0,
"{s:i s:f s:i s:i s:i s:[ssssss]}",
"{s:i s:f s:[ssssss] s:{s:[i]}}",
"max_entries", 0,
"since", ctx->since,
"userid", FLUX_USERID_UNKNOWN,
"states", FLUX_JOB_STATE_INACTIVE,
"results", 0,
"attrs",
"userid",
"ranks",
"t_submit",
"t_run",
"t_cleanup",
"t_inactive"))) {
"t_inactive",
"constraint",
"states", FLUX_JOB_STATE_INACTIVE))) {
flux_log_error (ctx->h, "%s: flux_rpc_pack", __FUNCTION__);
return;
}
Expand Down
28 changes: 26 additions & 2 deletions src/modules/job-list/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,18 @@ libjob_list_la_SOURCES = \
idsync.h \
idsync.c \
stats.h \
stats.c
stats.c \
Copy link
Contributor

@grondo grondo May 19, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Commit message: "add an intially job matching library" -> "add an initial job matching library" ?

match.h \
match.c \
state_match.h \
state_match.c \
match_util.h \
match_util.c

TESTS = \
test_job_data.t
test_job_data.t \
test_match.t \
test_state_match.t

test_ldadd = \
$(builddir)/libjob-list.la \
Expand Down Expand Up @@ -65,6 +73,22 @@ test_job_data_t_LDADD = \
test_job_data_t_LDFLAGS = \
$(test_ldflags)

test_match_t_SOURCES = test/match.c
test_match_t_CPPFLAGS = \
$(test_cppflags)
test_match_t_LDADD = \
$(test_ldadd)
test_match_t_LDFLAGS = \
$(test_ldflags)

test_state_match_t_SOURCES = test/state_match.c
test_state_match_t_CPPFLAGS = \
$(test_cppflags)
test_state_match_t_LDADD = \
$(test_ldadd)
test_state_match_t_LDFLAGS = \
$(test_ldflags)

EXTRA_DIST = \
test/R/1node_1core.R \
test/R/1node_4core.R \
Expand Down
Loading