Skip to content

Commit 3fd084a

Browse files
chu11chu11
committed
job-list: support constraint -> sql conversion
Problem: In the near future we would like to query jobs from the job db. In order to do so efficiently, we need to convert the job list constraint into equivalent SQL that can be used in a WHERE statement. Support a new helper library "constraint_sql" that supports converting a constraint into conditions that can be passed in via an SQL WHERE. Add unit tests as well.
1 parent f671194 commit 3fd084a

File tree

4 files changed

+871
-1
lines changed

4 files changed

+871
-1
lines changed

‎src/modules/job-list/Makefile.am‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,16 @@ libjob_list_la_SOURCES = \
3838
state_match.c \
3939
match_util.h \
4040
match_util.c \
41+
constraint_sql.h \
42+
constraint_sql.c \
4143
util.h \
4244
util.c
4345

4446
TESTS = \
4547
test_job_data.t \
4648
test_match.t \
47-
test_state_match.t
49+
test_state_match.t \
50+
test_constraint_sql.t
4851

4952
test_ldadd = \
5053
$(builddir)/libjob-list.la \
@@ -93,6 +96,14 @@ test_state_match_t_LDADD = \
9396
test_state_match_t_LDFLAGS = \
9497
$(test_ldflags)
9598

99+
test_constraint_sql_t_SOURCES = test/constraint_sql.c
100+
test_constraint_sql_t_CPPFLAGS = \
101+
$(test_cppflags)
102+
test_constraint_sql_t_LDADD = \
103+
$(test_ldadd)
104+
test_constraint_sql_t_LDFLAGS = \
105+
$(test_ldflags)
106+
96107
EXTRA_DIST = \
97108
test/R/1node_1core.R \
98109
test/R/1node_4core.R \

‎src/modules/job-list/constraint_sql.c‎

Lines changed: 342 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,342 @@
1+
/************************************************************\
2+
* Copyright 2023 Lawrence Livermore National Security, LLC
3+
* (c.f. AUTHORS, NOTICE.LLNS, COPYING)
4+
*
5+
* This file is part of the Flux resource manager framework.
6+
* For details, see https://github.com/flux-framework.
7+
*
8+
* SPDX-License-Identifier: LGPL-3.0
9+
\************************************************************/
10+
11+
#if HAVE_CONFIG_H
12+
#include "config.h"
13+
#endif
14+
15+
#include <stdlib.h>
16+
#include <string.h>
17+
#include <errno.h>
18+
#include <jansson.h>
19+
20+
#include "src/common/libutil/errprintf.h"
21+
#include "ccan/str/str.h"
22+
23+
#include "constraint_sql.h"
24+
#include "match_util.h"
25+
26+
static int create_userid_query (json_t *values, char **query, flux_error_t *errp)
27+
{
28+
json_t *entry;
29+
size_t index;
30+
char *q = NULL;
31+
json_array_foreach (values, index, entry) {
32+
uint32_t userid;
33+
if (!json_is_integer (entry)) {
34+
errprintf (errp, "userid value must be an integer");
35+
goto error;
36+
}
37+
userid = json_integer_value (entry);
38+
/* special case FLUX_USERID_UNKNOWN matches all, so no query result */
39+
if (userid == FLUX_USERID_UNKNOWN)
40+
continue;
41+
if (!q) {
42+
if (asprintf (&q, "userid = %u", userid) < 0) {
43+
errno = ENOMEM;
44+
goto error;
45+
}
46+
}
47+
else {
48+
char *tmp;
49+
if (asprintf (&tmp, "%s OR userid = %u", q, userid) < 0) {
50+
errno = ENOMEM;
51+
goto error;
52+
}
53+
free (q);
54+
q = tmp;
55+
}
56+
}
57+
(*query) = q;
58+
return 0;
59+
error:
60+
free (q);
61+
return -1;
62+
}
63+
64+
static int create_string_query (json_t *values,
65+
const char *op,
66+
char **query,
67+
flux_error_t *errp)
68+
{
69+
json_t *entry;
70+
size_t index;
71+
char *q = NULL;
72+
json_array_foreach (values, index, entry) {
73+
const char *str;
74+
if (!json_is_string (entry)) {
75+
errprintf (errp, "%s value must be a string", op);
76+
goto error;
77+
}
78+
str = json_string_value (entry);
79+
if (!q) {
80+
if (asprintf (&q, "%s = '%s'", op, str) < 0) {
81+
errno = ENOMEM;
82+
goto error;
83+
}
84+
}
85+
else {
86+
char *tmp;
87+
if (asprintf (&tmp, "%s OR %s = '%s'", q, op, str) < 0) {
88+
errno = ENOMEM;
89+
goto error;
90+
}
91+
free (q);
92+
q = tmp;
93+
}
94+
}
95+
(*query) = q;
96+
return 0;
97+
error:
98+
free (q);
99+
return -1;
100+
}
101+
102+
static int create_name_query (json_t *values,
103+
char **query,
104+
flux_error_t *errp)
105+
{
106+
return create_string_query (values, "name", query, errp);
107+
}
108+
109+
static int create_queue_query (json_t *values,
110+
char **query,
111+
flux_error_t *errp)
112+
{
113+
return create_string_query (values, "queue", query, errp);
114+
}
115+
116+
static int create_bitmask_query (const char *col,
117+
json_t *values,
118+
array_to_bitmask_f array_to_bitmask_cb,
119+
char **query,
120+
flux_error_t *errp)
121+
{
122+
char *q = NULL;
123+
int tmp;
124+
if ((tmp = array_to_bitmask_cb (values, errp)) < 0)
125+
return -1;
126+
if (asprintf (&q, "(%s & %d) > 0", col, tmp) < 0) {
127+
errno = ENOMEM;
128+
return -1;
129+
}
130+
(*query) = q;
131+
return 0;
132+
}
133+
134+
static int create_states_query (json_t *values,
135+
char **query,
136+
flux_error_t *errp)
137+
{
138+
return create_bitmask_query ("state",
139+
values,
140+
array_to_states_bitmask,
141+
query,
142+
errp);
143+
}
144+
145+
static int create_results_query (json_t *values,
146+
char **query,
147+
flux_error_t *errp)
148+
{
149+
return create_bitmask_query ("result",
150+
values,
151+
array_to_results_bitmask,
152+
query,
153+
errp);
154+
}
155+
156+
static int create_timestamp_query (const char *type,
157+
json_t *values,
158+
char **query,
159+
flux_error_t *errp)
160+
{
161+
const char *str;
162+
const char *comp;
163+
char *q = NULL;
164+
double t;
165+
char *endptr;
166+
json_t *v = json_array_get (values, 0);
167+
168+
if (!v) {
169+
errprintf (errp, "timestamp value not specified");
170+
return -1;
171+
}
172+
if (!json_is_string (v)) {
173+
errprintf (errp, "%s value must be a string", type);
174+
return -1;
175+
}
176+
str = json_string_value (v);
177+
if (strstarts (str, ">=")) {
178+
comp = ">=";
179+
str += 2;
180+
}
181+
else if (strstarts (str, "<=")) {
182+
comp = "<=";
183+
str += 2;
184+
}
185+
else if (strstarts (str, ">")) {
186+
comp = ">";
187+
str +=1;
188+
}
189+
else if (strstarts (str, "<")) {
190+
comp = "<";
191+
str += 1;
192+
}
193+
else {
194+
errprintf (errp, "timestamp comparison operator not specified");
195+
return -1;
196+
}
197+
198+
errno = 0;
199+
t = strtod (str, &endptr);
200+
if (errno != 0 || *endptr != '\0') {
201+
errprintf (errp, "Invalid timestamp value specified");
202+
return -1;
203+
}
204+
if (t < 0.0) {
205+
errprintf (errp, "timestamp value must be >= 0.0");
206+
return -1;
207+
}
208+
209+
if (asprintf (&q, "%s %s %s", type, comp, str) < 0) {
210+
errno = ENOMEM;
211+
return -1;
212+
}
213+
214+
(*query) = q;
215+
return 0;
216+
}
217+
218+
static int conditional_query (const char *type,
219+
json_t *values,
220+
char **query,
221+
flux_error_t *errp)
222+
{
223+
char *q = NULL;
224+
char *cond;
225+
json_t *entry;
226+
size_t index;
227+
228+
if (streq (type, "and"))
229+
cond = "AND";
230+
else if (streq (type, "or"))
231+
cond = "OR";
232+
else /* streq (type, "not") */
233+
/* we will "NOT" it at the end */
234+
cond = "AND";
235+
236+
json_array_foreach (values, index, entry) {
237+
char *subquery;
238+
if (constraint2sql (entry, &subquery, errp) < 0)
239+
goto error;
240+
if (!q)
241+
q = subquery;
242+
else if (subquery) {
243+
char *tmp;
244+
if (asprintf (&tmp, "%s %s %s", q, cond, subquery) < 0) {
245+
free (subquery);
246+
errno = ENOMEM;
247+
goto error;
248+
}
249+
free (q);
250+
q = tmp;
251+
}
252+
}
253+
if (q && streq (type, "not")) {
254+
char *tmp;
255+
if (asprintf (&tmp, "NOT (%s)", q) < 0) {
256+
errno = ENOMEM;
257+
goto error;
258+
}
259+
free (q);
260+
q = tmp;
261+
}
262+
(*query) = q;
263+
return 0;
264+
265+
error:
266+
free (q);
267+
return -1;
268+
}
269+
270+
int constraint2sql (json_t *constraint, char **query, flux_error_t *errp)
271+
{
272+
char *q = NULL;
273+
char *rv = NULL;
274+
const char *op;
275+
json_t *values;
276+
277+
if (!query)
278+
return -1;
279+
280+
if (!constraint)
281+
return 0;
282+
283+
if (!json_is_object (constraint)) {
284+
errprintf (errp, "constraint must be JSON object");
285+
return -1;
286+
}
287+
if (json_object_size (constraint) > 1) {
288+
errprintf (errp, "constraint must only contain 1 element");
289+
return -1;
290+
}
291+
json_object_foreach (constraint, op, values) {
292+
int ret;
293+
if (!json_is_array (values)) {
294+
errprintf (errp, "operator %s values not an array", op);
295+
goto error;
296+
}
297+
if (streq (op, "userid"))
298+
ret = create_userid_query (values, &q, errp);
299+
else if (streq (op, "name"))
300+
ret = create_name_query (values, &q, errp);
301+
else if (streq (op, "queue"))
302+
ret = create_queue_query (values, &q, errp);
303+
else if (streq (op, "states"))
304+
ret = create_states_query (values, &q, errp);
305+
else if (streq (op, "results"))
306+
ret = create_results_query (values, &q, errp);
307+
else if (streq (op, "hostlist")) {
308+
/* no hostlist column, no conversion to be done */
309+
ret = 0;
310+
}
311+
else if (streq (op, "t_submit")
312+
|| streq (op, "t_depend")
313+
|| streq (op, "t_run")
314+
|| streq (op, "t_cleanup")
315+
|| streq (op, "t_inactive"))
316+
ret = create_timestamp_query (op, values, &q, errp);
317+
else if (streq (op, "or") || streq (op, "and") || streq (op, "not"))
318+
ret = conditional_query (op, values, &q, errp);
319+
else {
320+
errprintf (errp, "unknown constraint operator: %s", op);
321+
goto error;
322+
}
323+
if (ret < 0)
324+
goto error;
325+
}
326+
if (q) {
327+
if (asprintf (&rv, "(%s)", q) < 0) {
328+
free (q);
329+
errno = ENOMEM;
330+
goto error;
331+
}
332+
}
333+
(*query) = rv;
334+
return 0;
335+
336+
error:
337+
free (q);
338+
return -1;
339+
}
340+
341+
/* vi: ts=4 sw=4 expandtab
342+
*/

‎src/modules/job-list/constraint_sql.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/************************************************************\
2+
* Copyright 2023 Lawrence Livermore National Security, LLC
3+
* (c.f. AUTHORS, NOTICE.LLNS, COPYING)
4+
*
5+
* This file is part of the Flux resource manager framework.
6+
* For details, see https://github.com/flux-framework.
7+
*
8+
* SPDX-License-Identifier: LGPL-3.0
9+
\************************************************************/
10+
11+
#ifndef HAVE_JOB_LIST_CONSTRAINT_SQL_H
12+
#define HAVE_JOB_LIST_CONSTRAINT_SQL_H 1
13+
14+
#if HAVE_CONFIG_H
15+
#include "config.h"
16+
#endif
17+
18+
#include <flux/core.h>
19+
#include <jansson.h>
20+
21+
int constraint2sql (json_t *constraint, char **query, flux_error_t *errp);
22+
23+
#endif /* !HAVE_JOB_LIST_MATCH_H */

0 commit comments

Comments
 (0)