Skip to content

Commit b46a315

Browse files
chu11chu11
committed
testsuite: add job-list database tests
1 parent 43220c4 commit b46a315

File tree

2 files changed

+392
-0
lines changed

2 files changed

+392
-0
lines changed

‎t/Makefile.am‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ TESTSCRIPTS = \
137137
t2250-job-archive.t \
138138
t2260-job-list.t \
139139
t2261-job-list-update.t \
140+
t2262-job-list-db.t \
140141
t2270-job-dependencies.t \
141142
t2271-job-dependency-after.t \
142143
t2272-job-begin-time.t \

‎t/t2262-job-list-db.t‎

Lines changed: 391 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,391 @@
1+
#!/bin/sh
2+
3+
test_description='Test flux job list db'
4+
5+
. $(dirname $0)/job-list/job-list-helper.sh
6+
7+
. $(dirname $0)/sharness.sh
8+
9+
export FLUX_CONF_DIR=$(pwd)
10+
if test -z "${TEST_UNDER_FLUX_ACTIVE}"; then
11+
STATEDIR=$(mktemp -d)
12+
fi
13+
test_under_flux 4 job -o,-Sstatedir=${STATEDIR}
14+
15+
RPC=${FLUX_BUILD_DIR}/t/request/rpc
16+
17+
QUERYCMD="flux python ${FLUX_SOURCE_DIR}/t/scripts/sqlite-query.py"
18+
19+
fj_wait_event() {
20+
flux job wait-event --timeout=20 "$@"
21+
}
22+
23+
get_statedir_dbpath() {
24+
statedir=`flux getattr statedir` &&
25+
echo ${statedir}/job-db.sqlite
26+
}
27+
28+
# count number of entries in database
29+
# arg1 - database path
30+
db_count_entries() {
31+
local dbpath=$1
32+
query="select id from jobs;"
33+
count=`${QUERYCMD} -t 10000 ${dbpath} "${query}" | grep "^id =" | wc -l`
34+
echo $count
35+
}
36+
37+
# verify entries stored in database
38+
# arg1 - jobid
39+
# arg2 - database path
40+
db_check_entries() {
41+
local id=$(flux job id $1)
42+
local dbpath=$2
43+
query="select * from jobs where id=$id;"
44+
${QUERYCMD} -t 10000 ${dbpath} "${query}" > query.out
45+
if grep -q "^id = " query.out \
46+
&& grep -q "t_inactive = " query.out \
47+
&& grep -q "jobdata = " query.out \
48+
&& grep -q "eventlog = " query.out \
49+
&& grep -q "jobspec = " query.out \
50+
&& grep -q "R = " query.out
51+
then
52+
return 0
53+
fi
54+
return 1
55+
}
56+
57+
# get job values via job list
58+
# arg1 - jobid
59+
# arg2 - database path
60+
get_job_list_values() {
61+
local id=$(flux job id $1)
62+
jobdata=`flux job list-ids ${id}`
63+
list_userid=`echo ${jobdata} | jq ".userid"`
64+
list_urgency=`echo ${jobdata} | jq ".urgency"`
65+
list_priority=`echo ${jobdata} | jq ".priority"`
66+
list_state=`echo ${jobdata} | jq ".state"`
67+
list_ranks=`echo ${jobdata} | jq ".ranks"`
68+
list_nnodes=`echo ${jobdata} | jq ".nnodes"`
69+
list_nodelist=`echo ${jobdata} | jq -r ".nodelist"`
70+
list_ntasks=`echo ${jobdata} | jq ".ntasks"`
71+
list_name=`echo ${jobdata} | jq -r ".name"`
72+
list_waitstatus=`echo ${jobdata} | jq ".waitstatus"`
73+
list_success=`echo ${jobdata} | jq ".success"`
74+
list_result=`echo ${jobdata} | jq ".result"`
75+
list_expiration=`echo ${jobdata} | jq ".expiration"`
76+
list_annotations=`echo ${jobdata} | jq ".annotations"`
77+
list_dependencies=`echo ${jobdata} | jq ".dependencies"`
78+
list_exception_occurred=`echo ${jobdata} | jq ".exception_occurred"`
79+
list_exception_type=`echo ${jobdata} | jq -r ".exception_type"`
80+
list_exception_severity=`echo ${jobdata} | jq ".exception_severity"`
81+
list_exception_note=`echo ${jobdata} | jq -r ".exception_note"`
82+
list_t_submit=`echo ${jobdata} | jq ".t_submit"`
83+
list_t_run=`echo ${jobdata} | jq ".t_run"`
84+
list_t_cleanup=`echo ${jobdata} | jq ".t_cleanup"`
85+
list_t_inactive=`echo ${jobdata} | jq ".t_inactive"`
86+
}
87+
88+
# get job values from database
89+
# arg1 - jobid
90+
# arg2 - database path
91+
get_db_values() {
92+
local id=$(flux job id $1)
93+
local dbpath=$2
94+
query="select * from jobs where id=$id;"
95+
${QUERYCMD} -t 10000 ${dbpath} "${query}" > query.out
96+
db_jobdata=`grep "jobdata = " query.out | cut -f3- -d' '`
97+
db_eventlog=`grep "eventlog = " query.out | awk '{print \$3}'`
98+
db_jobspec=`grep "jobspec = " query.out | awk '{print \$3}'`
99+
db_R=`grep "R = " query.out | awk '{print \$3}'`
100+
db_userid=`echo ${jobdata} | jq ".userid"`
101+
db_urgency=`echo ${jobdata} | jq ".urgency"`
102+
db_priority=`echo ${jobdata} | jq ".priority"`
103+
db_state=`echo ${jobdata} | jq ".state"`
104+
db_ranks=`echo ${jobdata} | jq ".ranks"`
105+
db_nnodes=`echo ${jobdata} | jq ".nnodes"`
106+
db_nodelist=`echo ${jobdata} | jq -r ".nodelist"`
107+
db_ntasks=`echo ${jobdata} | jq ".ntasks"`
108+
db_name=`echo ${jobdata} | jq -r ".name"`
109+
db_waitstatus=`echo ${jobdata} | jq ".waitstatus"`
110+
db_success=`echo ${jobdata} | jq ".success"`
111+
db_result=`echo ${jobdata} | jq ".result"`
112+
db_expiration=`echo ${jobdata} | jq ".expiration"`
113+
db_annotations=`echo ${jobdata} | jq ".annotations"`
114+
db_dependencies=`echo ${jobdata} | jq ".dependencies"`
115+
db_exception_occurred=`echo ${jobdata} | jq ".exception_occurred"`
116+
db_exception_type=`echo ${jobdata} | jq -r ".exception_type"`
117+
db_exception_severity=`echo ${jobdata} | jq ".exception_severity"`
118+
db_exception_note=`echo ${jobdata} | jq -r ".exception_note"`
119+
db_t_submit=`echo ${jobdata} | jq ".t_submit"`
120+
db_t_run=`echo ${jobdata} | jq ".t_run"`
121+
db_t_cleanup=`echo ${jobdata} | jq ".t_cleanup"`
122+
db_t_inactive=`echo ${jobdata} | jq ".t_inactive"`
123+
}
124+
125+
# compare data from job list and job db
126+
# arg1 - job id
127+
# arg2 - dbpath
128+
db_compare_data() {
129+
local id=$(flux job id $1)
130+
local dbpath=$2
131+
get_job_list_values ${id}
132+
get_db_values ${id} ${dbpath}
133+
if [ "${list_userid}" != "${db_userid}" ] \
134+
|| [ "${list_urgency}" != "${db_urgency}" ] \
135+
|| [ "${list_priority}" != "${db_priority}" ] \
136+
|| [ "${list_state}" != "${db_state}" ] \
137+
|| [ "${list_ranks}" != "${db_ranks}" ] \
138+
|| [ "${list_nnodes}" != "${db_nnodes}" ] \
139+
|| [ "${list_nodelist}" != "${db_nodelist}" ] \
140+
|| [ "${list_ntasks}" != "${db_ntasks}" ] \
141+
|| [ "${list_name}" != "${db_name}" ] \
142+
|| [ "${list_waitstatus}" != "${db_waitstatus}" ] \
143+
|| [ "${list_success}" != "${db_success}" ] \
144+
|| [ "${list_result}" != "${db_result}" ] \
145+
|| [ "${list_expiration}" != "${db_expiration}" ] \
146+
|| [ "${list_annotations}" != "${db_annotations}" ] \
147+
|| [ "${list_dependencies}" != "${db_dependencies}" ] \
148+
|| [ "${list_exception_occurred}" != "${db_exception_occurred}" ] \
149+
|| [ "${list_exception_type}" != "${db_exception_type}" ] \
150+
|| [ "${list_exception_severity}" != "${db_exception_severity}" ] \
151+
|| [ "${list_exception_note}" != "${db_exception_note}" ] \
152+
|| [ "${list_t_submit}" != "${db_t_submit}" ] \
153+
|| [ "${list_t_run}" != "${db_t_run}" ] \
154+
|| [ "${list_t_cleanup}" != "${db_t_cleanup}" ] \
155+
|| [ "${list_t_inactive}" != "${db_t_inactive}" ]
156+
then
157+
return 1
158+
fi
159+
return 0
160+
}
161+
162+
# wait for inactive job list to reach expected count
163+
# arg1 - expected final count
164+
# Usage: wait_inactive_count method target tries
165+
# where method is job-manager, job-list, or job-list-stats (jq required)
166+
wait_inactive_list_count() {
167+
local target=$1
168+
local tries=50
169+
local count
170+
while test $tries -gt 0; do
171+
count=$(flux module stats -p jobs.inactive job-list)
172+
test $count -eq $target && return 0
173+
sleep 0.25
174+
tries=$(($tries-1))
175+
done
176+
return 1
177+
}
178+
179+
# submit jobs for job list & job-list db testing
180+
181+
test_expect_success 'submit jobs for job list testing' '
182+
# Create `hostname` jobspec
183+
# N.B. Used w/ `flux job submit` for serial job submission
184+
# for efficiency (vs serial `flux mini submit`.
185+
#
186+
flux mini submit --dry-run hostname >hostname.json &&
187+
#
188+
# submit jobs that will complete
189+
#
190+
for i in $(seq 0 3); do
191+
flux job submit hostname.json >> inactiveids
192+
fj_wait_event `tail -n 1 inactiveids` clean
193+
done &&
194+
#
195+
# Currently all inactive ids are "completed"
196+
#
197+
tac inactiveids | flux job id > completed.ids &&
198+
#
199+
# Run a job that will fail, copy its JOBID to both inactive and
200+
# failed lists.
201+
#
202+
! jobid=`flux mini submit --wait nosuchcommand` &&
203+
echo $jobid >> inactiveids &&
204+
flux job id $jobid > failed.ids &&
205+
touch pending.ids &&
206+
touch running.ids &&
207+
tac inactiveids | flux job id > inactive.ids &&
208+
cat inactive.ids > all.ids &&
209+
#
210+
# Synchronize all expected states
211+
#
212+
job_list_wait_states
213+
'
214+
215+
test_expect_success HAVE_JQ 'flux job list inactive jobs (pre purge)' '
216+
flux job list -s inactive | jq .id > listI.out &&
217+
test_cmp listI.out inactive.ids
218+
'
219+
220+
test_expect_success HAVE_JQ 'flux job list inactive jobs w/ count (pre purge)' '
221+
count=$(job_list_state_count inactive) &&
222+
count=$((count - 2)) &&
223+
flux job list -s inactive -c ${count} | jq .id > listI_count.out &&
224+
head -n ${count} inactive.ids > listI_count.exp &&
225+
test_cmp listI_count.out listI_count.exp
226+
'
227+
228+
test_expect_success HAVE_JQ 'flux job list-inactive jobs (pre purge)' '
229+
flux job list-inactive | jq .id > list_inactive.out &&
230+
test_cmp list_inactive.out inactive.ids
231+
'
232+
233+
test_expect_success HAVE_JQ 'flux job list-inactive jobs w/ count (pre purge)' '
234+
count=$(job_list_state_count inactive) &&
235+
count=$((count - 2)) &&
236+
flux job list-inactive -c ${count} | jq .id > list_inactive_count.out &&
237+
head -n ${count} inactive.ids > list_inactive_count.exp &&
238+
test_cmp list_inactive_count.out list_inactive_count.exp
239+
'
240+
241+
test_expect_success 'flux job list-ids jobs (pre purge)' '
242+
for id in `cat inactive.ids`
243+
do
244+
flux job list-ids ${id}
245+
done
246+
'
247+
248+
test_expect_success 'flux job list has all inactive jobs cached' '
249+
cached=`flux module stats -p jobs.inactive job-list` &&
250+
test ${cached} -eq $(job_list_state_count inactive)
251+
'
252+
253+
test_expect_success 'job-list db: db stored in statedir' '
254+
dbpath=$(get_statedir_dbpath) &&
255+
ls ${dbpath}
256+
'
257+
258+
test_expect_success 'job-list db: has correct number of entries' '
259+
dbpath=$(get_statedir_dbpath) &&
260+
entries=$(db_count_entries ${dbpath}) &&
261+
test ${entries} -eq $(job_list_state_count inactive)
262+
'
263+
264+
test_expect_success 'job-list db: make sure job data looks ok' '
265+
dbpath=$(get_statedir_dbpath) &&
266+
for id in `cat list_inactive.out`
267+
do
268+
db_check_entries ${id} ${dbpath}
269+
done
270+
'
271+
272+
test_expect_success HAVE_JQ 'job-list db: make sure job data correct' '
273+
dbpath=$(get_statedir_dbpath) &&
274+
for id in `cat list_inactive.out`
275+
do
276+
db_compare_data ${id} ${dbpath}
277+
done
278+
'
279+
280+
test_expect_success 'job-list db: purge 2 jobs' '
281+
len=$(job_list_state_count inactive) &&
282+
len=$((len - 2)) &&
283+
flux job purge --force --num-limit=${len} &&
284+
mv inactive.ids inactive.ids.orig &&
285+
head -n ${len} inactive.ids.orig > inactive.ids &&
286+
wait_inactive_list_count ${len}
287+
'
288+
289+
test_expect_success HAVE_JQ 'flux job list inactive jobs (post purge)' '
290+
flux job list -s inactive | jq .id > listI2.out &&
291+
test_cmp listI2.out inactive.ids.orig
292+
'
293+
294+
test_expect_success HAVE_JQ 'flux job list inactive jobs w/ count (post purge)' '
295+
count=$(cat inactive.ids.orig | wc -l) &&
296+
count=$((count - 2)) &&
297+
flux job list -s inactive -c ${count} | jq .id > listI_count2.out &&
298+
head -n ${count} inactive.ids.orig > listI_count2.exp &&
299+
test_cmp listI_count2.out listI_count2.exp
300+
'
301+
302+
test_expect_success HAVE_JQ 'flux job list-inactive jobs (post purge)' '
303+
flux job list-inactive | jq .id > list_inactive2.out &&
304+
test_cmp list_inactive2.out inactive.ids.orig
305+
'
306+
307+
test_expect_success HAVE_JQ 'flux job list-inactive jobs w/ count (post purge)' '
308+
count=$(cat inactive.ids.orig | wc -l) &&
309+
count=$((count - 2)) &&
310+
flux job list-inactive -c ${count} | jq .id > list_inactive_count2.out &&
311+
head -n ${count} inactive.ids.orig > list_inactive_count2.exp &&
312+
test_cmp list_inactive_count2.out list_inactive_count2.exp
313+
'
314+
315+
test_expect_success 'flux job list-ids jobs (post purge)' '
316+
for id in `cat inactive.ids.orig`
317+
do
318+
flux job list-ids ${id}
319+
done
320+
'
321+
322+
test_expect_success 'job-list db: purge all jobs' '
323+
len=$(job_list_state_count inactive) &&
324+
flux job purge --force --num-limit=0 &&
325+
: > inactive.ids &&
326+
wait_inactive_list_count 0
327+
'
328+
329+
test_expect_success HAVE_JQ 'flux job list inactive jobs (all purge)' '
330+
flux job list -s inactive | jq .id > listI3.out &&
331+
test_cmp listI3.out inactive.ids.orig
332+
'
333+
334+
test_expect_success HAVE_JQ 'flux job list inactive jobs w/ count (all purge)' '
335+
count=$(cat inactive.ids.orig | wc -l) &&
336+
count=$((count - 2)) &&
337+
flux job list -s inactive -c ${count} | jq .id > listI_count3.out &&
338+
head -n ${count} inactive.ids.orig > listI_count3.exp &&
339+
test_cmp listI_count3.out listI_count3.exp
340+
'
341+
342+
test_expect_success HAVE_JQ 'flux job list-inactive jobs (all purge)' '
343+
flux job list-inactive | jq .id > list_inactive3.out &&
344+
test_cmp list_inactive3.out inactive.ids.orig
345+
'
346+
347+
test_expect_success HAVE_JQ 'flux job list-inactive jobs w/ count (all purge)' '
348+
count=$(cat inactive.ids.orig | wc -l) &&
349+
count=$((count - 2)) &&
350+
flux job list-inactive -c ${count} | jq .id > list_inactive_count3.out &&
351+
head -n ${count} inactive.ids.orig > list_inactive_count3.exp &&
352+
test_cmp list_inactive_count3.out list_inactive_count3.exp
353+
'
354+
355+
test_expect_success 'flux job list-ids jobs (all purge)' '
356+
for id in `cat inactive.ids.orig`
357+
do
358+
flux job list-ids ${id}
359+
done
360+
'
361+
362+
test_expect_success 'flux job db stats works' '
363+
${RPC} job-list.db-stats 0 < /dev/null
364+
'
365+
366+
test_expect_success 'reload the job-list module with alternate config' '
367+
statedir=`flux getattr statedir` &&
368+
cat >job-list.toml <<EOF &&
369+
[job-list]
370+
dbpath = "${statedir}/testdb.sqlite"
371+
busytimeout = "0.1s"
372+
EOF
373+
flux config reload &&
374+
flux module reload job-list
375+
'
376+
377+
test_expect_success 'configured db is there' '
378+
ls ${statedir}/testdb.sqlite
379+
'
380+
381+
test_expect_success 'flux job list inactive jobs (new db)' '
382+
count=$(flux job list -s inactive | wc -l) &&
383+
test ${count} -eq 0
384+
'
385+
386+
test_expect_success 'flux job list-inactive jobs (new db)' '
387+
count=$(flux job list-inactive | wc -l) &&
388+
test ${count} -eq 0
389+
'
390+
391+
test_done

0 commit comments

Comments
 (0)