job-manager: do not send purged events

Problem: If a job has been purged from the job-manager, events
in the journal may still exist for those purged jobs.  They can
be sent to journal requestors.

Solution: When a new request is made for the journal, check that
the events in the journal are still valid before sending them.

Fixes #4331

Author: chu11

src/modules/job-manager/journal.c

@@ -185,15 +185,22 @@ static void journal_handle_request (flux_t *h,
     wrapped_entry = zlist_first (journal->events);
     while (wrapped_entry) {
         const char *name;
+        flux_jobid_t id;
 
         if (json_unpack (wrapped_entry,
-                         "{s:{s:s}}",
+                         "{s:I s:{s:s}}",
+                         "id", &id,
                          "entry",
                            "name", &name) < 0) {
             flux_log (h, LOG_ERR, "invalid wrapped entry");
             goto error;
         }
 
+        /* ensure job has not been purged */
+        if (!zhashx_lookup (ctx->active_jobs, &id)
+            && !zhashx_lookup (ctx->inactive_jobs, &id))
+            goto next;
+
         if (allow_deny_check (msg, name)) {
             if (!a) {
                 if (!(a = json_array ()))
@@ -202,6 +209,7 @@ static void journal_handle_request (flux_t *h,
             if (json_array_append (a, wrapped_entry) < 0)
                 goto nomem;
         }
+next:
         wrapped_entry = zlist_next (journal->events);
     }
 

t/Makefile.am

@@ -290,6 +290,7 @@ dist_check_SCRIPTS = \
 	issues/t4182-resource-rerank.sh \
 	issues/t4184-sched-simple-restart.sh \
 	issues/t4222-kvs-assume-empty-dir.sh \
+	issues/t4331-job-manager-purged-events.sh \
 	python/__init__.py \
 	python/subflux.py \
 	python/tap \

t/issues/t4331-job-manager-purged-events.sh

@@ -0,0 +1,59 @@
+#!/bin/bash -e
+
+# test-prereqs: HAVE_JQ
+
+EVENTS_JOURNAL_STREAM=${FLUX_BUILD_DIR}/t/job-manager/events_journal_stream
+
+# wait for jobid in events file to avoid race
+# arg1 - file
+# arg2 - jobid
+wait_jobid_event() {
+    local file=$1
+    local jobid=$2
+    for i in `seq 1 30`
+    do
+        if grep -q ${jobid} ${file}
+        then
+            break
+        fi
+        sleep 1
+    done
+    if [ "${i}" -eq "30" ]
+    then
+        return 1
+    fi
+    return 0
+}
+
+jobid1=`flux mini submit --wait hostname`
+jobid2=`flux mini submit --wait hostname`
+
+jq -j -c -n "{}" | $EVENTS_JOURNAL_STREAM > events1.out &
+pid1=$!
+
+jobid1dec=`flux job id --to=dec ${jobid1}`
+jobid2dec=`flux job id --to=dec ${jobid2}`
+
+wait_jobid_event events1.out ${jobid2dec}
+
+# kill background process
+kill -s SIGUSR1 ${pid1}
+
+# jobid1 completed first, so if jobid2 is in events journal, jobid1
+# should be too
+
+grep ${jobid1dec} events1.out
+
+flux job purge --force --num-limit=1
+
+jq -j -c -n "{}" | $EVENTS_JOURNAL_STREAM > events2.out &
+pid2=$!
+
+wait_jobid_event events2.out ${jobid2dec}
+
+# kill background process
+kill -s SIGUSR1 ${pid2}
+
+# jobid1 should be purged now and not in the events journal
+
+! grep ${jobid1dec} events2.out