Merge pull request #5710 from garlick/issue#5708

job-ingest: fix FLUID initialization error handling to allow scaling beyond 16K brokers

Author: mergify[bot]

src/modules/job-ingest/job-ingest.c

@@ -852,8 +852,19 @@ int mod_main (flux_t *h, int argc, char **argv)
         }
         flux_future_destroy (f);
         if (fluid_init (&ctx.gen, 0, fluid_get_timestamp (max_jobid) + 1) < 0) {
-            flux_log (h, LOG_ERR, "fluid_init failed");
-            errno = EINVAL;
+            // See flux-framework/flux-core#5708
+            if (rank > 16383) {
+                flux_log (h,
+                          LOG_DEBUG,
+                          "job-ingest cannot allocate job IDs on ranks > 16383."
+                          " Exiting - upstream will handle ingest requests.");
+                rc = 0;
+            }
+            else {
+                flux_log (h, LOG_ERR, "fluid_init failed");
+                errno = EINVAL;
+            }
+            goto done;
         }
     }
     else {
@@ -873,12 +884,10 @@ int mod_main (flux_t *h, int argc, char **argv)
             goto done;
         }
         flux_future_destroy (f);
-        /* fluid_init() will fail on rank > 16K.
-         * Just skip loading the job module on those ranks.
-         */
         if (fluid_init (&ctx.gen, rank, timestamp) < 0) {
             flux_log (h, LOG_ERR, "fluid_init failed");
             errno = EINVAL;
+            goto done;
         }
     }
     flux_log (h, LOG_DEBUG, "fluid ts=%jums", (uint64_t)ctx.gen.timestamp);