metadata HDF5 concurancy issue

[kslong]

This may just be a Mac issues, but I am running into problems when running multiple exposures from the same day in parallel. The problem occurs becuase run_drp runs update_meta data and so if there two version of run drp are started at the same time these operate operate on the same file.

This does not happen of course if run_drp receives a list of exposures only if it receives two simultaneous attempts to update the hdf file at the same time.

There are ways to lock the file to prevent the error, but there may be other negative implications of doing this, like getting multiple versions of the same information in the file:

The location of the error is on line 367 of metadata.py

stores.append(h5py.File(metadata_path, mode=mode))

The error one receives looks like this:

^[[0;31m[ERROR]: ^[[0mTraceback (most recent call last):
  File ^[[36m"/Users/long/anaconda3/envs/lvmdrp_test/bin/drp"^[[39;49;00m, line ^[[34m7^[[39;49;00m, in <module>^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mexec(^[[36mcompile^[[39;49;00m(f.read(), ^[[31m__file__^[[39;49;00m, ^[[33m'^[[39;49;00m^[[33mexec^[[39;49;00m^[[33m'^[[39;49;00m))^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/bin/drp"^[[39;49;00m, line ^[[34m675^[[39;49;00m, in <module>^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mcli()^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/anaconda3/envs/lvmdrp_test/lib/python3.10/site-packages/click/core.py"^[[39;49;00m, line ^[[34m1157^[[39;49;00m, in __call__^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00m^[[34mreturn^[[39;49;00m ^[[36mself^[[39;49;00m.main(*args, **kwargs)^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/anaconda3/envs/lvmdrp_test/lib/python3.10/site-packages/click/core.py"^[[39;49;00m, line ^[[34m1078^[[39;49;00m, in main^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mrv = ^[[36mself^[[39;49;00m.invoke(ctx)^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/anaconda3/envs/lvmdrp_test/lib/python3.10/site-packages/click/core.py"^[[39;49;00m, line ^[[34m1688^[[39;49;00m, in invoke^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00m^[[34mreturn^[[39;49;00m _process_result(sub_ctx.command.invoke(sub_ctx))^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/anaconda3/envs/lvmdrp_test/lib/python3.10/site-packages/click/core.py"^[[39;49;00m, line ^[[34m1434^[[39;49;00m, in invoke^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00m^[[34mreturn^[[39;49;00m ctx.invoke(^[[36mself^[[39;49;00m.callback, **ctx.params)^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/anaconda3/envs/lvmdrp_test/lib/python3.10/site-packages/click/core.py"^[[39;49;00m, line ^[[34m783^[[39;49;00m, in invoke^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00m^[[34mreturn^[[39;49;00m __callback(*args, **kwargs)^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/bin/drp"^[[39;49;00m, line ^[[34m169^[[39;49;00m, in run^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mrun_drp(mjd=mjd, expnum=expnum, no_sci=no_sci, with_cals=with_cals, fluxcal_method=fluxcal_method,^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/python/lvmdrp/main.py"^[[39;49;00m, line ^[[34m1876^[[39;49;00m, in run_drp^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mrun_drp(mjd=mjd, expnum=expnum, with_cals=with_cals, no_sci=no_sci,^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/python/lvmdrp/main.py"^[[39;49;00m, line ^[[34m1908^[[39;49;00m, in run_drp^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mupdate_metadata(mjd=mjd)^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/python/lvmdrp/utils/metadata.py"^[[39;49;00m, line ^[[34m990^[[39;49;00m, in update_metadata^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mextract_metadata(new_paths)^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/python/lvmdrp/utils/metadata.py"^[[39;49;00m, line ^[[34m677^[[39;49;00m, in extract_metadata^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00madd_raws(new_metadata)^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/python/lvmdrp/utils/metadata.py"^[[39;49;00m, line ^[[34m705^[[39;49;00m, in add_raws^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mstore = _load_or_create_store(tileid=tileid, mjd=mjd, kind=^[[33m"^[[39;49;00m^[[33mraw^[[39;49;00m^[[33m"^[[39;49;00m, mode=^[[33m"^[[39;49;00m^[[33ma^[[39;49;00m^[[33m"^[[39;49;00m)[^[[34m0^[[39;49;00m]^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/SDSS/lvmdrp/python/lvmdrp/utils/metadata.py"^[[39;49;00m, line ^[[34m367^[[39;49;00m, in _load_or_create_store^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mstores.append(h5py.File(metadata_path, mode=mode))^[[37m^[[39;49;00m
  File ^[[36m"/Users/long/anaconda3/envs/lvmdrp_test/lib/python3.10/site-packages/h5py/_hl/files.py"^[[39;49;00m, line ^[[34m562^[[39;49;00m, in __init__^[[37m^[[39;49;00m
^[[37m    ^[[39;49;00mfid = make_fid(name, mode, userblock_size, fapl, fcpl, swmr=swmr)^[[37m^[[39;49;00m

[kslong]

My workaround was for running n jobs in parallel with the same mjd at once was just to delay the starts by a minute so the hdf updates were separated in time.