Update header

config/was.yaml

@@ -14,12 +14,11 @@ modules:
     # the globals dict it uses when evaluating Eval items, so we can tell it to import it here.
     - datetime
 
-# Variables used to request LSST flux in the output catalog
+# We evaluate the Euclid zeropoint here to save time in the output catalog.
 eval_variables:
-    frubin_area:
+    feuclid_AB_ZP: 
         type : Eval
-        str : '3.14159 * (418**2 - 255**2)'
-    frubin_exptime: 30
+        str : 'euclidlike.getBandpasses()["VIS"].zeropoint'
 
 # Define some other information about the images
 image:
@@ -77,6 +76,10 @@ image:
     filter: { type: ObSeqData, field: filter }
     exptime: { type: ObSeqData, field: exptime }
 
+    # This is not a required field for the processing but this is the easiest way to propagate
+    # this information to the output images.
+    obs_id: { type: ObSeqData, field: obs_id }
+
     # Photon shooting is way faster for chromatic objects than fft, especially when most of them
     # are fairly faint.  The cross-over point for achromatic objects is generally of order
     # flux=1.e6 or so (depending on the profile).  Most of our objects here are much fainter than
@@ -126,7 +129,7 @@ input:
         file_name: /path/to/skyCatalog.yaml
         edge_pix: 512
         mjd: { type: ObSeqData, field: mjd }
-        exptime: { type: ObSeqData, field: exptime }          
+        exptime: { type: ObSeqData, field: exptime }
         obj_types: ['diffsky_galaxy','star','snana']
 
 output:
@@ -135,51 +138,56 @@ output:
     dir: /path/to/output/images
     file_name:
         type: FormattedStr
-        format: "Euclidlike_WAS_truth_%s_%i_%i.fits.gz"
+        format: "EUC_ELSIM_SWL_DET-%s-%s-1-%s__%sZ.fits"
         items:
-            - { type: ObSeqData, field: filter }
-            - { type: ObSeqData, field: visit }
-            - '@image.CCD'
+            - { type: Eval, iobsId: { type: ObSeqData, field: obs_id }, str: 'str(obsId).zfill(6)' }
+            - { type: Eval, idither: { type: ObSeqData, field: dither_id }, str: 'str(dither).zfill(2)' }
+            - { type: Eval, sccd: '@image.CCD', str: 'ccd.zfill(7)' }
+            - { type: Eval, sdate: { type: ObSeqData, field: date }, str: 'datetime.datetime.fromisoformat(date).strftime("%Y%m%dT%H%M%S.%f")' }
 
     noise_image:
         dir: /path/to/output/noise_images
         file_name:
             type: FormattedStr
-            format: "Euclidlike_WAS_truth_%s_%i_%i.noise.fits.gz"
+            format: "EUC_ELSIM_SWL_VAR-%s-%s-1-%s__%sZ.fits"
             items:
-                - { type: ObSeqData, field: filter }
-                - { type: ObSeqData, field: visit }
-                - '@image.CCD'
+                - { type: Eval, iobsId: { type: ObSeqData, field: obs_id }, str: 'str(obsId).zfill(6)' }
+                - { type: Eval, idither: { type: ObSeqData, field: dither_id }, str: 'str(dither).zfill(2)' }
+                - { type: Eval, sccd: '@image.CCD', str: 'ccd.zfill(7)' }
+                - { type: Eval, sdate: { type: ObSeqData, field: date }, str: 'datetime.datetime.fromisoformat(date).strftime("%Y%m%dT%H%M%S.%f")' }
 
     sky_image:
         dir: /path/to/output/sky_images
         file_name:
             type: FormattedStr
-            format: "Euclidlike_WAS_truth_%s_%i_%i.sky.fits.gz"
+            format: "EUC_ELSIM_SWL_BKG-%s-%s-1-%s__%sZ.fits"
             items:
-                - { type: ObSeqData, field: filter }
-                - { type: ObSeqData, field: visit }
-                - '@image.CCD'
+                - { type: Eval, iobsId: { type: ObSeqData, field: obs_id }, str: 'str(obsId).zfill(6)' }
+                - { type: Eval, idither: { type: ObSeqData, field: dither_id }, str: 'str(dither).zfill(2)' }
+                - { type: Eval, sccd: '@image.CCD', str: 'ccd.zfill(7)' }
+                - { type: Eval, sdate: { type: ObSeqData, field: date }, str: 'datetime.datetime.fromisoformat(date).strftime("%Y%m%dT%H%M%S.%f")' }
 
     weight_image:
         dir: /path/to/output/weight_images
         file_name:
             type: FormattedStr
-            format: "Euclidlike_WAS_truth_%s_%i_%i.weight.fits.gz"
+            format: "EUC_ELSIM_SWL_WGT-%s-%s-1-%s__%sZ.fits"
             items:
-                - { type: ObSeqData, field: filter }
-                - { type: ObSeqData, field: visit }
-                - '@image.CCD'
+                - { type: Eval, iobsId: { type: ObSeqData, field: obs_id }, str: 'str(obsId).zfill(6)' }
+                - { type: Eval, idither: { type: ObSeqData, field: dither_id }, str: 'str(dither).zfill(2)' }
+                - { type: Eval, sccd: '@image.CCD', str: 'ccd.zfill(7)' }
+                - { type: Eval, sdate: { type: ObSeqData, field: date }, str: 'datetime.datetime.fromisoformat(date).strftime("%Y%m%dT%H%M%S.%f")' }
 
     truth:
         dir: /path/to/output/truth
         file_name:
             type: FormattedStr
-            format: "Euclidlike_WAS_index_%s_%i_%i.txt"
+            format: "EUC_ELSIM_SWL_WGT-%s-%s-1-%s__%sZ.txt"
             items:
-                - { type: ObSeqData, field: filter }
-                - { type: ObSeqData, field: visit }
-                - '@image.CCD'
+                - { type: Eval, iobsId: { type: ObSeqData, field: obs_id }, str: 'str(obsId).zfill(6)' }
+                - { type: Eval, idither: { type: ObSeqData, field: dither_id }, str: 'str(dither).zfill(2)' }
+                - { type: Eval, sccd: '@image.CCD', str: 'ccd.zfill(7)' }
+                - { type: Eval, sdate: { type: ObSeqData, field: date }, str: 'datetime.datetime.fromisoformat(date).strftime("%Y%m%dT%H%M%S.%f")' }
         columns:
             object_id: "@object_id"
             ra: "$sky_pos.ra.deg"
@@ -189,15 +197,40 @@ output:
             realized_flux: "@realized_flux"
             flux: "@flux"
             mag: "@mag"
+            mag_AB: { type: Eval, str: '-2.5 * np.log10(@flux/@image.exptime/euclidlike.collecting_area) + euclid_AB_ZP' }
             obj_type: "@object_type"
-            gal_disk_hlr: { type: SkyCatValue, field: diskHalfLightRadiusArcsec }
             gal_redshift: { type: SkyCatValue, field: redshift }
+            gal_disk_hlr: { type: SkyCatValue, field: diskHalfLightRadiusArcsec }
+            gal_disk_e1: { type: SkyCatValue, field: diskEllipticity1 }
+            gal_disk_e2: { type: SkyCatValue, field: diskEllipticity2 }
+            gal_bulge_hlr: { type: SkyCatValue, field: spheroidHalfLightRadiusArcsec }
+            gal_bulge_e1: { type: SkyCatValue, field: spheroidEllipticity1 }
+            gal_bulge_e2: { type: SkyCatValue, field: spheroidEllipticity2 }
+            gal_shear1: { type: SkyCatValue, field: shear1 }
+            gal_shear2: { type: SkyCatValue, field: shear2 }
+            gal_convergence: { type: SkyCatValue, field: convergence }
             sn_host_id: { type: SkyCatValue, field: host_id }
-            lsst_flux_r:
+            lsst_flux_u: { type: SkyCatValue, field: lsst_flux_u }
+            lsst_mag_u:
+                type : Eval
+                str : "-2.5 * np.log10(@output.truth.columns.lsst_flux_u) + 12.628"
+            lsst_flux_g: { type: SkyCatValue, field: lsst_flux_g }
+            lsst_mag_g:
                 type : Eval
-                fflux : { type: SkyCatValue, field: lsst_flux_r }
-                str : 'flux * rubin_exptime * rubin_area'
+                str : "-2.5 * np.log10(@output.truth.columns.lsst_flux_g) + 14.486"
+            lsst_flux_r: { type: SkyCatValue, field: lsst_flux_r }
             lsst_mag_r:
                 type : Eval
-                fflux : { type: SkyCatValue, field: lsst_flux_r }
-                str : "-2.5 * np.log10(@output.truth.columns.lsst_flux_r) + 28.36"
+                str : "-2.5 * np.log10(@output.truth.columns.lsst_flux_r) + 14.325"
+            lsst_flux_i: { type: SkyCatValue, field: lsst_flux_i }
+            lsst_mag_i:
+                type : Eval
+                str : "-2.5 * np.log10(@output.truth.columns.lsst_flux_i) + 13.999"
+            lsst_flux_z: { type: SkyCatValue, field: lsst_flux_z }
+            lsst_mag_z:
+                type : Eval
+                str : "-2.5 * np.log10(@output.truth.columns.lsst_flux_z) + 13.613"
+            lsst_flux_y: { type: SkyCatValue, field: lsst_flux_y }
+            lsst_mag_y:
+                type : Eval
+                str : "-2.5 * np.log10(@output.truth.columns.lsst_flux_y) + 12.785"

euclidlike/instrument_params.py

@@ -21,7 +21,7 @@
 short_exptime_vis = 95 # s (for the shorter exposures with VIS taken in parallel with NISP imaging)
 short_exptime_vis_eff = 89.52  # s (effective time VIS imaging exposures) (This is the one to use for flux computation)
 read_noise = 4.4  # e-, https://www.euclid-ec.org/public/mission/vis/
-saturation = 200000 # e-, from https://www.euclid-ec.org/public/mission/vis/
+saturation = 65535  # ADU, https://arxiv.org/pdf/2503.15303 Sect. 3.3 (max for uint32)
 n_dithers = 4
 n_ccd = 36
 n_ccd_row = 6
@@ -124,6 +124,9 @@
 vis_blue_limit = 540
 vis_red_limit = 910
 
+# Coadd info
+coadd_zeropoint = 23.9  # Euclid zero-point in AB mag, used in all bands
+
 # Items to potentially do later; part of the galsim.roman setup that currently has no correspondence
 # here.
 #   dark_current
@@ -138,5 +141,4 @@
 # separately, so there is no parameter for setting them (e.g. a Gaussian sigma) in
 # this module.  However, pointing errors / jitter are quantified in the VIS paper.
 #
-# The impact of the dichroic and of charge transfer inefficiency are not included.  Same for
-# saturation (200000 e-).
+# The impact of the dichroic and of charge transfer inefficiency are not included.

euclidlike_imsim/ccd.py

@@ -4,14 +4,28 @@
 from galsim.config.image_scattered import ScatteredImageBuilder
 from galsim.errors import GalSimConfigValueError
 from galsim.image import Image
+from galsim.config.value import ParseValue
 from astropy.time import Time
 import numpy as np
 
 import euclidlike
-from euclidlike.instrument_params import gain, saturation
+from euclidlike.instrument_params import gain, saturation, coadd_zeropoint, read_noise
 from .noise import cfg_noise_key, parse_noise_config, get_noise
 
 
+def get_flux_scaling(current_zp, target_zp=coadd_zeropoint):
+    """Get the scaling factor to apply to the flux to get the target zeropoint.
+
+    Parameters:
+        current_zp:   The current zeropoint.
+        target_zp:    The target zeropoint.
+
+    Returns:
+        The scaling factor to apply to the flux.
+    """
+    return 10.0 ** ((target_zp - current_zp) * 0.4)
+
+
 class EuclidlikeCCDImageBuilder(ScatteredImageBuilder):
     def setup(self, config, base, image_num, obj_num, ignore, logger):
         """Do the initialization and setup for building the image.
@@ -52,6 +66,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger):
         req = {"CCD": int, "filter": str, "mjd": float, "exptime": float}
         opt = {
             "draw_method": str,
+            "obs_id": int,
         }
         opt.update({key: bool for key in cfg_noise_key})
         params = galsim.config.GetAllParams(
@@ -77,6 +92,14 @@ def setup(self, config, base, image_num, obj_num, ignore, logger):
                 base["image"], "bandpass", base, logger=logger
             )
 
+        # Get obs_id
+        # NOTE: I don't understand why but I have to explicitly parse the value
+        # it is not recognize during the GetAllParams call
+        if "obs_id" in config:
+            self.obs_id = ParseValue(config, "obs_id", base, int)[0]
+        else:
+            self.obs_id = 0
+
         return euclidlike.n_pix_col, euclidlike.n_pix_row
 
     def buildImage(self, config, base, image_num, obj_num, logger):
@@ -102,15 +125,28 @@ def buildImage(self, config, base, image_num, obj_num, logger):
         full_image.setZero()
 
         full_image.header = galsim.FitsHeader()
-        full_image.header["EXPTIME"] = self.exptime
         full_image.header["MJD-OBS"] = self.mjd
         full_image.header["DATE-OBS"] = str(
             Time(self.mjd, format="mjd").datetime
         )
         full_image.header["FILTER"] = self.filter
         full_image.header["GAIN"] = gain
-        full_image.header["ZPTMAG"] = 2.5 * np.log10(
-            self.exptime * euclidlike.collecting_area
+        if self.cfg_noise["read_noise"]:
+            full_image.header["RDNOISE"] = read_noise
+        else:
+            full_image.header["RDNOISE"] = 0.0
+        full_image.header["EXPTIME"] = self.exptime
+        full_image.header["SATURATE"] = saturation
+        full_image.header["MAGZEROP"] = (
+            2.5 * np.log10(euclidlike.collecting_area)
+            - 2.5 * np.log10(gain)
+            + base["bandpass"].zeropoint
+        )
+        current_zeropoint = full_image.header["MAGZEROP"] + 2.5 * np.log10(
+            full_image.header["EXPTIME"]
+        )
+        full_image.header["FLXSCALE"] = get_flux_scaling(
+            current_zeropoint, coadd_zeropoint
         )
 
         base["current_image"] = full_image
@@ -211,10 +247,9 @@ def addNoise(
         image += base["noise_image"]
 
         # Apply saturation
-        saturation_ADU = np.round(saturation / gain)
-        mask_saturated = image.array > saturation_ADU
+        mask_saturated = image.array > saturation
         base["saturated_mask"] = mask_saturated
-        image.array[mask_saturated] = saturation_ADU
+        image.array[mask_saturated] = saturation
 
         if self.cfg_noise["sky_subtract"]:
             image -= base["sky_image"]

euclidlike_imsim/noise.py

@@ -151,6 +151,7 @@ def initialize(self, data, scratch, config, base, logger):
             "stamp_xsize",
             "stamp_ysize",
             "nobjects",
+            "obs_id",
         ]
         params = galsim.config.GetAllParams(
             base["image"], base, req=req, opt=opt, ignore=ignore+extra_ignore,

euclidlike_imsim/obseq.py

@@ -60,11 +60,13 @@ def read_obseq(self):
             _ob = {}
             _ob["visit"] = self.visit
             _ob["ccd"] = self.ccd
+            _ob["obs_id"] = ob.loc[obs_kind]["obs_id"]
+            _ob["dither_id"] = ob.loc[obs_kind]["dither_id"]
             _ob["ra"] = ob.loc[obs_kind]["ra"] * galsim.degrees
             _ob["dec"] = ob.loc[obs_kind]["dec"] * galsim.degrees
             _ob["pa"] = ob.loc[obs_kind]["pa"] * galsim.degrees
             _ob["saa"] = ob.loc[obs_kind]["saa"] * galsim.degrees
-            _ob["date"] = Time(ob.loc[obs_kind]["date"], format="mjd").datetime
+            _ob["date"] = str(Time(ob.loc[obs_kind]["date"], format="mjd").datetime)
             _ob["mjd"] = ob.loc[obs_kind]["date"]
             _ob["filter"] = ob.loc[obs_kind]["filter"]
             _ob["exptime"] = ob.loc[obs_kind]["exptime"]