From b39be283768beb02e9a85b23bfbfe7415773ba6b Mon Sep 17 00:00:00 2001 From: Anatole Storck Date: Wed, 23 Apr 2025 10:23:02 +0100 Subject: [PATCH 01/20] first working version of RT fields in MEGATRON --- yt/frontends/ramses/data_structures.py | 4 +++ yt/frontends/ramses/field_handlers.py | 7 +++++ yt/frontends/ramses/io_utils.pyx | 39 ++++++++++++++++++-------- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/yt/frontends/ramses/data_structures.py b/yt/frontends/ramses/data_structures.py index 4918d31ea2a..38cf6992cfc 100644 --- a/yt/frontends/ramses/data_structures.py +++ b/yt/frontends/ramses/data_structures.py @@ -429,6 +429,9 @@ def _fill_no_ghostzones(self, fd, fields, selector, file_handler): # Initializing data container for field in fields: data[field] = np.zeros(cell_count, "float64") + + # Get the size of the data, single precision if RT field, double otherwise + field_size = "single" if any(field.startswith("Photon") for field in fields) else "double" # Do an early exit if the cell count is null if cell_count == 0: @@ -452,6 +455,7 @@ def _fill_no_ghostzones(self, fd, fields, selector, file_handler): fields, data, oct_handler, + field_size # Hardcoded for now ) return data diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index fafd18b5967..087e1cb95a8 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -251,6 +251,11 @@ def offset(self): *structure* of your fluid file is non-canonical, change this. """ nvars = len(self._detected_field_list[self.ds.unique_identifier]) + + first_field = self.field_list[0][1] + # Get the size of the data, single precision if RT field, double otherwise + field_size = "single" if first_field.startswith("Photon") else "double" + with FortranFile(self.fname) as fd: # Skip headers nskip = len(self.attrs) @@ -269,6 +274,7 @@ def offset(self): # # So there are 8 * nvars records each with length (nocts, ) # at each (level, cpus) + offset, level_count = read_offset( fd, min_level, @@ -276,6 +282,7 @@ def offset(self): self.parameters[self.ds.unique_identifier]["nvar"], self.domain.amr_header, Nskip=nvars * 8, + size=field_size, ) self._level_count = level_count diff --git a/yt/frontends/ramses/io_utils.pyx b/yt/frontends/ramses/io_utils.pyx index 805c6e020d4..ebfb37b045d 100644 --- a/yt/frontends/ramses/io_utils.pyx +++ b/yt/frontends/ramses/io_utils.pyx @@ -19,9 +19,13 @@ ctypedef np.float64_t DOUBLE_t cdef INT64_t INT32_SIZE = sizeof(np.int32_t) cdef INT64_t INT64_SIZE = sizeof(np.int64_t) cdef INT64_t DOUBLE_SIZE = sizeof(np.float64_t) +cdef INT64_t SINGLE_SIZE = sizeof(np.float32_t) -cdef inline INT64_t skip_len(INT64_t Nskip, INT64_t record_len) noexcept nogil: +cdef inline int skip_len(int Nskip, int record_len, str size="double") noexcept nogil: + # If the data is single precision, we need to skip 4 bytes + if size == "single": + return Nskip * (record_len * SINGLE_SIZE + INT64_SIZE) return Nskip * (record_len * DOUBLE_SIZE + INT64_SIZE) @@ -111,7 +115,7 @@ def read_amr(FortranFile f, dict headers, @cython.wraparound(False) @cython.cdivision(True) @cython.nonecheck(False) -cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t nvar, dict headers, int Nskip): +cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t nvar, dict headers, int Nskip, str size="double"): cdef np.ndarray[np.int64_t, ndim=2] offset, level_count cdef INT64_t ndim, twotondim, nlevelmax, n_levels, nboundary, ncpu, ncpu_and_bound @@ -153,7 +157,7 @@ cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t n if ilevel >= min_level: offset_view[icpu, ilevel - min_level] = f.tell() level_count_view[icpu, ilevel - min_level] = file_ncache - f.seek(skip_len(Nskip, file_ncache), SEEK_CUR) + f.seek(skip_len(Nskip, file_ncache, size), SEEK_CUR) return offset, level_count @@ -172,7 +176,9 @@ def fill_hydro(FortranFile f, INT64_t ndim, list all_fields, list fields, dict tr, RAMSESOctreeContainer oct_handler, - np.ndarray[np.int32_t, ndim=1] domain_inds=np.array([], dtype='int32')): + str size, + np.ndarray[np.int32_t, ndim=1] domain_inds=np.array([], dtype='int32'), + ): cdef INT64_t offset cdef dict tmp cdef str field @@ -197,7 +203,8 @@ def fill_hydro(FortranFile f, # The ordering is very important here, as we'll write directly into the memory # address the content of the files. - cdef np.float64_t[::1, :, :] buffer + cdef np.float32_t[::1, :, :] buffer_single + cdef np.float64_t[::1, :, :] buffer_double jump_len = 0 j = 0 @@ -211,7 +218,11 @@ def fill_hydro(FortranFile f, jumps[j] = jump_len cdef int first_field_index = jumps[0] - buffer = np.empty((level_count.max(), twotondim, nfields_selected), dtype="float64", order='F') + # The buffer is different depending on the size of the data + if size == "single": + buffer_single = np.empty((level_count.max(), twotondim, nfields_selected), dtype="float32", order='F') + else: + buffer_double = np.empty((level_count.max(), twotondim, nfields_selected), dtype="float64", order='F') # Precompute which levels we need to read Ncells = len(level_inds) @@ -231,7 +242,7 @@ def fill_hydro(FortranFile f, offset = offsets[icpu, ilevel] if offset == -1: continue - f.seek(offset + skip_len(first_field_index, nc), SEEK_SET) + f.seek(offset + skip_len(first_field_index, nc, size), SEEK_SET) # We have already skipped the first fields (if any) # so we "rewind" (this will cancel the first seek) @@ -241,9 +252,12 @@ def fill_hydro(FortranFile f, for j in range(nfields_selected): jump_len += jumps[j] if jump_len > 0: - f.seek(skip_len(jump_len, nc), SEEK_CUR) + f.seek(skip_len(jump_len, nc, size), SEEK_CUR) jump_len = 0 - f.read_vector_inplace('d', &buffer[0, i, j]) + if size == "single": + f.read_vector_inplace('f', &buffer_single[0, i, j]) + else: + f.read_vector_inplace('d', &buffer_double[0, i, j]) jump_len += jumps[nfields_selected] @@ -251,12 +265,15 @@ def fill_hydro(FortranFile f, # but since we're doing an absolute seek at the beginning of # the loop on CPUs, we can spare one seek here ## if jump_len > 0: - ## f.seek(skip_len(jump_len, nc), SEEK_CUR) + ## f.seek(skip_len(jump_len, nc, size), SEEK_CUR) # Alias buffer into dictionary tmp = {} for i, field in enumerate(fields): - tmp[field] = buffer[:, :, i] + if size == "single": + tmp[field] = np.asarray(buffer_single[:, :, i], dtype="float64") + else: + tmp[field] = buffer_double[:, :, i] if ncpu_selected > 1: oct_handler.fill_level_with_domain( From ea3fa915e6fb8ca584dc1f61b63153a0f23690bc Mon Sep 17 00:00:00 2001 From: Anatole Storck Date: Wed, 23 Apr 2025 10:53:28 +0100 Subject: [PATCH 02/20] changed field check for single precision to a boolean instead of string --- yt/frontends/ramses/data_structures.py | 4 ++-- yt/frontends/ramses/field_handlers.py | 4 ++-- yt/frontends/ramses/io_utils.pyx | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/yt/frontends/ramses/data_structures.py b/yt/frontends/ramses/data_structures.py index 38cf6992cfc..b8926931baa 100644 --- a/yt/frontends/ramses/data_structures.py +++ b/yt/frontends/ramses/data_structures.py @@ -431,7 +431,7 @@ def _fill_no_ghostzones(self, fd, fields, selector, file_handler): data[field] = np.zeros(cell_count, "float64") # Get the size of the data, single precision if RT field, double otherwise - field_size = "single" if any(field.startswith("Photon") for field in fields) else "double" + is_single = True if any(field.startswith("Photon") for field in fields) else False # Do an early exit if the cell count is null if cell_count == 0: @@ -455,7 +455,7 @@ def _fill_no_ghostzones(self, fd, fields, selector, file_handler): fields, data, oct_handler, - field_size # Hardcoded for now + is_single # Hardcoded for now ) return data diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 087e1cb95a8..587a45a0495 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -254,7 +254,7 @@ def offset(self): first_field = self.field_list[0][1] # Get the size of the data, single precision if RT field, double otherwise - field_size = "single" if first_field.startswith("Photon") else "double" + is_single = True if first_field.startswith("Photon") else False with FortranFile(self.fname) as fd: # Skip headers @@ -282,7 +282,7 @@ def offset(self): self.parameters[self.ds.unique_identifier]["nvar"], self.domain.amr_header, Nskip=nvars * 8, - size=field_size, + single=is_single, ) self._level_count = level_count diff --git a/yt/frontends/ramses/io_utils.pyx b/yt/frontends/ramses/io_utils.pyx index ebfb37b045d..a03976c0c67 100644 --- a/yt/frontends/ramses/io_utils.pyx +++ b/yt/frontends/ramses/io_utils.pyx @@ -22,9 +22,9 @@ cdef INT64_t DOUBLE_SIZE = sizeof(np.float64_t) cdef INT64_t SINGLE_SIZE = sizeof(np.float32_t) -cdef inline int skip_len(int Nskip, int record_len, str size="double") noexcept nogil: +cdef inline int skip_len(int Nskip, int record_len, bint single) noexcept nogil: # If the data is single precision, we need to skip 4 bytes - if size == "single": + if single: return Nskip * (record_len * SINGLE_SIZE + INT64_SIZE) return Nskip * (record_len * DOUBLE_SIZE + INT64_SIZE) @@ -115,7 +115,7 @@ def read_amr(FortranFile f, dict headers, @cython.wraparound(False) @cython.cdivision(True) @cython.nonecheck(False) -cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t nvar, dict headers, int Nskip, str size="double"): +cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t nvar, dict headers, int Nskip, bint single): cdef np.ndarray[np.int64_t, ndim=2] offset, level_count cdef INT64_t ndim, twotondim, nlevelmax, n_levels, nboundary, ncpu, ncpu_and_bound @@ -157,7 +157,7 @@ cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t n if ilevel >= min_level: offset_view[icpu, ilevel - min_level] = f.tell() level_count_view[icpu, ilevel - min_level] = file_ncache - f.seek(skip_len(Nskip, file_ncache, size), SEEK_CUR) + f.seek(skip_len(Nskip, file_ncache, single), SEEK_CUR) return offset, level_count @@ -176,7 +176,7 @@ def fill_hydro(FortranFile f, INT64_t ndim, list all_fields, list fields, dict tr, RAMSESOctreeContainer oct_handler, - str size, + bint single, np.ndarray[np.int32_t, ndim=1] domain_inds=np.array([], dtype='int32'), ): cdef INT64_t offset @@ -219,7 +219,7 @@ def fill_hydro(FortranFile f, cdef int first_field_index = jumps[0] # The buffer is different depending on the size of the data - if size == "single": + if single: buffer_single = np.empty((level_count.max(), twotondim, nfields_selected), dtype="float32", order='F') else: buffer_double = np.empty((level_count.max(), twotondim, nfields_selected), dtype="float64", order='F') @@ -242,7 +242,7 @@ def fill_hydro(FortranFile f, offset = offsets[icpu, ilevel] if offset == -1: continue - f.seek(offset + skip_len(first_field_index, nc, size), SEEK_SET) + f.seek(offset + skip_len(first_field_index, nc, single), SEEK_SET) # We have already skipped the first fields (if any) # so we "rewind" (this will cancel the first seek) @@ -252,9 +252,9 @@ def fill_hydro(FortranFile f, for j in range(nfields_selected): jump_len += jumps[j] if jump_len > 0: - f.seek(skip_len(jump_len, nc, size), SEEK_CUR) + f.seek(skip_len(jump_len, nc, single), SEEK_CUR) jump_len = 0 - if size == "single": + if single: f.read_vector_inplace('f', &buffer_single[0, i, j]) else: f.read_vector_inplace('d', &buffer_double[0, i, j]) @@ -265,12 +265,12 @@ def fill_hydro(FortranFile f, # but since we're doing an absolute seek at the beginning of # the loop on CPUs, we can spare one seek here ## if jump_len > 0: - ## f.seek(skip_len(jump_len, nc, size), SEEK_CUR) + ## f.seek(skip_len(jump_len, nc, single), SEEK_CUR) # Alias buffer into dictionary tmp = {} for i, field in enumerate(fields): - if size == "single": + if single: tmp[field] = np.asarray(buffer_single[:, :, i], dtype="float64") else: tmp[field] = buffer_double[:, :, i] From 4669357fb0f4ea2eeac1ca88757c00e98214ccd3 Mon Sep 17 00:00:00 2001 From: Anatole Storck Date: Fri, 16 May 2025 12:16:40 +0100 Subject: [PATCH 03/20] read energies in rt groups of the info_rt_file_xxxxx.txt --- yt/frontends/ramses/field_handlers.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 587a45a0495..6b6c3398bcf 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -549,10 +549,13 @@ def detect_fields(cls, ds): rheader = {} - def read_rhs(cast): + def read_rhs(cast, group=None): line = f.readline() p, v = line.split("=") - rheader[p.strip()] = cast(v) + if group is not None: + rheader[f"Group {group} {p.strip()}"] = cast(v) + else: + rheader[p.strip()] = cast(v) with open(fname) as f: # Read nRTvar, nions, ngroups, iions @@ -574,7 +577,7 @@ def read_rhs(cast): read_rhs(float) f.readline() - # Reat unit_np, unit_pfd + # Read unit_np, unit_pf for _ in range(2): read_rhs(float) @@ -587,9 +590,20 @@ def read_rhs(cast): # Read n star, t2star, g_star for _ in range(3): read_rhs(float) - - # Touchy part, we have to read the photon group properties - mylog.debug("Not reading photon group properties") + + f.readline() + f.readline() + + # Get global group properties (groupL0, groupL1, spec2group) + for _ in range(3): + read_rhs(lambda line: [float(e) for e in line.split()]) + + # get egy for each group (to get proper energy densities) + for _ in range(4): + group = int(f.readline().split()[1]) + read_rhs(lambda line: [float(e) for e in line.split()], group) + f.readline() + f.readline() cls.rt_parameters[ds.unique_identifier] = rheader From 4454d77ec56e20633a2d664ed73b9f2abd59f5b1 Mon Sep 17 00:00:00 2001 From: Anatole Storck Date: Fri, 16 May 2025 12:20:23 +0100 Subject: [PATCH 04/20] specifit that rt fields are single precision (temp megatron fix) --- yt/frontends/ramses/data_structures.py | 6 +++++- yt/frontends/ramses/io_utils.pyx | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/yt/frontends/ramses/data_structures.py b/yt/frontends/ramses/data_structures.py index b8926931baa..a0923c3a163 100644 --- a/yt/frontends/ramses/data_structures.py +++ b/yt/frontends/ramses/data_structures.py @@ -455,7 +455,7 @@ def _fill_no_ghostzones(self, fd, fields, selector, file_handler): fields, data, oct_handler, - is_single # Hardcoded for now + is_single, ) return data @@ -478,6 +478,9 @@ def _fill_with_ghostzones( # Initializing data container for field in fields: tr[field] = np.zeros(cell_count, "float64") + + # Get the size of the data, single precision if RT field, double otherwise + is_single = True if any(field.startswith("Photon") for field in fields) else False # Do an early exit if the cell count is null if cell_count == 0: @@ -511,6 +514,7 @@ def _fill_with_ghostzones( fields, tr, oct_handler, + is_single, domain_inds=domain_inds, ) return tr diff --git a/yt/frontends/ramses/io_utils.pyx b/yt/frontends/ramses/io_utils.pyx index a03976c0c67..9a959a12f49 100644 --- a/yt/frontends/ramses/io_utils.pyx +++ b/yt/frontends/ramses/io_utils.pyx @@ -22,7 +22,7 @@ cdef INT64_t DOUBLE_SIZE = sizeof(np.float64_t) cdef INT64_t SINGLE_SIZE = sizeof(np.float32_t) -cdef inline int skip_len(int Nskip, int record_len, bint single) noexcept nogil: +cdef inline int skip_len(int Nskip, int record_len, np.npy_bool single) noexcept nogil: # If the data is single precision, we need to skip 4 bytes if single: return Nskip * (record_len * SINGLE_SIZE + INT64_SIZE) @@ -115,7 +115,7 @@ def read_amr(FortranFile f, dict headers, @cython.wraparound(False) @cython.cdivision(True) @cython.nonecheck(False) -cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t nvar, dict headers, int Nskip, bint single): +cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t nvar, dict headers, int Nskip, np.npy_bool single): cdef np.ndarray[np.int64_t, ndim=2] offset, level_count cdef INT64_t ndim, twotondim, nlevelmax, n_levels, nboundary, ncpu, ncpu_and_bound @@ -176,7 +176,7 @@ def fill_hydro(FortranFile f, INT64_t ndim, list all_fields, list fields, dict tr, RAMSESOctreeContainer oct_handler, - bint single, + np.npy_bool single, np.ndarray[np.int32_t, ndim=1] domain_inds=np.array([], dtype='int32'), ): cdef INT64_t offset From f675a7b3a570edcb2648b7bd5959f60a4354b771 Mon Sep 17 00:00:00 2001 From: Anatole Storck Date: Fri, 16 May 2025 16:57:10 +0100 Subject: [PATCH 05/20] there are 8 energy groups in MEGATRON, not 4. --- yt/frontends/ramses/field_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 6b6c3398bcf..e54fff7a0e7 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -599,7 +599,7 @@ def read_rhs(cast, group=None): read_rhs(lambda line: [float(e) for e in line.split()]) # get egy for each group (to get proper energy densities) - for _ in range(4): + for _ in range(8): group = int(f.readline().split()[1]) read_rhs(lambda line: [float(e) for e in line.split()], group) f.readline() From 49854d235cee91bd77a68a2399692226ae72b0de Mon Sep 17 00:00:00 2001 From: Anatole Storck Date: Tue, 27 May 2025 17:09:46 +0100 Subject: [PATCH 06/20] add note: rt field paramater readin only works for up to 8 energy bands --- yt/frontends/ramses/field_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index e54fff7a0e7..f53fa0ded3c 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -590,7 +590,6 @@ def read_rhs(cast, group=None): # Read n star, t2star, g_star for _ in range(3): read_rhs(float) - f.readline() f.readline() @@ -598,7 +597,8 @@ def read_rhs(cast, group=None): for _ in range(3): read_rhs(lambda line: [float(e) for e in line.split()]) - # get egy for each group (to get proper energy densities) + # get egy for each group (used to get proper energy densities) + # NOTE: this assumes there are 8 energy groups for _ in range(8): group = int(f.readline().split()[1]) read_rhs(lambda line: [float(e) for e in line.split()], group) From 9073e6374511ef83771ec1093cad38ad078f099d Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 12:14:05 +0100 Subject: [PATCH 07/20] Do not assume we have 8 energy groups --- yt/frontends/ramses/field_handlers.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index f53fa0ded3c..f08235b326a 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -251,11 +251,11 @@ def offset(self): *structure* of your fluid file is non-canonical, change this. """ nvars = len(self._detected_field_list[self.ds.unique_identifier]) - + first_field = self.field_list[0][1] # Get the size of the data, single precision if RT field, double otherwise is_single = True if first_field.startswith("Photon") else False - + with FortranFile(self.fname) as fd: # Skip headers nskip = len(self.attrs) @@ -274,7 +274,7 @@ def offset(self): # # So there are 8 * nvars records each with length (nocts, ) # at each (level, cpus) - + offset, level_count = read_offset( fd, min_level, @@ -592,18 +592,20 @@ def read_rhs(cast, group=None): read_rhs(float) f.readline() f.readline() - + # Get global group properties (groupL0, groupL1, spec2group) for _ in range(3): read_rhs(lambda line: [float(e) for e in line.split()]) - + # get egy for each group (used to get proper energy densities) - # NOTE: this assumes there are 8 energy groups - for _ in range(8): - group = int(f.readline().split()[1]) + line = f.readline().strip() + while line.startswith("---Group"): + group = int(line.split()[1]) read_rhs(lambda line: [float(e) for e in line.split()], group) + # Skip cross sections weighted by number/energy f.readline() f.readline() + line = f.readline().strip() cls.rt_parameters[ds.unique_identifier] = rheader From e3bc0537adab20fd3adb9d3589ebf0ca67cf2eac Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 12:15:14 +0100 Subject: [PATCH 08/20] Symmetrize skip_len function [cosmetic] --- yt/frontends/ramses/io_utils.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/frontends/ramses/io_utils.pyx b/yt/frontends/ramses/io_utils.pyx index 9a959a12f49..2ffe8f4577d 100644 --- a/yt/frontends/ramses/io_utils.pyx +++ b/yt/frontends/ramses/io_utils.pyx @@ -26,7 +26,8 @@ cdef inline int skip_len(int Nskip, int record_len, np.npy_bool single) noexcept # If the data is single precision, we need to skip 4 bytes if single: return Nskip * (record_len * SINGLE_SIZE + INT64_SIZE) - return Nskip * (record_len * DOUBLE_SIZE + INT64_SIZE) + else: + return Nskip * (record_len * DOUBLE_SIZE + INT64_SIZE) @cython.cpow(True) From e3fff199952a7b7bce30d9755fa1f563a97506fa Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 12:47:12 +0100 Subject: [PATCH 09/20] Generalize support for single precision --- yt/frontends/ramses/data_structures.py | 21 ++-- yt/frontends/ramses/field_handlers.py | 162 ++++++++++++++++++------- yt/frontends/ramses/io_utils.pyx | 30 +++-- 3 files changed, 150 insertions(+), 63 deletions(-) diff --git a/yt/frontends/ramses/data_structures.py b/yt/frontends/ramses/data_structures.py index a0923c3a163..51916f3a4b0 100644 --- a/yt/frontends/ramses/data_structures.py +++ b/yt/frontends/ramses/data_structures.py @@ -429,9 +429,6 @@ def _fill_no_ghostzones(self, fd, fields, selector, file_handler): # Initializing data container for field in fields: data[field] = np.zeros(cell_count, "float64") - - # Get the size of the data, single precision if RT field, double otherwise - is_single = True if any(field.startswith("Photon") for field in fields) else False # Do an early exit if the cell count is null if cell_count == 0: @@ -455,7 +452,7 @@ def _fill_no_ghostzones(self, fd, fields, selector, file_handler): fields, data, oct_handler, - is_single, + file_handler.single_precision, ) return data @@ -478,9 +475,11 @@ def _fill_with_ghostzones( # Initializing data container for field in fields: tr[field] = np.zeros(cell_count, "float64") - + # Get the size of the data, single precision if RT field, double otherwise - is_single = True if any(field.startswith("Photon") for field in fields) else False + is_single = ( + True if any(field.startswith("Photon") for field in fields) else False + ) # Do an early exit if the cell count is null if cell_count == 0: @@ -813,7 +812,7 @@ def __init__( if isinstance(fields, str): fields = field_aliases[fields] """ - fields: + fields: list[tuple[str, str]] | list[str] | None An array of hydro variable fields in order of position in the hydro_XXXXX.outYYYYY file. If set to None, will try a default set of fields. @@ -830,7 +829,13 @@ def __init__( This affects the fields related to cooling and the mean molecular weight. """ - self._fields_in_file = fields + self._fields_in_file: list[tuple[str, str]] = [] + if fields: + for field in fields: + if isinstance(field, tuple): + self._fields_in_file.append(field) + else: + self._fields_in_file.append((field, "d")) # By default, extra fields have not triggered a warning self._warned_extra_fields = defaultdict(lambda: False) self._extra_particle_fields = extra_particle_fields diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index f08235b326a..e8786ef9d08 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -24,6 +24,51 @@ def register_field_handler(ph): DETECTED_FIELDS = {} # type: ignore +def check_field_precision(fields: list[tuple[str, str]]) -> bool: + """Check if all fields have the same precision. + + Parameters + ---------- + fields : list of (str, str) + List of tuples containing field names and their precision + ('f' for single, 'd' for double). + + Returns + ------- + bool + True if all fields have the same precision, False otherwise. + """ + if not fields: + return True + else: + return all(precision == fields[0][1] for _, precision in fields) + + +def get_fields_and_single_precision( + fields: list[tuple[str, str]], +) -> tuple[list[str], bool]: + """Get the common precision of fields. + + Parameters + ---------- + fields : list of (str, str) + List of tuples containing field names and their precision + ('f' for single, 'd' for double). + + Returns + ------- + tuple of (list of str, bool) + A tuple containing a list of field names and a boolean indicating + if all fields are single precision, False if double precision. + """ + field_names = [field for field, _ in fields] + if fields: + single_precision = check_field_precision(fields) and fields[0][1] == "f" + else: + single_precision = False + return field_names, single_precision + + class HandlerMixin: """This contains all the shared methods to handle RAMSES files. @@ -169,7 +214,6 @@ def __init_subclass__(cls, *args, **kwargs): cls._unique_registry = {} cls.parameters = {} cls.rt_parameters = {} - cls._detected_field_list = {} return cls def __init__(self, domain): @@ -177,7 +221,7 @@ def __init__(self, domain): @classmethod @abc.abstractmethod - def detect_fields(cls, ds): + def detect_fields(cls, ds) -> tuple[list[str], bool]: """ Called once to setup the fields of this type @@ -192,7 +236,7 @@ def detect_fields(cls, ds): pass @classmethod - def get_detected_fields(cls, ds): + def get_detected_fields(cls, ds) -> tuple[list[str], bool] | None: """ Get the detected fields from the registry. """ @@ -204,14 +248,16 @@ def get_detected_fields(cls, ds): return None @classmethod - def set_detected_fields(cls, ds, fields): + def set_detected_fields(cls, ds, fields, single_precision): """ Store the detected fields into the registry. """ if ds.unique_identifier not in DETECTED_FIELDS: DETECTED_FIELDS[ds.unique_identifier] = {} - DETECTED_FIELDS[ds.unique_identifier].update({cls.ftype: fields}) + DETECTED_FIELDS[ds.unique_identifier].update( + {cls.ftype: (fields, single_precision)} + ) @classmethod def purge_detected_fields(cls, ds): @@ -237,7 +283,13 @@ def level_count(self): @property def field_list(self): - return self._detected_field_list[self.ds.unique_identifier] + field_list, _single_precision = self.detect_fields(self.ds) + return [(self.ftype, f) for f in field_list] + + @property + def single_precision(self): + _field_list, single_precision = self.detect_fields(self.ds) + return single_precision @cached_property def offset(self): @@ -250,11 +302,7 @@ def offset(self): It should be generic enough for most of the cases, but if the *structure* of your fluid file is non-canonical, change this. """ - nvars = len(self._detected_field_list[self.ds.unique_identifier]) - - first_field = self.field_list[0][1] - # Get the size of the data, single precision if RT field, double otherwise - is_single = True if first_field.startswith("Photon") else False + nvars = len(self.field_list) with FortranFile(self.fname) as fd: # Skip headers @@ -282,17 +330,32 @@ def offset(self): self.parameters[self.ds.unique_identifier]["nvar"], self.domain.amr_header, Nskip=nvars * 8, - single=is_single, + single_precision=self.single_precision, ) self._level_count = level_count return offset @classmethod - def load_fields_from_yt_config(cls) -> list[str]: + def load_fields_from_yt_config(cls) -> list[tuple[str, str]]: if cls.config_field and ytcfg.has_section(cls.config_field): cfg = ytcfg.get(cls.config_field, "fields") - fields = [_.strip() for _ in cfg if _.strip() != ""] + fields = [] + for item in cfg: + item = item.strip() + if not item: + continue + content = item.split(",") + if len(content) == 2: + field_name, precision = content + elif len(content) == 1: + field_name, precision = content[0], "d" + else: + raise ValueError( + f"Invalid field specification '{item}' in config section " + f"'{cls.config_field}'. Expected format: field_name[,precision]" + ) + fields.append((field_name.strip(), precision.strip())) return fields return [] @@ -314,7 +377,7 @@ class HydroFieldFileHandler(FieldFileHandler): ) @classmethod - def detect_fields(cls, ds): + def detect_fields(cls, ds) -> tuple[list[str], bool]: # Try to get the detected fields detected_fields = cls.get_detected_fields(ds) if detected_fields: @@ -336,24 +399,42 @@ def detect_fields(cls, ds): nvar = hvals["nvar"] ok = False + single_precision = False - if ds._fields_in_file is not None: + if ds._fields_in_file: # Case 1: fields are provided by users on construction of dataset - fields = list(ds._fields_in_file) + fields, single_precision = get_fields_and_single_precision( + ds._fields_in_file + ) ok = True else: # Case 2: fields are provided by users in the config - fields = cls.load_fields_from_yt_config() + fields_with_precision = cls.load_fields_from_yt_config() + fields, single_precision = get_fields_and_single_precision( + fields_with_precision + ) + ok = len(fields) > 0 if not ok and os.path.exists(fname_desc): # Case 3: there is a file descriptor # Or there is an hydro file descriptor mylog.debug("Reading hydro file descriptor.") - # For now, we can only read double precision fields - fields = [ - e[0] for e in _read_fluid_file_descriptor(fname_desc, prefix="hydro") - ] + + field_with_precision = _read_fluid_file_descriptor( + fname_desc, prefix="hydro" + ) + + # Make sure all fields have the same precision + if all(dtype == "f" for _, dtype in field_with_precision): + single_precision = True + elif all(dtype == "d" for _, dtype in field_with_precision): + single_precision = False + else: + raise RuntimeError( + "Mixed precision in hydro file descriptor. This is not supported." + ) + fields = [e[0] for e in field_with_precision] # We get no fields for old-style hydro file descriptor ok = len(fields) > 0 @@ -460,13 +541,10 @@ def detect_fields(cls, ds): count_extra += 1 if count_extra > 0: mylog.debug("Detected %s extra fluid fields.", count_extra) - cls._detected_field_list[ds.unique_identifier] = [ - (cls.ftype, e) for e in fields - ] - cls.set_detected_fields(ds, fields) + cls.set_detected_fields(ds, fields, single_precision) - return fields + return (fields, single_precision) class GravFieldFileHandler(FieldFileHandler): @@ -482,7 +560,7 @@ class GravFieldFileHandler(FieldFileHandler): ) @classmethod - def detect_fields(cls, ds): + def detect_fields(cls, ds) -> tuple[list[str], bool]: # Try to get the detected fields detected_fields = cls.get_detected_fields(ds) if detected_fields: @@ -514,13 +592,10 @@ def detect_fields(cls, ds): for i in range(nvar - ndetected): fields.append(f"var{i}") - cls._detected_field_list[ds.unique_identifier] = [ - (cls.ftype, e) for e in fields - ] + single_precision = False + cls.set_detected_fields(ds, fields, single_precision) - cls.set_detected_fields(ds, fields) - - return fields + return (fields, single_precision) class RTFieldFileHandler(FieldFileHandler): @@ -619,6 +694,7 @@ def read_rhs(cast, group=None): cls.parameters[ds.unique_identifier] = fd.read_attrs(cls.attrs) ok = False + single_precision = False if ds._fields_in_file is not None: # Case 1: fields are provided by users on construction of dataset @@ -633,10 +709,12 @@ def read_rhs(cast, group=None): # Case 3: there is a file descriptor # Or there is an hydro file descriptor mylog.debug("Reading rt file descriptor.") - # For now, we can only read double precision fields - fields = [ - e[0] for e in _read_fluid_file_descriptor(fname_desc, prefix="rt") - ] + + field_with_precision = _read_fluid_file_descriptor(fname_desc, prefix="rt") + fields, single_precision = get_fields_and_single_precision( + field_with_precision + ) + ok = len(fields) > 0 if not ok: @@ -650,12 +728,8 @@ def read_rhs(cast, group=None): for ng in range(ngroups): fields.extend([t % (ng + 1) for t in tmp]) - cls._detected_field_list[ds.unique_identifier] = [ - (cls.ftype, e) for e in fields - ] - - cls.set_detected_fields(ds, fields) - return fields + cls.set_detected_fields(ds, fields, single_precision) + return (fields, single_precision) @classmethod def get_rt_parameters(cls, ds): diff --git a/yt/frontends/ramses/io_utils.pyx b/yt/frontends/ramses/io_utils.pyx index 2ffe8f4577d..585c8b88881 100644 --- a/yt/frontends/ramses/io_utils.pyx +++ b/yt/frontends/ramses/io_utils.pyx @@ -22,9 +22,9 @@ cdef INT64_t DOUBLE_SIZE = sizeof(np.float64_t) cdef INT64_t SINGLE_SIZE = sizeof(np.float32_t) -cdef inline int skip_len(int Nskip, int record_len, np.npy_bool single) noexcept nogil: +cdef inline int skip_len(int Nskip, int record_len, np.npy_bool single_precision) noexcept nogil: # If the data is single precision, we need to skip 4 bytes - if single: + if single_precision: return Nskip * (record_len * SINGLE_SIZE + INT64_SIZE) else: return Nskip * (record_len * DOUBLE_SIZE + INT64_SIZE) @@ -116,7 +116,15 @@ def read_amr(FortranFile f, dict headers, @cython.wraparound(False) @cython.cdivision(True) @cython.nonecheck(False) -cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t nvar, dict headers, int Nskip, np.npy_bool single): +cpdef read_offset( + FortranFile f, + INT64_t min_level, + INT64_t domain_id, + INT64_t nvar, + dict headers, + int Nskip, + np.npy_bool single_precision +): cdef np.ndarray[np.int64_t, ndim=2] offset, level_count cdef INT64_t ndim, twotondim, nlevelmax, n_levels, nboundary, ncpu, ncpu_and_bound @@ -158,7 +166,7 @@ cpdef read_offset(FortranFile f, INT64_t min_level, INT64_t domain_id, INT64_t n if ilevel >= min_level: offset_view[icpu, ilevel - min_level] = f.tell() level_count_view[icpu, ilevel - min_level] = file_ncache - f.seek(skip_len(Nskip, file_ncache, single), SEEK_CUR) + f.seek(skip_len(Nskip, file_ncache, single_precision), SEEK_CUR) return offset, level_count @@ -177,7 +185,7 @@ def fill_hydro(FortranFile f, INT64_t ndim, list all_fields, list fields, dict tr, RAMSESOctreeContainer oct_handler, - np.npy_bool single, + np.npy_bool single_precision, np.ndarray[np.int32_t, ndim=1] domain_inds=np.array([], dtype='int32'), ): cdef INT64_t offset @@ -220,7 +228,7 @@ def fill_hydro(FortranFile f, cdef int first_field_index = jumps[0] # The buffer is different depending on the size of the data - if single: + if single_precision: buffer_single = np.empty((level_count.max(), twotondim, nfields_selected), dtype="float32", order='F') else: buffer_double = np.empty((level_count.max(), twotondim, nfields_selected), dtype="float64", order='F') @@ -243,7 +251,7 @@ def fill_hydro(FortranFile f, offset = offsets[icpu, ilevel] if offset == -1: continue - f.seek(offset + skip_len(first_field_index, nc, single), SEEK_SET) + f.seek(offset + skip_len(first_field_index, nc, single_precision), SEEK_SET) # We have already skipped the first fields (if any) # so we "rewind" (this will cancel the first seek) @@ -253,9 +261,9 @@ def fill_hydro(FortranFile f, for j in range(nfields_selected): jump_len += jumps[j] if jump_len > 0: - f.seek(skip_len(jump_len, nc, single), SEEK_CUR) + f.seek(skip_len(jump_len, nc, single_precision), SEEK_CUR) jump_len = 0 - if single: + if single_precision: f.read_vector_inplace('f', &buffer_single[0, i, j]) else: f.read_vector_inplace('d', &buffer_double[0, i, j]) @@ -266,12 +274,12 @@ def fill_hydro(FortranFile f, # but since we're doing an absolute seek at the beginning of # the loop on CPUs, we can spare one seek here ## if jump_len > 0: - ## f.seek(skip_len(jump_len, nc, single), SEEK_CUR) + ## f.seek(skip_len(jump_len, nc, single_precision), SEEK_CUR) # Alias buffer into dictionary tmp = {} for i, field in enumerate(fields): - if single: + if single_precision: tmp[field] = np.asarray(buffer_single[:, :, i], dtype="float64") else: tmp[field] = buffer_double[:, :, i] From bebf74adb2f8923265434c1bd77f99a1da1cdc3e Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 13:16:04 +0100 Subject: [PATCH 10/20] Use file_handler --- yt/frontends/ramses/data_structures.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yt/frontends/ramses/data_structures.py b/yt/frontends/ramses/data_structures.py index 51916f3a4b0..f3d85ba9a28 100644 --- a/yt/frontends/ramses/data_structures.py +++ b/yt/frontends/ramses/data_structures.py @@ -476,11 +476,6 @@ def _fill_with_ghostzones( for field in fields: tr[field] = np.zeros(cell_count, "float64") - # Get the size of the data, single precision if RT field, double otherwise - is_single = ( - True if any(field.startswith("Photon") for field in fields) else False - ) - # Do an early exit if the cell count is null if cell_count == 0: return tr @@ -513,7 +508,7 @@ def _fill_with_ghostzones( fields, tr, oct_handler, - is_single, + file_handler.single_precision, domain_inds=domain_inds, ) return tr From a8ee5bfe5ade7f4f85a3cfcbc2cf82d2acd68251 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 13:42:56 +0100 Subject: [PATCH 11/20] Do not use dataset-level config for RT --- yt/frontends/ramses/field_handlers.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index e8786ef9d08..344e299b954 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -696,14 +696,9 @@ def read_rhs(cast, group=None): ok = False single_precision = False - if ds._fields_in_file is not None: - # Case 1: fields are provided by users on construction of dataset - fields = list(ds._fields_in_file) - ok = True - else: - # Case 2: fields are provided by users in the config - fields = cls.load_fields_from_yt_config() - ok = len(fields) > 0 + # Are fields are provided by users in the config? + fields = cls.load_fields_from_yt_config() + ok = len(fields) > 0 if not ok and os.path.exists(fname_desc): # Case 3: there is a file descriptor From 122bca18d678c1ee0165a823ed13b80d4c93c395 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 13:53:59 +0100 Subject: [PATCH 12/20] Have load_fields_from_yt_config return the field list + precision --- yt/frontends/ramses/field_handlers.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 344e299b954..139e0c50e79 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -337,7 +337,7 @@ def offset(self): return offset @classmethod - def load_fields_from_yt_config(cls) -> list[tuple[str, str]]: + def load_fields_from_yt_config(cls) -> tuple[list[str], bool]: if cls.config_field and ytcfg.has_section(cls.config_field): cfg = ytcfg.get(cls.config_field, "fields") fields = [] @@ -356,9 +356,10 @@ def load_fields_from_yt_config(cls) -> list[tuple[str, str]]: f"'{cls.config_field}'. Expected format: field_name[,precision]" ) fields.append((field_name.strip(), precision.strip())) - return fields - return [] + return get_fields_and_single_precision(fields) + + return ([], False) class HydroFieldFileHandler(FieldFileHandler): @@ -409,10 +410,7 @@ def detect_fields(cls, ds) -> tuple[list[str], bool]: ok = True else: # Case 2: fields are provided by users in the config - fields_with_precision = cls.load_fields_from_yt_config() - fields, single_precision = get_fields_and_single_precision( - fields_with_precision - ) + fields, single_precision = cls.load_fields_from_yt_config() ok = len(fields) > 0 @@ -576,7 +574,7 @@ def detect_fields(cls, ds) -> tuple[list[str], bool]: nvar = cls.parameters[ds.unique_identifier]["nvar"] ndim = ds.dimensionality - fields = cls.load_fields_from_yt_config() + fields, single_precision = cls.load_fields_from_yt_config() if not fields: if nvar == ndim + 1: @@ -592,7 +590,6 @@ def detect_fields(cls, ds) -> tuple[list[str], bool]: for i in range(nvar - ndetected): fields.append(f"var{i}") - single_precision = False cls.set_detected_fields(ds, fields, single_precision) return (fields, single_precision) @@ -694,10 +691,9 @@ def read_rhs(cast, group=None): cls.parameters[ds.unique_identifier] = fd.read_attrs(cls.attrs) ok = False - single_precision = False # Are fields are provided by users in the config? - fields = cls.load_fields_from_yt_config() + fields, single_precision = cls.load_fields_from_yt_config() ok = len(fields) > 0 if not ok and os.path.exists(fname_desc): From ff2c43aeb0fbe45664ed91fc0ba544e9b108a375 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 17:05:35 +0100 Subject: [PATCH 13/20] Fix logic --- yt/frontends/ramses/field_handlers.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 139e0c50e79..36a47ef5703 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -40,8 +40,17 @@ def check_field_precision(fields: list[tuple[str, str]]) -> bool: """ if not fields: return True - else: - return all(precision == fields[0][1] for _, precision in fields) + + if not all(precision == fields[0][1] for _, precision in fields): + raise RuntimeError("Mixed precision in field list. This is not supported.") + + if fields[0][1] not in ("f", "d"): + raise ValueError( + f"Unknown precision specifier '{fields[0][1]}'. " + "Only 'f' or 'd' are supported." + ) + + return True def get_fields_and_single_precision( @@ -62,8 +71,9 @@ def get_fields_and_single_precision( if all fields are single precision, False if double precision. """ field_names = [field for field, _ in fields] + check_field_precision(fields) if fields: - single_precision = check_field_precision(fields) and fields[0][1] == "f" + single_precision = fields[0][1] == "f" else: single_precision = False return field_names, single_precision From ee4910f0135415df79bd7991ed3d9e48dba1b1f5 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 11 Nov 2025 17:08:18 +0100 Subject: [PATCH 14/20] Refactorize with `get_fields_and_single_precision` --- yt/frontends/ramses/field_handlers.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 36a47ef5703..3d89250c9fd 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -432,17 +432,9 @@ def detect_fields(cls, ds) -> tuple[list[str], bool]: field_with_precision = _read_fluid_file_descriptor( fname_desc, prefix="hydro" ) - - # Make sure all fields have the same precision - if all(dtype == "f" for _, dtype in field_with_precision): - single_precision = True - elif all(dtype == "d" for _, dtype in field_with_precision): - single_precision = False - else: - raise RuntimeError( - "Mixed precision in hydro file descriptor. This is not supported." - ) - fields = [e[0] for e in field_with_precision] + fields, single_precision = get_fields_and_single_precision( + field_with_precision + ) # We get no fields for old-style hydro file descriptor ok = len(fields) > 0 From 4f9d94a892436d35e24648c19786569bbf71fa5a Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Fri, 21 Nov 2025 10:52:10 +0100 Subject: [PATCH 15/20] Type-hinting --- yt/frontends/ramses/field_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 3d89250c9fd..eca75016844 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -211,6 +211,7 @@ class FieldFileHandler(abc.ABC, HandlerMixin): field_types = ( None # Mapping from field to the type of the data (float, integer, ...) ) + parameters: dict # Parameters read from the header def __init_subclass__(cls, *args, **kwargs): """ @@ -611,6 +612,7 @@ class RTFieldFileHandler(FieldFileHandler): ("nboundary", 1, "i"), ("gamma", 1, "d"), ) + rt_parameters: dict @classmethod def detect_fields(cls, ds): From fd78e01af9c9b2a1c428c2a078f3716a6b6b97f2 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Fri, 21 Nov 2025 11:24:35 +0100 Subject: [PATCH 16/20] Add doc --- doc/source/examining/loading_data.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/source/examining/loading_data.rst b/doc/source/examining/loading_data.rst index 89d623a1efc..be5b16f4dea 100644 --- a/doc/source/examining/loading_data.rst +++ b/doc/source/examining/loading_data.rst @@ -3171,12 +3171,15 @@ There are three way to make yt detect all the particle fields. For example, if y particle_metallicity, d """ - Each line should contain the name of the field and its data type (``d`` for double precision, ``f`` for single precision, ``i`` for integer and ``l`` for long integer). You can also configure the auto detected fields for fluid types by adding a section ``ramses-hydro``, ``ramses-grav`` or ``ramses-rt`` in the config file. For example, if you customized your gravity files so that they contain the potential, the potential in the previous timestep and the x, y and z accelerations, you can use : + Each line should contain the name of the field and its data type (``d`` for double precision, ``f`` for single precision, ``i`` for integer and ``l`` for long integer). You can also configure the auto detected fields for fluid types by adding a section ``ramses-hydro``, ``ramses-grav`` or ``ramses-rt`` in the config file. For example, if you customized your gravity files so that they contain the potential, the potential in the previous timestep and the x, y and z accelerations, all in single-precision, you can use : .. code-block:: none [ramses-grav] - fields = [ "Potential", "Potential-old", "x-acceleration", "y-acceleration", "z-acceleration" ] + fields = [ "Potential,f", "Potential-old,f", "x-acceleration,f", "y-acceleration,f", "z-acceleration,f" ] + + Importantly, the order of the fields should match the order in which they are written in the RAMSES output files. + yt will also assume that each entry is formed of a field name followed by its type (``f`` for single precision, ``d`` for double precision), separated by a comma. Field names containing commas are not supported, other precisions (e.g. integers) are not supported either. All fields in a given file type (gravity, hydro, ...) need to have the same precision. 3. New RAMSES way. Recent versions of RAMSES automatically write in their output an ``hydro_file_descriptor.txt`` file that gives information about which field is where. If you wish, you can simply create such a file in the folder containing the ``info_xxxxx.txt`` file @@ -3197,6 +3200,8 @@ There are three way to make yt detect all the particle fields. For example, if y 11, metallicity, d It is important to note that this file should not end with an empty line (but in this case with ``11, metallicity, d``). + Note that, for grid fields (hydro, gravity, rt), all the fields need to have the same precision (``f`` or ``d``). + Combinations are not supported. .. note:: From 8ad8cadc9636fe97cd6e7966ad88ec2faa056b62 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Thu, 12 Feb 2026 13:20:02 +0100 Subject: [PATCH 17/20] Rename+change signature of field precision checker --- yt/frontends/ramses/field_handlers.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index eca75016844..233fe8684e3 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -24,7 +24,7 @@ def register_field_handler(ph): DETECTED_FIELDS = {} # type: ignore -def check_field_precision(fields: list[tuple[str, str]]) -> bool: +def validate_field_precision(fields: list[tuple[str, str]]): """Check if all fields have the same precision. Parameters @@ -32,14 +32,9 @@ def check_field_precision(fields: list[tuple[str, str]]) -> bool: fields : list of (str, str) List of tuples containing field names and their precision ('f' for single, 'd' for double). - - Returns - ------- - bool - True if all fields have the same precision, False otherwise. """ if not fields: - return True + return if not all(precision == fields[0][1] for _, precision in fields): raise RuntimeError("Mixed precision in field list. This is not supported.") @@ -50,8 +45,6 @@ def check_field_precision(fields: list[tuple[str, str]]) -> bool: "Only 'f' or 'd' are supported." ) - return True - def get_fields_and_single_precision( fields: list[tuple[str, str]], @@ -71,7 +64,7 @@ def get_fields_and_single_precision( if all fields are single precision, False if double precision. """ field_names = [field for field, _ in fields] - check_field_precision(fields) + validate_field_precision(fields) if fields: single_precision = fields[0][1] == "f" else: From 077aee5873fd7bb1d7c1915c1630f4e466e5cdb4 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Thu, 12 Feb 2026 13:20:39 +0100 Subject: [PATCH 18/20] Only validate non-zero length fields --- yt/frontends/ramses/field_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 233fe8684e3..5dc298b42af 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -33,7 +33,7 @@ def validate_field_precision(fields: list[tuple[str, str]]): List of tuples containing field names and their precision ('f' for single, 'd' for double). """ - if not fields: + if len(fields) == 0: return if not all(precision == fields[0][1] for _, precision in fields): From 2fe2a975a5f32f104e9930833631d68ac26f81ed Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Thu, 12 Feb 2026 13:21:06 +0100 Subject: [PATCH 19/20] Fix typo --- yt/frontends/ramses/field_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py index 5dc298b42af..01b98c43030 100644 --- a/yt/frontends/ramses/field_handlers.py +++ b/yt/frontends/ramses/field_handlers.py @@ -689,7 +689,7 @@ def read_rhs(cast, group=None): ok = False - # Are fields are provided by users in the config? + # Are fields provided by users in the config? fields, single_precision = cls.load_fields_from_yt_config() ok = len(fields) > 0 From 6a38fe7845c8f92d66bcd9006b8f0023d129a306 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Thu, 12 Feb 2026 13:22:51 +0100 Subject: [PATCH 20/20] Make doc more explicit about single/double precision --- doc/source/examining/loading_data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/examining/loading_data.rst b/doc/source/examining/loading_data.rst index be5b16f4dea..6139b0a02e8 100644 --- a/doc/source/examining/loading_data.rst +++ b/doc/source/examining/loading_data.rst @@ -3179,7 +3179,7 @@ There are three way to make yt detect all the particle fields. For example, if y fields = [ "Potential,f", "Potential-old,f", "x-acceleration,f", "y-acceleration,f", "z-acceleration,f" ] Importantly, the order of the fields should match the order in which they are written in the RAMSES output files. - yt will also assume that each entry is formed of a field name followed by its type (``f`` for single precision, ``d`` for double precision), separated by a comma. Field names containing commas are not supported, other precisions (e.g. integers) are not supported either. All fields in a given file type (gravity, hydro, ...) need to have the same precision. + yt will also assume that each entry is formed of a field name followed by its type (``f`` for single precision, ``d`` for double precision), separated by a comma. Field names containing commas are not supported, other precisions (e.g. integers) are not supported either. Presently, all fields in a given file type (gravity, hydro, ...) need to have the same precision (e.g. all single-precision or all double-precision), combinations are not supported. Internally, yt will convert all data into double-precisoin floats, but this will allow it to read the data correctly from file. 3. New RAMSES way. Recent versions of RAMSES automatically write in their output an ``hydro_file_descriptor.txt`` file that gives information about which field is where. If you wish, you can simply create such a file in the folder containing the ``info_xxxxx.txt`` file