diff --git a/Exec/HaloFinder/32.nyx b/Exec/HaloFinder/32.nyx new file mode 100644 index 000000000..827a00591 Binary files /dev/null and b/Exec/HaloFinder/32.nyx differ diff --git a/Exec/HaloFinder/64sssss_20mpc.nyx b/Exec/HaloFinder/64sssss_20mpc.nyx new file mode 100644 index 000000000..019faac7a Binary files /dev/null and b/Exec/HaloFinder/64sssss_20mpc.nyx differ diff --git a/Exec/HaloFinder/CMakeLists.txt b/Exec/HaloFinder/CMakeLists.txt new file mode 100644 index 000000000..73ab32f95 --- /dev/null +++ b/Exec/HaloFinder/CMakeLists.txt @@ -0,0 +1,11 @@ +set(_sources Prob.cpp Prob_error.cpp) +set(_input_files inputs inputs.summit inputs_gimlet_in_transit.dsc) +list(APPEND _input_files inputs_nohydro.rt inputs.rt inputs.rt.garuda) +list(APPEND _input_files 32.nyx 64sssss_20mpc.nyx TREECOOL_middle) +list(APPEND _input_files ascent_actions_nohydro.yaml ascent_actions.yaml ascent_actions_slicefile.yaml) + +# setup_nyx_executable(_sources _input_files) +nyx_setup_executable(_sources _input_files) + +unset(_sources) +unset(_input_files) diff --git a/Exec/HaloFinder/DarkMatterParticleContainer.H b/Exec/HaloFinder/DarkMatterParticleContainer.H new file mode 100644 index 000000000..638cab59b --- /dev/null +++ b/Exec/HaloFinder/DarkMatterParticleContainer.H @@ -0,0 +1,73 @@ + +#ifndef _DarkMatterParticleContainer_H_ +#define _DarkMatterParticleContainer_H_ + +#include + +class DarkMatterParticleContainer + : public NyxParticleContainer<1+AMREX_SPACEDIM> +{ +public: + DarkMatterParticleContainer (amrex::Amr* amr) + : NyxParticleContainer<1+AMREX_SPACEDIM>(amr) + { + real_comp_names.clear(); + real_comp_names.push_back("mass"); + real_comp_names.push_back("xvel"); + real_comp_names.push_back("yvel"); + real_comp_names.push_back("zvel"); + } + + using MyParIter = amrex::ParIter<1+AMREX_SPACEDIM>; + using MyConstParIter = amrex::ParConstIter<1+AMREX_SPACEDIM>; + + virtual ~DarkMatterParticleContainer () {} + + void InitCosmo1ppcMultiLevel(amrex::MultiFab& mf, const amrex::Real disp_fac[], const amrex::Real vel_fac[], + const amrex::Real particleMass, int disp_idx, int vel_idx, + amrex::BoxArray &baWhereNot, int lev, int nlevs); + + void AssignDensityAndVels (amrex::Vector >& mf, int lev_min = 0) const; + + virtual void moveKickDrift (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_old = 1.0, amrex::Real a_half = 1.0, int where_width = 0, amrex::Real radius_inner = -1.e34, amrex::Real radius_outer = -1.e34); + virtual void moveKick (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_new = 1.0, amrex::Real a_half = 1.0); + + void InitFromBinaryMortonFile(const std::string& particle_directory, int nextra, int skip_factor); + +}; + +AMREX_GPU_HOST_DEVICE AMREX_INLINE void update_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move); + +AMREX_GPU_HOST_DEVICE AMREX_INLINE void update_dm_particle_move_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move); + +AMREX_GPU_HOST_DEVICE AMREX_INLINE void store_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + amrex::ParticleContainer<1+AMREX_SPACEDIM+6, 0>::SuperParticleType& p2, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& phi, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move, + const amrex::Real& radius_inner, + const amrex::Real& radius_outer, + int index, const bool is_file_write=false); + +#endif /* _DarkMatterParticleContainer_H_ */ + diff --git a/Exec/HaloFinder/DarkMatterParticleContainer.cpp b/Exec/HaloFinder/DarkMatterParticleContainer.cpp new file mode 100644 index 000000000..c3979c3e4 --- /dev/null +++ b/Exec/HaloFinder/DarkMatterParticleContainer.cpp @@ -0,0 +1,1068 @@ +#include + +#include +#ifdef AMREX_USE_HDF5 +#include +#endif + +using namespace amrex; +#include +#include + +/// These are helper functions used when initializing from a morton-ordered +/// binary particle file. +namespace { + + inline uint64_t split(unsigned int a) { + uint64_t x = a & 0x1fffff; + x = (x | x << 32) & 0x1f00000000ffff; + x = (x | x << 16) & 0x1f0000ff0000ff; + x = (x | x << 8) & 0x100f00f00f00f00f; + x = (x | x << 4) & 0x10c30c30c30c30c3; + x = (x | x << 2) & 0x1249249249249249; + return x; + } + + inline uint64_t get_morton_index(unsigned int x, + unsigned int y, + unsigned int z) { + uint64_t morton_index = 0; + morton_index |= split(x) | ( split(y) << 1) | (split(z) << 2); + return morton_index; + } + + struct BoxMortonKey { + uint64_t morton_id; + int box_id; + }; + + struct by_morton_id { + bool operator()(const BoxMortonKey &a, const BoxMortonKey &b) { + return a.morton_id < b.morton_id; + } + }; + + std::string get_file_name(const std::string& base, int file_num) { + std::stringstream ss; + ss << base << file_num; + return ss.str(); + } + + struct ParticleMortonFileHeader { + long NP; + int DM; + int NX; + int SZ; + int NF; + }; + + void ReadHeader(const std::string& dir, + const std::string& file, + ParticleMortonFileHeader& hdr) { + std::string header_filename = dir; + header_filename += "/"; + header_filename += file; + + Vector fileCharPtr; + ParallelDescriptor::ReadAndBcastFile(header_filename, fileCharPtr); + std::string fileCharPtrString(fileCharPtr.dataPtr()); + std::istringstream HdrFile(fileCharPtrString, std::istringstream::in); + + HdrFile >> hdr.NP; + HdrFile >> hdr.DM; + HdrFile >> hdr.NX; + HdrFile >> hdr.SZ; + HdrFile >> hdr.NF; + } + +struct ShellFilter +{ + GpuArray m_plo, m_phi, m_center; + Real m_radius_inner, m_radius_outer, m_z, m_t, m_dt; + Box m_domain; + + ShellFilter (const GpuArray& plo, + const GpuArray& phi, + const GpuArray& center, + Real radius_inner, + Real radius_outer, + Real z, + Real t, + Real dt, + const Box& domain) + : m_plo(plo), m_phi(phi), m_center(center), m_radius_inner(radius_inner), m_radius_outer(radius_outer), m_z(z), m_t(t), m_dt(dt), m_domain(domain) + {} +#if 1 + //c_light is the constant for speed of light [cm/s] + template + AMREX_GPU_HOST_DEVICE + int operator() (const SrcData& src, int i) const noexcept + { + int result=0; + if(m_radius_inner<=0 || m_radius_outer<=0) + return 0; + + if(src.m_aos[i].id()>0) { + + Real xlen, ylen, zlen; + + Real lenx = m_phi[0]-m_plo[0]; + Real leny = m_phi[1]-m_plo[1]; + Real lenz = m_phi[2]-m_plo[2]; + int maxind[3]; + maxind[0] = floor((m_radius_outer+lenx*0.5)/lenx); + maxind[1] = floor((m_radius_outer+leny*0.5)/leny); + maxind[2] = floor((m_radius_outer+lenz*0.5)/lenz); + + //printf("Value is %d\n", maxind); + for(int idir=-maxind[0];idir<=maxind[0];idir++) + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) + { + xlen = src.m_aos[i].rdata(0+1+3)+(idir)*(m_phi[0]-m_plo[0]) - m_center[0]; + ylen = src.m_aos[i].rdata(1+1+3)+(jdir)*(m_phi[1]-m_plo[1]) - m_center[1]; + zlen = src.m_aos[i].rdata(2+1+3)+(kdir)*(m_phi[2]-m_plo[2]) - m_center[2]; + Real mag = sqrt(xlen*xlen+ylen*ylen+zlen*zlen); + result+=int(mag>m_radius_inner && mag + AMREX_GPU_HOST_DEVICE + bool operator() (const SrcData& src, int i) const noexcept + { + bool result=false; + if(m_radius_inner<=0 || m_radius_outer<=0) + return false; + if(src.m_aos[i].id()>0) { + + Real xlen, ylen, zlen; + + Real lenx = m_phi[0]-m_plo[0]; + Real leny = m_phi[1]-m_plo[1]; + Real lenz = m_phi[2]-m_plo[2]; + int maxind[3]; + maxind[0] = floor((m_radius_outer+lenx*0.5)/lenx); + maxind[1] = floor((m_radius_outer+leny*0.5)/leny); + maxind[2] = floor((m_radius_outer+lenz*0.5)/lenz); + + //printf("Value is %d\n", maxind); + + for(int idir=-maxind[0];idir<=maxind[0];idir++) + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) + { + xlen = src.m_aos[i].rdata(0+1+3)+(idir)*(m_phi[0]-m_plo[0]) - m_center[0]; + ylen = src.m_aos[i].rdata(1+1+3)+(jdir)*(m_phi[1]-m_plo[1]) - m_center[1]; + zlen = src.m_aos[i].rdata(2+1+3)+(kdir)*(m_phi[2]-m_plo[2]) - m_center[2]; + Real mag = sqrt(xlen*xlen+ylen*ylen+zlen*zlen); + result=result? true : (mag>m_radius_inner && mag m_plo, m_phi, m_center; + Real m_radius_inner, m_radius_outer, m_z, m_t, m_dt, m_dt_a_cur_inv; + Box m_domain; + + ShellStoreFilter (const GpuArray& plo, + const GpuArray& phi, + const GpuArray& center, + Real radius_inner, + Real radius_outer, + Real z, + Real t, + Real dt, + const Box& domain, + const Real dt_a_cur_inv) + : m_plo(plo), m_phi(phi), m_center(center), m_radius_inner(radius_inner), m_radius_outer(radius_outer), m_z(z), m_t(t), m_dt(dt), m_domain(domain), m_dt_a_cur_inv(dt_a_cur_inv) + {} + + template + AMREX_GPU_HOST_DEVICE + void operator() (const DstData& dst, const SrcData& src, + int src_i, int dst_i, int index) const noexcept + { + int local_index=-1; + int nc=AMREX_SPACEDIM; + bool result=false; + Real xlen = m_phi[0]-m_plo[0]; + Real ylen = m_phi[1]-m_plo[1]; + Real zlen = m_phi[2]-m_plo[2]; + int maxind[3]; + maxind[0] = floor((m_radius_outer+xlen*0.5)/xlen); + maxind[1] = floor((m_radius_outer+ylen*0.5)/ylen); + maxind[2] = floor((m_radius_outer+zlen*0.5)/zlen); + + for(int idir=-maxind[0];idir<=maxind[0];idir++) + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) + { + xlen = src.m_aos[src_i].rdata(0+1+3)+(idir)*(m_phi[0]-m_plo[0]) - m_center[0]; + ylen = src.m_aos[src_i].rdata(1+1+3)+(jdir)*(m_phi[1]-m_plo[1]) - m_center[1]; + zlen = src.m_aos[src_i].rdata(2+1+3)+(kdir)*(m_phi[2]-m_plo[2]) - m_center[2]; + Real mag = sqrt(xlen*xlen+ylen*ylen+zlen*zlen); + + if(int(mag>m_radius_inner && mag bool] maps each particle to + * either the left [return true] or the right [return false] partition. + * It must return the same result if evaluated multiple times for the same particle. + * + * \param ptile the ParticleTile to partition + * \param is_left functor to map particles to a partition + */ +template , int> foo = 0, typename Pred, typename F> +Index filterAndTransformAndCopyParticles (DstTile& dst, const SrcTile& src, + Index src_start, Index dst_start, N n, Pred&& p, F&& f) noexcept +{ + long np = n; + const auto & src_data = src.getConstParticleTileData(); + + if (np == 0) { return 0; } + Gpu::DeviceVector mask_vec(np); + auto * mask=mask_vec.dataPtr(); + const auto & ptd = src; + AMREX_HOST_DEVICE_FOR_1D( np, i, + { + mask[i] = int(p(src_data,i)); + }); + + Gpu::DeviceVector offsets(np); + Gpu::exclusive_scan(mask, mask+np, offsets.begin()); + const int num_output = offsets[np-1]; + dst.resize(num_output); + + auto dst_data = dst.getParticleTileData(); + + Index last_mask=0, last_offset=0; + Gpu::copyAsync(Gpu::deviceToHost, mask+np-1, mask + np, &last_mask); + Gpu::copyAsync(Gpu::deviceToHost, offsets.data()+np-1, offsets.data()+np, &last_offset); + + auto const* p_offsets = offsets.dataPtr(); + + // f here should do the copy and transform parts (or maybe instead just do copy here, although we want the images specifically) + AMREX_HOST_DEVICE_FOR_1D( np, i, + { + if(mask[i]>0) + for(int j=0;j +void filterParticles (PC& pc, F&& f) +{ + BL_PROFILE("filterParticles"); + + using ParIter = typename PC::ParIterType; + using ParticleTileType = typename PC::ParticleTileType; + + for (int lev = 0; lev <= pc.finestLevel(); ++lev) + { + for(ParIter pti(pc, lev); pti.isValid(); ++pti) + { + auto& ptile = pc.ParticlesAt(lev, pti); + + ParticleTileType ptile_tmp; + ptile_tmp.resize(ptile.size()); + + auto num_output = amrex::filterParticles(ptile_tmp, ptile, f); + + ptile.swap(ptile_tmp); + ptile.resize(num_output); + } + } +} + + +} + +void +DarkMatterParticleContainer::moveKickDrift (amrex::MultiFab& acceleration, + int lev, + amrex::Real t, + amrex::Real dt, + amrex::Real a_old, + amrex::Real a_half, + int where_width, + amrex::Real radius_inner, + amrex::Real radius_outer) +{ + BL_PROFILE("DarkMatterParticleContainer::moveKickDrift()"); + + //If there are no particles at this level + if (lev >= this->GetParticles().size()) + return; + const auto dxi = Geom(lev).InvCellSizeArray(); + + amrex::MultiFab* ac_ptr; + if (this->OnSameGrids(lev, acceleration)) + { + ac_ptr = &acceleration; + } + else + { + const IntVect& ng = acceleration.nGrowVect(); + ac_ptr = new amrex::MultiFab(this->ParticleBoxArray(lev), + this->ParticleDistributionMap(lev), + acceleration.nComp(),acceleration.nGrow()); + ac_ptr->setVal(0.); + if(acceleration.boxArray() == ac_ptr->boxArray())//this->finestLevel() == 0) + { + ac_ptr->Redistribute(acceleration,0,0,acceleration.nComp(),ng); + ac_ptr->FillBoundary(); + } + else + { + ac_ptr->ParallelCopy(acceleration,0,0,acceleration.nComp(),ng,ng); + ac_ptr->FillBoundary(); + } + } + + const GpuArray plo = Geom(lev).ProbLoArray(); + + int do_move = 1; + + auto pc= this; + auto ShellPC = new ParticleContainer<10,0>(pc->Geom(lev), pc->ParticleDistributionMap(lev), pc->ParticleBoxArray(lev)); + ShellPC->resizeData(); + ParticleContainer<10,0>::ParticleInitData pdata = {{}, {}, {}, {}}; + ShellPC->InitNRandomPerCell(1,pdata); + // ShellPC->resize(pc.TotalNumberOfParticles()); + auto geom_test=pc->Geom(lev); + const GpuArray phi=geom_test.ProbHiArray(); + const GpuArray center({AMREX_D_DECL((phi[0]-plo[0])*0.5,(phi[1]-plo[1])*0.5,(phi[2]-plo[2])*0.5)}); + //const Real a_half = 0.5 * (a_old + a_new); + + auto domain=geom_test.Domain(); + auto z=1/a_old-1; + //From write_info.cpp + amrex::Real a_cur = 1.0 / a_half; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; + ShellFilter shell_filter_test(plo, phi, center, radius_inner, radius_outer, z, t, dt, domain); + ShellStoreFilter shell_store_filter_test(plo, phi, center, radius_inner, radius_outer, z, t, dt, domain, dt_a_cur_inv); +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + AoS& particles = pti.GetArrayOfStructs(); + ParticleType* pstruct = particles().data(); + const long np = pti.numParticles(); + int grid = pti.index(); + + auto& ptile = ShellPC->DefineAndReturnParticleTile(lev, pti); + int old_np = ptile.size(); + int num_to_add = np; + int new_np = old_np + num_to_add; + ptile.resize(new_np); + + const FArrayBox& accel_fab= ((*ac_ptr)[grid]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + update_dm_particle_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_old, a_half,do_move); + }); + + } + /* + for (MFIter mfi(*ShellPC->m_dummy_mf[0], false); mfi.isValid(); ++mfi) { + Box grid = ParticleBoxArray(0)[mfi.index()]; + auto ind = std::make_pair(mfi.index(), mfi.LocalTileIndex()); + RealBox grid_box (grid,dx,geom.ProbLo()); + ParticleTile ptile_tmp; + + Gpu::Device::streamSynchronize(); + }*/ +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + auto& particles = (this->ParticlesAt(lev,pti)).GetArrayOfStructs(); + + auto* pstruct = particles().data(); + auto& ptile = ShellPC->ParticlesAt(lev,pti); + + auto& particles2 = (ShellPC->ParticlesAt(lev,pti)).GetArrayOfStructs(); + auto* pstruct2 = particles2().data(); +#ifdef AMREX_USE_GPU + const long np = pti.numParticles(); + int grid = pti.index(); + ParticleContainer<10,0>::ParticleTileType ptile_tmp; + + auto num_output = filterAndTransformAndCopyParticles(ptile_tmp, ptile, 0, 0, np, shell_filter_test, shell_store_filter_test); + ptile.swap(ptile_tmp); + ptile.resize(num_output); + } +#else + auto ptile_tmp = ptile; + ParticleContainer<10,0>::ParticleTileType ptile_tmp2; + // ptile_tmp.resize((ShellPC->ParticlesAt(lev,pti)).size()); + ptile_tmp.resize((this->ParticlesAt(lev,pti)).size()); + + const long np = pti.numParticles(); + int grid = pti.index(); + + const FArrayBox& accel_fab= ((*ac_ptr)[0]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + int num_output=0; + for(int i=0;i::SuperParticleType p = pstruct2[i]; + int mask=shell_filter_test(ptile.getConstParticleTileData(),i); + if(mask>0) { + for(int j=0;j accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + update_dm_particle_move_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_old, a_half,do_move); + }); + + } + + auto dir=Concatenate("plt_light", int(100*(1/a_old-1)), 7); + auto name="ShellPC"; + amrex::Vector real_comp_names_shell; + real_comp_names_shell.clear(); + real_comp_names_shell.push_back("mass"); + real_comp_names_shell.push_back("xvel"); + real_comp_names_shell.push_back("yvel"); + real_comp_names_shell.push_back("zvel"); + real_comp_names_shell.push_back("xposold"); + real_comp_names_shell.push_back("yposold"); + real_comp_names_shell.push_back("zposold"); + real_comp_names_shell.push_back("xposvalid"); + real_comp_names_shell.push_back("yposvalid"); + real_comp_names_shell.push_back("zposvalid"); + std::string compression = "None@0"; + if(radius_inner>0&&radius_outer>radius_inner) { + int write_hdf5=0; +#ifdef AMREX_USE_HDF5 + write_hdf5=1; +#endif + if(write_hdf5!=1){ + //ShellPC->WritePlotFile(dir, name, real_comp_names_shell); + } +#ifdef AMREX_USE_HDF5 + else{ + ShellPC->WritePlotFileHDF5(dir, name, real_comp_names_shell, compression); + } +#endif + } + Print()<<"After write\t"<TotalNumberOfParticles()<<"\t"<amrex::ParticleContainer<7,0>::WritePlotFile(dir, name, real_comp_names_shell); + if (ac_ptr != &acceleration) delete ac_ptr; + + ParticleLevel& pmap = this->GetParticles(lev); + if (lev > 0 && sub_cycle) + { + if (! m_particle_locator.isValid(GetParGDB())) m_particle_locator.build(GetParGDB()); + m_particle_locator.setGeometry(GetParGDB()); + AmrAssignGrid> assign_grid = m_particle_locator.getGridAssignor(); + + amrex::ParticleLocData pld; + for (auto& kv : pmap) { + AoS& particles = kv.second.GetArrayOfStructs(); + ParticleType* pstruct = particles().data(); + const long np = particles.size(); + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + // amrex::ParticleContainer<4, 0>::SuperParticleType& p=pstruct[i]; + auto& p=pstruct[i]; + if(p.id()>0) { + const auto tup = assign_grid(p, lev, lev, where_width); + auto p_boxes = amrex::get<0>(tup); + auto p_levs = amrex::get<1>(tup); + if(p_boxes<0||p_levs<0) { + if (p.id() == amrex::GhostParticleID) + { + p.id() = -1; + } + else + { + amrex::Error("Trying to get rid of a non-ghost particle in moveKickDrift"); + } + } + } + }); + Gpu::streamSynchronize(); + } + } + delete ShellPC; +} + +void +DarkMatterParticleContainer::moveKick (MultiFab& acceleration, + int lev, + Real t, + Real dt, + Real a_new, + Real a_half) +{ + BL_PROFILE("DarkMatterParticleContainer::moveKick()"); + + const auto dxi = Geom(lev).InvCellSizeArray(); + + MultiFab* ac_ptr; + if (OnSameGrids(lev,acceleration)) + { + ac_ptr = &acceleration; + } + else + { + const IntVect& ng = acceleration.nGrowVect(); + ac_ptr = new amrex::MultiFab(this->ParticleBoxArray(lev), + this->ParticleDistributionMap(lev), + acceleration.nComp(),acceleration.nGrow()); + ac_ptr->setVal(0.); + if(acceleration.boxArray() == ac_ptr->boxArray())//this->finestLevel() == 0) + { + ac_ptr->Redistribute(acceleration,0,0,acceleration.nComp(),ng); + ac_ptr->FillBoundary(); + } + else + { + ac_ptr->ParallelCopy(acceleration,0,0,acceleration.nComp(),ng,ng); + ac_ptr->FillBoundary(); + } + } + + const GpuArray plo = Geom(lev).ProbLoArray(); + + int do_move = 0; + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + AoS& particles = pti.GetArrayOfStructs(); + ParticleType* pstruct = particles().data(); + const long np = pti.numParticles(); + int grid = pti.index(); + + const FArrayBox& accel_fab= ((*ac_ptr)[grid]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + update_dm_particle_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_half,a_new,do_move); + update_dm_particle_move_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_half,a_new,do_move); + }); + } + + + if (ac_ptr != &acceleration) delete ac_ptr; +} + +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void update_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move) +{ + amrex::Real half_dt = 0.5 * dt; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; + + amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; + amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5; + amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; + + int i = static_cast(amrex::Math::floor(lx)); + int j = static_cast(amrex::Math::floor(ly)); + int k = static_cast(amrex::Math::floor(lz)); + + amrex::Real xint = lx - i; + amrex::Real yint = ly - j; + amrex::Real zint = lz - k; + + amrex::Real sx[] = {amrex::Real(1.)-xint, xint}; + amrex::Real sy[] = {amrex::Real(1.)-yint, yint}; + amrex::Real sz[] = {amrex::Real(1.)-zint, zint}; + + for (int d=0; d < AMREX_SPACEDIM; ++d) + { + amrex::Real val = 0.0; + for (int kk = 0; kk<=1; ++kk) + { + for (int jj = 0; jj <= 1; ++jj) + { + for (int ii = 0; ii <= 1; ++ii) + { + val += sx[amrex::Math::abs(ii-1)]* + sy[amrex::Math::abs(jj-1)]* + sz[amrex::Math::abs(kk-1)]*acc(i-ii,j-jj,k-kk,d); + } + } + } + + + p.rdata(d+1)=a_prev*p.rdata(d+1)+half_dt * val; + p.rdata(d+1)*=a_cur_inv; + } + +} + +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void store_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + amrex::ParticleContainer<1+AMREX_SPACEDIM+6, 0>::SuperParticleType& p2, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& phi, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move, const Real& radius_inner, + const Real& radius_outer, int index, const bool is_file_write) +{ + amrex::Real half_dt = 0.5 * dt; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; + const GpuArray center({AMREX_D_DECL((phi[0]-plo[0])*0.5,(phi[1]-plo[1])*0.5,(phi[2]-plo[2])*0.5)}); + + if (do_move == 1) { + p2.rdata(0)=p.rdata(0); + bool result=false; + Real lenx = phi[0]-plo[0]; + Real leny = phi[1]-plo[1]; + Real lenz = phi[2]-plo[2]; + int maxind[3]; + maxind[0] = floor((radius_outer+lenx*0.5)/lenx); + maxind[1] = floor((radius_outer+leny*0.5)/leny); + maxind[2] = floor((radius_outer+lenz*0.5)/lenz); + + Real xlen, ylen, zlen; + int local_index=-1; + for(int idir=-maxind[0];idir<=maxind[0];idir++) { + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) { + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) { + xlen = p.pos(0)+(idir)*(phi[0]-plo[0]) - center[0]; + ylen = p.pos(1)+(jdir)*(phi[1]-plo[1]) - center[1]; + zlen = p.pos(2)+(kdir)*(phi[2]-plo[2]) - center[2]; + Real mag = sqrt(xlen*xlen+ylen*ylen+zlen*zlen); + result=result? true : (mag>radius_inner && magradius_inner && mag::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move) +{ + amrex::Real half_dt = 0.5 * dt; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; + + if (do_move == 1) + { + for (int comp=0; comp < nc; ++comp) { + p.pos(comp) = p.pos(comp) + dt_a_cur_inv * p.rdata(comp+1); + } + } + +} + +void +DarkMatterParticleContainer::InitCosmo1ppcMultiLevel( + MultiFab& mf, const Real disp_fac[], const Real vel_fac[], + const Real particleMass, int disp_idx, int vel_idx, + BoxArray &baWhereNot, int lev, int nlevs) +{ + BL_PROFILE("DarkMatterParticleContainer::InitCosmo1ppcMultiLevel()"); + const int MyProc = ParallelDescriptor::MyProc(); + const Geometry& geom = m_gdb->Geom(lev); + const Real* dx = geom.CellSize(); + + static Vector calls; + + calls.resize(nlevs); + + calls[lev]++; + + if (calls[lev] > 1) return; + + Vector& particles = this->GetParticles(); + + particles.reserve(15); // So we don't ever have to do any copying on a resize. + + particles.resize(nlevs); + + ParticleType p; + Real disp[AMREX_SPACEDIM]; + Real vel[AMREX_SPACEDIM]; + + Real mean_disp[AMREX_SPACEDIM]={AMREX_D_DECL(0,0,0)}; + + + // + // The mf should be initialized according to the ics... + // + int outside_counter=0; + long outcount[3]={0,0,0}; + long outcountminus[3]={0,0,0}; + long totalcount=0; +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(mf,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + FArrayBox& myFab = mf[mfi]; + const Box& vbx = mfi.validbox(); + const int *fab_lo = vbx.loVect(); + const int *fab_hi = vbx.hiVect(); + ParticleLocData pld; + for (int kx = fab_lo[2]; kx <= fab_hi[2]; kx++) + { + for (int jx = fab_lo[1]; jx <= fab_hi[1]; jx++) + { + for (int ix = fab_lo[0]; ix <= fab_hi[0]; ix++) + { + IntVect indices(AMREX_D_DECL(ix, jx, kx)); + totalcount++; + if (baWhereNot.contains(indices)) + { + continue; + } + + for (int n = 0; n < AMREX_SPACEDIM; n++) + { + disp[n] = myFab(indices,disp_idx+n); + // + // Start with homogeneous distribution (for 1 p per cell in the center of the cell), + // + p.pos(n) = geom.ProbLo(n) + + (indices[n]+Real(0.5))*dx[n]; + if(disp[n]*disp_fac[n]>dx[n]/2.0) + outcount[n]++; + if(disp[n]*disp_fac[n]<-dx[n]/2.0) + outcountminus[n]++; + mean_disp[n]+=fabs(disp[n]); + // + // then add the displacement (input values weighted by domain length). + // + p.pos(n) += disp[n] * disp_fac[n]; + + // + // Set the velocities. + // + vel[n] = myFab(indices,vel_idx+n); + p.rdata(n+1) = vel[n] * vel_fac[n]; + } + // + // Set the mass of the particle from the input value. + // + p.rdata(0) = particleMass; + p.id() = ParticleType::NextID(); + p.cpu() = MyProc; + + if (!this->Where(p, pld)) + { + this->PeriodicShift(p); + + if (!this->Where(p, pld)) + amrex::Abort("DarkMatterParticleContainer::InitCosmo1ppcMultiLevel():invalid particle"); + } + + BL_ASSERT(pld.m_lev >= 0 && pld.m_lev <= m_gdb->finestLevel()); + //handle particles that ran out of this level into a finer one. + if (baWhereNot.contains(pld.m_cell)) + { + outside_counter++; + ParticleType newp[8]; + ParticleLocData new_pld; + for (int i=0;i<8;i++) + { + newp[i].rdata(0) = particleMass/8.0; + newp[i].id() = ParticleType::NextID(); + newp[i].cpu() = MyProc; + for (int dim=0;dimWhere(newp[i], new_pld)) + { + this->PeriodicShift(newp[i]); + + if (!this->Where(newp[i], new_pld)) + amrex::Abort("DarkMatterParticleContainer::InitCosmo1ppcMultiLevel():invalid particle"); + } + particles[new_pld.m_lev][std::make_pair(new_pld.m_grid, + new_pld.m_tile)].push_back(newp[i]); + } + + } + + // + // Add it to the appropriate PBox at the appropriate level. + // + else + particles[pld.m_lev][std::make_pair(pld.m_grid, pld.m_tile)].push_back(p); + } + } + } + } + Redistribute(); +} + +/* + Particle deposition +*/ + +void +DarkMatterParticleContainer::AssignDensityAndVels (Vector >& mf, int lev_min) const +{ + AssignDensity(mf, lev_min, AMREX_SPACEDIM+1); +} + +void +DarkMatterParticleContainer::InitFromBinaryMortonFile(const std::string& particle_directory, + int /*nextra*/, int skip_factor) { + BL_PROFILE("DarkMatterParticleContainer::InitFromBinaryMortonFile"); + + ParticleMortonFileHeader hdr; + ReadHeader(particle_directory, "Header", hdr); + + uint64_t num_parts = hdr.NP; + int DM = hdr.DM; + int NX = hdr.NX; + int float_size = hdr.SZ; + int num_files = hdr.NF; + size_t psize = (DM + NX) * float_size; + + std::string particle_file_base = particle_directory + "/particles."; + std::vector file_names; + for (int i = 0; i < num_files; ++i) + file_names.push_back(get_file_name(particle_file_base, i)); + + const int lev = 0; + const BoxArray& ba = ParticleBoxArray(lev); + int num_boxes = ba.size(); + uint64_t num_parts_per_box = num_parts / num_boxes; + uint64_t num_parts_per_file = num_parts / num_files; + uint64_t num_bytes_per_file = num_parts_per_file * psize; + + std::vector box_morton_keys(num_boxes); + for (int i = 0; i < num_boxes; ++i) { + const Box& box = ba[i]; + unsigned int x = box.smallEnd(0); + unsigned int y = box.smallEnd(1); + unsigned int z = box.smallEnd(2); + box_morton_keys[i].morton_id = get_morton_index(x, y, z); + box_morton_keys[i].box_id = i; + } + + std::sort(box_morton_keys.begin(), box_morton_keys.end(), by_morton_id()); + + std::vector file_indices(num_boxes); + for (int i = 0; i < num_boxes; ++i) + file_indices[box_morton_keys[i].box_id] = i; + + ParticleType p; + for (MFIter mfi = MakeMFIter(lev, false); mfi.isValid(); ++mfi) { // no tiling + const int grid = mfi.index(); + const int tile = mfi.LocalTileIndex(); + auto& particles = GetParticles(lev); + + uint64_t start = file_indices[grid]*num_parts_per_box; + uint64_t stop = start + num_parts_per_box; + + int file_num = start / num_parts_per_file; + uint64_t seek_pos = (start * psize ) % num_bytes_per_file; + std::string file_name = file_names[file_num]; + + std::ifstream ifs; + ifs.open(file_name.c_str(), std::ios::in|std::ios::binary); + if (!ifs ) { + amrex::Print() << "Failed to open file " << file_name << " for reading. \n"; + amrex::Abort(); + } + + ifs.seekg(seek_pos, std::ios::beg); + + for (uint64_t i = start; i < stop; ++i) { + int next_file = i / num_parts_per_file; + if (next_file != file_num) { + file_num = next_file; + file_name = file_names[file_num]; + ifs.close(); + ifs.open(file_name.c_str(), std::ios::in|std::ios::binary); + if (!ifs ) { + amrex::Print() << "Failed to open file " << file_name << " for reading. \n"; + amrex::Abort(); + } + } + + Vector fpos(DM); + Vector fextra(NX); + ifs.read((char*)&fpos[0], DM*sizeof(float)); + ifs.read((char*)&fextra[0], NX*sizeof(float)); + + if ( (i - start) % skip_factor == 0 ) { + AMREX_D_TERM(p.pos(0) = fpos[0];, + p.pos(1) = fpos[1];, + p.pos(2) = fpos[2];); + + for (int comp = 0; comp < NX; comp++) + p.rdata(AMREX_SPACEDIM+comp) = fextra[comp]; + + p.rdata(AMREX_SPACEDIM) *= skip_factor; + + p.id() = ParticleType::NextID(); + p.cpu() = ParallelDescriptor::MyProc(); + particles[std::make_pair(grid, tile)].push_back(p); + } + } + } + + Redistribute(); +} + diff --git a/Exec/HaloFinder/Filelist_128x128x128.txt b/Exec/HaloFinder/Filelist_128x128x128.txt new file mode 100644 index 000000000..935d96bce --- /dev/null +++ b/Exec/HaloFinder/Filelist_128x128x128.txt @@ -0,0 +1,4 @@ +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-0 +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-1 +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-2 +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-3 diff --git a/Exec/HaloFinder/GNUmakefile b/Exec/HaloFinder/GNUmakefile new file mode 100644 index 000000000..5de330926 --- /dev/null +++ b/Exec/HaloFinder/GNUmakefile @@ -0,0 +1,89 @@ +# AMREX_HOME defines the directory in which we will find all the AMReX code +AMREX_HOME ?= ../../subprojects/amrex + +# TOP defines the directory in which we will find Source, Exec, etc +TOP = ../.. + +# compilation options +COMP = gnu +USE_MPI = TRUE +# Use with Async IO +MPI_THREAD_MULTIPLE = TRUE +USE_OMP = FALSE +USE_CUDA = FALSE + +USE_SUNDIALS = TRUE +USE_FORT_ODE = FALSE +SUNDIALS_ROOT = /global/cfs/cdirs/m4106/jmsexton/sundials_shared/sundials/instdir +#SUNDIALS_ROOT ?= ${CURDIR}/$(TOP)/../sundials/instdir +USE_HDF5 = FALSE +HDF5_HOME = /opt/cray/pe/hdf5/1.12.2.9 + +COMMON_DIR = /global/common/software/nyx/nataraj2 +#ASCENT_HOME ?= ${CURDIR}/$(TOP)/../ascent/install-debug +ASCENT_HOME ?= ${COMMON_DIR}/ascent/install-debug + +USE_ASCENT_INSITU = TRUE +ifeq ($(USE_ASCENT_INSITU),TRUE) + ASCENT_HOME ?= NOT_SET + ifneq ($(ASCENT_HOME),NOT_SET) + include $(ASCENT_HOME)/share/ascent/ascent_config.mk + endif + USE_CONDUIT = TRUE + USE_ASCENT = TRUE +endif + + +PROFILE = FALSE +TRACE_PROFILE = FALSE +COMM_PROFILE = FALSE +TINY_PROFILE = FALSE + +PRECISION = DOUBLE +USE_SINGLE_PRECISION_PARTICLES = TRUE +DEBUG = FALSE + +# physics +DIM = 3 +USE_HEATCOOL = TRUE +USE_SAVE_REACT = FALSE +USE_AGN = FALSE +ifeq ($(NO_HYDRO),TRUE) +USE_SDC = FALSE +USE_SUNDIALS = FALSE +USE_FUSED = FALSE +else +ifeq ($(USE_HEATCOOL),TRUE) +USE_SDC = TRUE +USE_SUNDIALS = TRUE +ifeq ($(USE_HIP),TRUE) +USE_FUSED ?= $(USE_HIP) +endif +USE_FUSED ?= $(USE_CUDA) +else +USE_SDC = FALSE +USE_SUNDIALS = FALSE +USE_FUSED = FALSE +endif +endif + +USE_SDC = FALSE +USE_FUSED = TRUE +USE_HEATCOOL=FALSE + +USE_CONST_SPECIES = TRUE +NEUTRINO_PARTICLES = FALSE +NEUTRINO_DARK_PARTICLES = FALSE + +USE_OWN_BCS = FALSE + +# Halo finder +BOOST_INLUDE_DIR := /usr/include/boost +REEBER = TRUE +DIY_INCLUDE_DIR = /global/common/software/nyx/nataraj2/diy/include +REEBER_EXT_HOME = /global/common/software/nyx/nataraj2/Reeber2 + +Bpack := ./Make.package +Blocs := . + +include $(TOP)/Exec/Make.Nyx diff --git a/Exec/HaloFinder/LightConeParticle.H b/Exec/HaloFinder/LightConeParticle.H new file mode 100644 index 000000000..ce1fe8c7f --- /dev/null +++ b/Exec/HaloFinder/LightConeParticle.H @@ -0,0 +1,10 @@ +class LightConeParticle { + public: + double x, y, z, vx, vy, vz; +}; +extern std::vector shell_particles; + +void writeBinaryVTK(const std::string& filename, + const std::vector& shell_particles); +void writeBinarySimple(const std::string& filename, + const std::vector& shell_particles); diff --git a/Exec/HaloFinder/LightConeParticle.cpp b/Exec/HaloFinder/LightConeParticle.cpp new file mode 100644 index 000000000..b2f72db5d --- /dev/null +++ b/Exec/HaloFinder/LightConeParticle.cpp @@ -0,0 +1,151 @@ +#include +#include +#include +#include +#include +#include + +#include + +void SwapEnd(float& val) { + // Swap endianess if necessary + char* bytes = reinterpret_cast(&val); + std::swap(bytes[0], bytes[3]); + std::swap(bytes[1], bytes[2]); +} + +void writeBinaryVTK(const std::string& filename, const std::vector& particles) { + + int rank = amrex::ParallelDescriptor::MyProc(); + int size = amrex::ParallelDescriptor::NProcs(); + + long int local_num_particles = particles.size(); + long int total_num_particles = local_num_particles; + + // Get total particles across all ranks + amrex::ParallelDescriptor::ReduceLongSum(total_num_particles); + + // Compute offset for this rank's data + size_t offset = 0; + MPI_Exscan(&local_num_particles, &offset, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD); + + // Header handling + size_t header_size = 0; + + if (rank == 0) { + std::ofstream file(filename, std::ios::binary); + if (!file) { + amrex::Abort("Error: Could not open file " + filename + "\n"); + } + + // Write the header + file << "# vtk DataFile Version 2.0\n"; + file << "Particle Cloud Data\n"; + file << "BINARY\n"; + file << "DATASET POLYDATA\n"; + file << "POINTS " << total_num_particles << " float\n"; + + // Determine header size + file.seekp(0, std::ios::end); + header_size = file.tellp(); + file.close(); + } + + // Broadcast the header size to all ranks + amrex::ParallelDescriptor::Bcast(&header_size, 1, 0); + + // Use MPI collective I/O for binary data + MPI_File mpi_file; + MPI_File_open(MPI_COMM_WORLD, filename.c_str(), + MPI_MODE_WRONLY | MPI_MODE_APPEND, MPI_INFO_NULL, &mpi_file); + + // Compute byte offset for this rank + size_t byte_offset = header_size + sizeof(float) * 3 * offset; + + // Prepare local data + std::vector data(3 * local_num_particles); + for (size_t i = 0; i < local_num_particles; ++i) { + data[3 * i] = particles[i].x; + data[3 * i + 1] = particles[i].y; + data[3 * i + 2] = particles[i].z; + + // Convert to big-endian if needed + SwapEnd(data[3 * i]); + SwapEnd(data[3 * i + 1]); + SwapEnd(data[3 * i + 2]); + } + + // Write particle data collectively + MPI_File_write_at_all(mpi_file, byte_offset, data.data(), data.size(), MPI_FLOAT, MPI_STATUS_IGNORE); + + MPI_File_close(&mpi_file); + + if (rank == 0) { + std::cout << "Successfully wrote VTK file: " << filename << "\n"; + } +} + +void writeBinarySimple(const std::string& filename, const std::vector& particles) { + + int rank = amrex::ParallelDescriptor::MyProc(); + int size = amrex::ParallelDescriptor::NProcs(); + + long int local_num_particles = particles.size(); + + // Compute offset for this rank's data + size_t offset = 0; + MPI_Exscan(&local_num_particles, &offset, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD); + + // Header handling + unsigned long header_size = 0; + + if (rank == 0) { + std::ofstream file(filename, std::ios::binary); + if (!file) { + amrex::Abort("Error: Could not open file " + filename + "\n"); + } + file.seekp(0, std::ios::end); + header_size = file.tellp(); + file.close(); + } + + // Broadcast the header size to all ranks + amrex::ParallelDescriptor::Bcast(&header_size, 1, 0); + + // Use MPI collective I/O for binary data + MPI_File mpi_file; + MPI_File_open(MPI_COMM_WORLD, filename.c_str(), + MPI_MODE_WRONLY | MPI_MODE_APPEND, MPI_INFO_NULL, &mpi_file); + + // Compute byte offset for this rank + size_t byte_offset = header_size + sizeof(float) * 6 * offset; + + // Prepare local data + std::vector data(6 * local_num_particles); + for (size_t i = 0; i < local_num_particles; ++i) { + data[6 * i] = particles[i].x; + data[6 * i + 1] = particles[i].y; + data[6 * i + 2] = particles[i].z; + data[6 * i + 3] = particles[i].vx; + data[6 * i + 4] = particles[i].vy; + data[6 * i + 5] = particles[i].vz; + + // Convert to big-endian if needed + SwapEnd(data[6 * i]); + SwapEnd(data[6 * i + 1]); + SwapEnd(data[6 * i + 2]); + SwapEnd(data[6 * i + 3]); + SwapEnd(data[6 * i + 4]); + SwapEnd(data[6 * i + 5]); + } + + // Write particle data collectively + MPI_File_write_at_all(mpi_file, byte_offset, data.data(), data.size(), MPI_FLOAT, MPI_STATUS_IGNORE); + + MPI_File_close(&mpi_file); + + if (rank == 0) { + std::cout << "Successfully wrote VTK file: " << filename << "\n"; + } +} + diff --git a/Exec/HaloFinder/Make.package b/Exec/HaloFinder/Make.package new file mode 100644 index 000000000..7ac175cdc --- /dev/null +++ b/Exec/HaloFinder/Make.package @@ -0,0 +1,4 @@ +CEXE_sources += Prob.cpp +CEXE_sources += Prob_error.cpp +CEXE_sources += LightConeParticle.cpp +CEXE_headers += LightConeParticle.H diff --git a/Exec/HaloFinder/NeutrinoParticleContainer.H b/Exec/HaloFinder/NeutrinoParticleContainer.H new file mode 100644 index 000000000..1baa33743 --- /dev/null +++ b/Exec/HaloFinder/NeutrinoParticleContainer.H @@ -0,0 +1,49 @@ +#ifdef NEUTRINO_PARTICLES +#ifndef _NeutrinoParticleContainer_H_ +#define _NeutrinoParticleContainer_H_ + +#include + +class NeutrinoParticleContainer + : public NyxParticleContainer<2+AMREX_SPACEDIM> +{ + +private: + int m_relativistic; // if 1 then we weight the mass by gamma in AssignDensity* + amrex::Real m_csq; // the square of the speed of light -- used to compute relativistic effects + +public: + NeutrinoParticleContainer (amrex::Amr* amr) + : NyxParticleContainer<2+AMREX_SPACEDIM>(amr) + { } + + using MyParIter = amrex::ParIter<2+AMREX_SPACEDIM>; + using MyConstParIter = amrex::ParConstIter<2+AMREX_SPACEDIM>; + + virtual ~NeutrinoParticleContainer () {} + + void SetRelativistic (int relativistic) { m_relativistic = relativistic; } + + void SetCSquared (amrex::Real csq) { m_csq = csq; } + +#ifndef NEUTRINO_DARK_PARTICLES + void AssignDensity (amrex::Vector >& mf, int lev_min = 0, + int ncomp = 1, int finest_level = -1, int ngrow = 2) const + { AssignRelativisticDensity (mf,lev_min,ncomp,finest_level,ngrow); } + + void AssignRelativisticDensitySingleLevel (amrex::MultiFab& mf, int level, int ncomp=1, int particle_lvl_offset = 0) const; + + void AssignRelativisticDensity (amrex::Vector >& mf, + int lev_min = 0, int ncomp = 1, int finest_level = -1, int ngrow = 2) const; +#endif + + virtual void moveKickDrift (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_old = 1.0, amrex::Real a_half = 1.0, int where_width = 0); + virtual void moveKick (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_new = 1.0, amrex::Real a_half = 1.0); +}; + +#endif /*_NeutrinoParticleContainer_H_*/ +#endif /*NEUTRINO_PARTICLES*/ diff --git a/Exec/HaloFinder/NyxParticleContainer.H b/Exec/HaloFinder/NyxParticleContainer.H new file mode 100644 index 000000000..ee2a2e8db --- /dev/null +++ b/Exec/HaloFinder/NyxParticleContainer.H @@ -0,0 +1,522 @@ +#ifndef _NyxParticleContainer_H_ +#define _NyxParticleContainer_H_ + +#include +#include +#include +#include + +class NyxParticleContainerBase +{ +public: + + virtual ~NyxParticleContainerBase() {} + + virtual void moveKickDrift (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_old = 1.0, amrex::Real a_half = 1.0, int where_width = 0, + amrex::Real radius_inner = -1.e34, amrex::Real radius_outer = -1.e34) = 0; + virtual void moveKick (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_new = 1.0, amrex::Real a_half = 1.0) = 0; + virtual void Redistribute (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual void RedistributeLocal (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual void RedistributeOK (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual void RedistributeGPU (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual int finestLevel() const = 0; + virtual void ShrinkToFit() = 0; + virtual void RemoveParticlesAtLevel (int level) = 0; + virtual amrex::Real sumParticleMass (int level) const = 0; + virtual void Regrid (const amrex::DistributionMapping& dmap, const amrex::BoxArray& ba, const int lev) = 0; + // void RegridHat (const amrex::DistributionMapping& dmap, const amrex::BoxArray& ba, const int lev); + virtual amrex::Vector NumberOfParticlesInGrid (int lev, bool only_valid, bool only_local) = 0; + + virtual void AssignDensitySingleLevel (amrex::MultiFab& mf, int level, int ncomp=1, + int particle_lvl_offset = 0) const = 0; + virtual void AssignDensity (amrex::Vector >& mf, int lev_min = 0, int ncomp = 1, + int finest_level = -1, int ngrow = 1) const = 0; +}; + +template +class NyxParticleContainer + : public amrex::NeighborParticleContainer, + public NyxParticleContainerBase +{ +public: + + using ParticleTileType = amrex::ParticleTile,NAR,NAI>; + using MyParIter = amrex::ParIter; + using MyConstParIter = amrex::ParConstIter; + + NyxParticleContainer (amrex::Amr* amr, int nghost=0) + : amrex::NeighborParticleContainer((amrex::ParGDBBase*) amr->GetParGDB(), nghost), + sub_cycle(amr->subCycle()) + {} + + virtual ~NyxParticleContainer () {} + + void GetParticleVelocities (amrex::Vector& part_vels); + void SetParticleVelocities (amrex::Vector& part_data); + + virtual amrex::Real sumParticleMass (int level) const override { + using PType = typename amrex::ParticleContainer::SuperParticleType; + auto msum = amrex::ReduceSum(*this, level, + [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> amrex::Real + { + return (p.id() > 0) ? p.rdata(0) : 0.0; + }); + amrex::ParallelAllReduce::Sum(msum, amrex::ParallelContext::CommunicatorSub()); + return msum; + } + + virtual void Regrid (const amrex::DistributionMapping& dmap, const amrex::BoxArray& ba, const int lev) override { + amrex::NeighborParticleContainer::Regrid(dmap, ba, lev); + } + + virtual amrex::Vector NumberOfParticlesInGrid (int lev, bool only_valid, bool only_local) override { + return amrex::NeighborParticleContainer::NumberOfParticlesInGrid(lev,only_valid,only_local); + } + void sumParticleMomentum (int lev, amrex::Real* mom) const; + + virtual void AssignDensitySingleLevel (amrex::MultiFab& mf, int level, int ncomp=1, int particle_lvl_offset = 0) const override + { + amrex::NeighborParticleContainer::AssignCellDensitySingleLevel(0, mf, level, ncomp, particle_lvl_offset); + } + virtual void AssignDensity (amrex::Vector >& mf, int lev_min = 0, int ncomp = 1, int finest_level = -1, int ngrow = 1) const override + { + amrex::NeighborParticleContainer::AssignDensity(0, mf, lev_min, ncomp, finest_level, ngrow); + } + + void MultiplyParticleMass (int lev, amrex::Real mult); + + amrex::Real estTimestep (amrex::MultiFab& acceleration, int level, amrex::Real cfl) const; + amrex::Real estTimestep (amrex::MultiFab& acceleration, amrex::Real a, int level, amrex::Real cfl) const; + + virtual int finestLevel() const override + { + return amrex::NeighborParticleContainer::finestLevel(); + } + + virtual void ShrinkToFit() override + { + amrex::NeighborParticleContainer::ShrinkToFit(); + } + + virtual void Redistribute (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) override + { + // amrex::Gpu::synchronize(); + amrex::NeighborParticleContainer::Redistribute(lev_min, lev_max, nGrow); + // amrex::Gpu::synchronize(); + } + + virtual void RedistributeLocal (int /* lev_min = 0*/, + int /* lev_max =-1*/, + int /* nGrow = 0*/) override + { + AMREX_ASSERT(this->finestLevel() == 0); + + const int local = true; + const int lev_minal = 0; + const int lev_maxal = 0; + const int nGrowal = 0; + + amrex::NeighborParticleContainer + ::Redistribute(lev_minal, lev_maxal, nGrowal, local); + } + + virtual void RedistributeOK (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) override + { + + const int local = amrex::NeighborParticleContainer + ::OK(lev_min, lev_max); + // amrex::Print()<<"local flag is: "< + ::Redistribute(lev_min, lev_max, nGrow, local); + + } + + virtual void RedistributeGPU (int /*lev_min = 0*/, + int /*lev_max =-1*/, + int /*nGrow = 0*/) override + { + AMREX_ASSERT(this->finestLevel() == 0); + + int local = false; + const int lev_minal = 0; + const int lev_maxal = 0; + const int nGrowal = 0; + + amrex::Gpu::synchronize(); + local = amrex::NeighborParticleContainer + ::OK(lev_minal, lev_maxal); + // amrex::Print()<<"local flag is: "< + ::RedistributeGPU(lev_minal, lev_maxal, nGrowal, local); + else + amrex::NeighborParticleContainer + ::RedistributeCPU(lev_minal, lev_maxal, nGrowal, local); + } + + virtual void RemoveParticlesAtLevel (int level) override + { + amrex::NeighborParticleContainer::RemoveParticlesAtLevel(level); + } + + virtual void WriteNyxPlotFile (const std::string& dir, + const std::string& name) const; + + virtual void NyxCheckpoint (const std::string& dir, + const std::string& name) const; + + typedef amrex::Particle ParticleType; + using AoS = typename amrex::ParticleContainer::AoS; + using ParticleLevel = typename amrex::ParticleContainer::ParticleLevel; + +protected: + bool sub_cycle; + amrex::Vector real_comp_names; +}; + +template +void +NyxParticleContainer::GetParticleVelocities (amrex::Vector& part_data) +{ + BL_PROFILE("NyxParticleContainer::GetParticleVelocities()"); + // This assumes that the mass/charge is stored in the first position + // in the particle data, followed by the velocity components + int start_comp = 1; + int num_comp = AMREX_SPACEDIM; + this->GetParticleData(part_data,1,AMREX_SPACEDIM); +} + +template +void +NyxParticleContainer::SetParticleVelocities (amrex::Vector& part_data) +{ + BL_PROFILE("NyxParticleContainer::SetParticleVelocities()"); + // This gives us the starting point into the part_data array + // If only one processor (or no MPI), then that's all we need + int cnt = 0; + +#ifdef BL_USE_MPI + amrex::Vector cnts(amrex::ParallelDescriptor::NProcs()); + + // This returns the number of particles on this processor + long lcnt = this->TotalNumberOfParticles(true,true); + + // This accumulates the "lcnt" values into "cnts" + MPI_Gather(&lcnt,1, + amrex::ParallelDescriptor::Mpi_typemap::type(), + cnts.dataPtr(), + 1, + amrex::ParallelDescriptor::Mpi_typemap::type(), + amrex::ParallelDescriptor::IOProcessorNumber(), + amrex::ParallelDescriptor::Communicator()); + + amrex::ParallelDescriptor::Bcast(cnts.dataPtr(), cnts.size(), amrex::ParallelDescriptor::IOProcessorNumber()); + + for (int iproc = 0; iproc < amrex::ParallelDescriptor::MyProc(); iproc++) + cnt += cnts[iproc]; + + // Each particle takes up (AMREX_SPACEDIM) Reals + cnt*= (AMREX_SPACEDIM); +#endif + + // This is the total number of particles on *all* processors + long npart = this->TotalNumberOfParticles(true,false); + + // Velocities + if (part_data.size() != npart*(AMREX_SPACEDIM)) + amrex::Abort("Sending in wrong size part_data to SetParticleVelocities"); + + for (int lev = 0; lev <= this->m_gdb->finestLevel(); lev++) + { + ParticleLevel& pmap = this->GetParticles(lev); + + for (typename ParticleLevel::iterator pmap_it = pmap.begin(), pmapEnd = pmap.end(); pmap_it != pmapEnd; ++pmap_it) + { + AoS& pbx = pmap_it->second.GetArrayOfStructs(); + const int n = pbx.size(); + + for (int i = 0; i < n; i++) + { + ParticleType& p = pbx[i]; + if (p.id() > 0) + { + // Load velocities + for (int d=0; d < AMREX_SPACEDIM; d++) + p.rdata(d+1) = part_data[cnt+d]; + + // Update counter + cnt += AMREX_SPACEDIM; + } + } + } + } +} + +// +// Assumes mass is in rdata(0), vx in rdata(1), ...! +// dim defines the cartesian direction in which the momentum is summed, x is 0, y is 1, ... +// + +template +void +NyxParticleContainer::sumParticleMomentum (int lev, + amrex::Real* mom) const +{ + BL_PROFILE("NyxParticleContainer::sumParticleMomentum()"); + BL_ASSERT(NSR >= AMREX_SPACEDIM+1); + BL_ASSERT(lev >= 0 && lev < this->GetParticles().size()); + + const ParticleLevel& pmap = this->GetParticles(lev); + + AMREX_D_TERM(mom[0] = 0;, mom[1] = 0;, mom[2] = 0;); + + for (typename ParticleLevel::const_iterator pmap_it = pmap.begin(), pmapEnd = pmap.end(); pmap_it != pmapEnd; ++pmap_it) + { + const AoS& pbox = pmap_it->second.GetArrayOfStructs(); + const int n = pbox.size(); + + amrex::Real mom_0 = 0, mom_1 = 0, mom_2 = 0; + +#ifdef _OPENMP +#pragma omp parallel for reduction(+:mom_0,mom_1,mom_2) +#endif + for (int i = 0; i < n; i++) + { + const ParticleType& p = pbox[i]; + + if (p.id() > 0) + { + AMREX_D_TERM(mom_0 += p.rdata(0) * p.rdata(1);, + mom_1 += p.rdata(0) * p.rdata(2);, + mom_2 += p.rdata(0) * p.rdata(3);); + } + } + + AMREX_D_TERM(mom[0] += mom_0;, mom[1] += mom_1;, mom[2] += mom_2;); + } + + amrex::ParallelDescriptor::ReduceRealSum(mom,AMREX_SPACEDIM); +} + +template +amrex::Real +NyxParticleContainer::estTimestep (amrex::MultiFab& acceleration, + int lev, + amrex::Real cfl) const +{ + return estTimestep(acceleration,1.0,lev,cfl); +} + +template +amrex::Real +NyxParticleContainer::estTimestep (amrex::MultiFab& acceleration, + amrex::Real a, + int lev, + amrex::Real cfl) const +{ + BL_PROFILE("NyxParticleContainer::estTimestep(lev)"); + amrex::Real dt = 1e50; + BL_ASSERT(NSR >= AMREX_SPACEDIM+1); + BL_ASSERT(lev >= 0); + + if (this->GetParticles().size() == 0) + return dt; + + const amrex::Real strttime = amrex::ParallelDescriptor::second(); + + const amrex::Geometry& geom = this->m_gdb->Geom(lev); + const amrex::IntVect base = geom.Domain().smallEnd(); + const auto dxi = geom.InvCellSizeArray(); + amrex::GpuArray adxi{dxi[0]/a,dxi[1]/a,dxi[2]/a}; + const auto plo = geom.ProbLoArray(); + + long num_particles_at_level = 0; + + std::unique_ptr ac_pointer; + if (!this->OnSameGrids(lev, acceleration)) + { + ac_pointer.reset(new amrex::MultiFab(this->m_gdb->ParticleBoxArray(lev), + this->m_gdb->ParticleDistributionMap(lev), + acceleration.nComp(), acceleration.nGrow())); + if(acceleration.boxArray() == ac_pointer->boxArray())//this->finestLevel() == 0) + { + ac_pointer->Redistribute(acceleration,0,0,AMREX_SPACEDIM,acceleration.nGrowVect()); + ac_pointer->FillBoundary(geom.periodicity()); // DO WE NEED GHOST CELLS FILLED ??? + } + else + { + ac_pointer->ParallelCopy(acceleration,0,0,AMREX_SPACEDIM,acceleration.nGrowVect(),acceleration.nGrowVect()); + ac_pointer->FillBoundary(geom.periodicity()); // DO WE NEED GHOST CELLS FILLED ??? + } + } + +#ifdef _OPENMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) reduction(min:dt) +#endif + { + amrex::ReduceOps reduce_op; + amrex::ReduceData reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + + for (MyConstParIter pti(*this, lev); pti.isValid(); ++pti) { + const int grid = pti.index(); + const AoS& pbox = pti.GetArrayOfStructs(); + const ParticleType* pstruct = pbox().data(); + const int n = pbox.size(); + + const auto gfab = (ac_pointer) ? (*ac_pointer).array(grid) : acceleration.array(grid); + + num_particles_at_level += n; + + reduce_op.eval(n, reduce_data, + [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple { + const ParticleType& p = pstruct[i]; + + if (p.id() > 0) + { + + amrex::IntVect cell; + cell[0]= static_cast(amrex::Math::floor((p.pos(0)-plo[0])*dxi[0])); + cell[1]= static_cast(amrex::Math::floor((p.pos(1)-plo[1])*dxi[1])); + cell[2]= static_cast(amrex::Math::floor((p.pos(2)-plo[2])*dxi[2])); + cell += base; + + const amrex::Real mag_vel_over_dx = amrex::Math::abs(p.rdata(1))*adxi[0]; + const amrex::Real mag_vel_over_dy = amrex::Math::abs(p.rdata(2))*adxi[1]; + const amrex::Real mag_vel_over_dz = amrex::Math::abs(p.rdata(3))*adxi[2]; + + amrex::Real max_mag_vel_over_dx = amrex::max(mag_vel_over_dx,mag_vel_over_dy,mag_vel_over_dz); + amrex::Real dt_part = (max_mag_vel_over_dx > 0) ? (cfl / max_mag_vel_over_dx) : 1e50; + + const amrex::Real aval_x = gfab(cell[0],cell[1],cell[2],0); + const amrex::Real aval_y = gfab(cell[0],cell[1],cell[2],1); + const amrex::Real aval_z = gfab(cell[0],cell[1],cell[2],2); + const amrex::Real mag_accel = std::sqrt(aval_x*aval_x+aval_y*aval_y+aval_z*aval_z); + + if (mag_accel > 0) { + dt_part = amrex::min( dt_part, 1/std::sqrt(mag_accel*dxi[0]) ); + } + + return dt_part; + } + + return 1e50; + }); + } + + ReduceTuple hv = reduce_data.value(); + amrex::Real ldt_cpu = amrex::get<0>(hv); + + dt = std::min(dt,ldt_cpu); + } + + amrex::ParallelDescriptor::ReduceRealMin(dt); + + if (this->m_verbose > 1) + { + long min_local_num_particles_at_level = num_particles_at_level; + long max_local_num_particles_at_level = num_particles_at_level; + + amrex::ParallelDescriptor::ReduceLongMin(min_local_num_particles_at_level); + amrex::ParallelDescriptor::ReduceLongMax(max_local_num_particles_at_level); + + amrex::Print()<<"Min particles : "<m_verbose > 1) + { + amrex::Real stoptime = amrex::ParallelDescriptor::second() - strttime; + + amrex::ParallelDescriptor::ReduceRealMax(stoptime,amrex::ParallelDescriptor::IOProcessorNumber()); + + if (amrex::ParallelDescriptor::IOProcessor()) + { + std::cout << "NyxParticleContainer::estTimestep() time: " << stoptime << '\n'; + } + } + + return dt; +} + +template +void +NyxParticleContainer::MultiplyParticleMass (int lev, amrex::Real mult) +{ + BL_PROFILE("NyxParticleContainer::MultiplyParticleMass()"); + BL_ASSERT(lev == 0); + + ParticleLevel& pmap = this->GetParticles(lev); + + for (typename ParticleLevel::iterator pmap_it = pmap.begin(), pmapEnd = pmap.end(); pmap_it != pmapEnd; ++pmap_it) + { + AoS& pbx = pmap_it->second.GetArrayOfStructs(); + ParticleType* pstruct = pbx().data(); + const long np = pbx.size(); + + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + if (pstruct[i].id() > 0) + { + // + // Note: rdata(0) is mass, ... + // + pstruct[i].rdata(0) *= mult; + } + }); + } + amrex::Gpu::streamSynchronize(); +} + +template +void +NyxParticleContainer::WriteNyxPlotFile (const std::string& dir, + const std::string& name) const +{ + BL_PROFILE("NyxParticleContainer::WriteNyxPlotFile()"); + + amrex::NeighborParticleContainer::WritePlotFile(dir, name, real_comp_names); + +} + +template +void +NyxParticleContainer::NyxCheckpoint (const std::string& dir, + const std::string& name) const +{ + BL_PROFILE("NyxParticleContainer::NyxCheckpoint()"); + + bool is_checkpoint = true; + amrex::NeighborParticleContainer::Checkpoint(dir, name, is_checkpoint, real_comp_names); +} + +#endif /*_NyxParticleContainer_H_*/ diff --git a/Exec/HaloFinder/NyxParticles.cpp b/Exec/HaloFinder/NyxParticles.cpp new file mode 100644 index 000000000..aabe92ccc --- /dev/null +++ b/Exec/HaloFinder/NyxParticles.cpp @@ -0,0 +1,1273 @@ +#include +#include +#include + +#include + +using namespace amrex; + +namespace +{ + bool virtual_particles_set = false; + + std::string ascii_particle_file; + std::string binary_particle_file; + std::string sph_particle_file; + std::string restart_particle_file; + +#ifdef AGN + std::string agn_particle_file; +#endif + +#ifdef NEUTRINO_PARTICLES + std::string neutrino_particle_file; +#endif + + // const std::string chk_particle_file("DM"); + const std::string dm_chk_particle_file("DM"); + const std::string agn_chk_particle_file("AGN"); + const std::string npc_chk_particle_file("NPC"); + + // + // We want to call this routine when on exit to clean up particles. + // + + // + // Array of containers for all active particles + // + Vector ActiveParticles; + // + // Array of containers for all virtual particles + // + Vector VirtualParticles; + // + // Array of containers for all ghost particles + // + Vector GhostParticles; + + // + // Containers for the real "active" Particles + // + DarkMatterParticleContainer* DMPC = 0; + StellarParticleContainer* SPC = 0; +#ifdef AGN + AGNParticleContainer* APC = 0; +#endif +#ifdef NEUTRINO_PARTICLES + +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer* NPC = 0; +#else + NeutrinoParticleContainer* NPC = 0; +#endif + +#endif + + // + // This is only used as a temporary container for + // reading in SPH particles and using them to + // initialize the density and velocity field on the + // grids. + // + DarkMatterParticleContainer* SPHPC = 0; + // + // Container for temporary, virtual Particles + // + DarkMatterParticleContainer* VirtPC = 0; + StellarParticleContainer* VirtSPC = 0; +#ifdef AGN + AGNParticleContainer* VirtAPC = 0; +#endif +#ifdef NEUTRINO_PARTICLES + +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer* VirtNPC = 0; +#else + NeutrinoParticleContainer* VirtNPC = 0; + +#endif +#endif + // + // Container for temporary, ghost Particles + // + DarkMatterParticleContainer* GhostPC = 0; + StellarParticleContainer* GhostSPC = 0; +#ifdef AGN + AGNParticleContainer* GhostAPC = 0; +#endif +#ifdef NEUTRINO_PARTICLES +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer* GhostNPC = 0; +#else + NeutrinoParticleContainer* GhostNPC = 0; +#endif +#endif + DarkMatterParticleContainer* ShellPC = 0; + + void RemoveParticlesOnExit () + { + for (int i = 0; i < ActiveParticles.size(); i++) + { + delete ActiveParticles[i]; + ActiveParticles[i] = 0; + } + for (int i = 0; i < GhostParticles.size(); i++) + { + delete GhostParticles[i]; + GhostParticles[i] = 0; + } + for (int i = 0; i < VirtualParticles.size(); i++) + { + delete VirtualParticles[i]; + VirtualParticles[i] = 0; + } + } +} + +bool Nyx::do_dm_particles = false; +int Nyx::num_particle_ghosts = 1; +int Nyx::particle_skip_factor = 1; + +std::string Nyx::particle_init_type = ""; + +// Allows us to output particles in the plotfile +// in either single (IEEE32) or double (NATIVE) precision. +// Particles are always written in double precision +// in the checkpoint files. + +bool Nyx::particle_initrandom_serialize = false; +Real Nyx::particle_initrandom_mass = -1; +Real Nyx::particle_initrandom_mass_total = 869658119634944.0; +long Nyx::particle_initrandom_count; +long Nyx::particle_initrandom_count_per_box; +int Nyx::particle_initrandom_iseed; +Real Nyx::particle_inituniform_mass; +Real Nyx::particle_inituniform_vx; +Real Nyx::particle_inituniform_vy; +Real Nyx::particle_inituniform_vz; + +int Nyx::particle_launch_ics = 1; + +int Nyx::particle_verbose = 1; +int Nyx::write_particle_density_at_init = 0; +Real Nyx::particle_cfl = 0.5; +#ifdef NEUTRINO_PARTICLES +Real Nyx::neutrino_cfl = 0.5; +#endif + +Vector& +Nyx::theActiveParticles () +{ + return ActiveParticles; +} + +Vector& +Nyx::theGhostParticles () +{ + return GhostParticles; +} + +Vector& +Nyx::theVirtualParticles () +{ + return VirtualParticles; +} + +DarkMatterParticleContainer* +Nyx::theDMPC () +{ + return DMPC; +} + +DarkMatterParticleContainer* +Nyx::theVirtPC () +{ + return VirtPC; +} +DarkMatterParticleContainer* +Nyx::theGhostPC () +{ + return GhostPC; +} + +StellarParticleContainer* +Nyx::theSPC () +{ + return SPC; +} +StellarParticleContainer* +Nyx::theVirtSPC () +{ + return VirtSPC; +} +StellarParticleContainer* +Nyx::theGhostSPC () +{ + return GhostSPC; +} + +#ifdef AGN +AGNParticleContainer* +Nyx::theAPC () +{ + return APC; +} +AGNParticleContainer* +Nyx::theVirtAPC () +{ + return VirtAPC; +} +AGNParticleContainer* +Nyx::theGhostAPC () +{ + return GhostAPC; +} +#endif + +#ifdef NEUTRINO_PARTICLES +#ifdef NEUTRINO_DARK_PARTICLES +DarkMatterParticleContainer* +Nyx::theNPC () +{ + return NPC; +} +DarkMatterParticleContainer* +Nyx::theVirtNPC () +{ + return VirtNPC; +} +DarkMatterParticleContainer* +Nyx::theGhostNPC () +{ + return GhostNPC; +} +#else +NeutrinoParticleContainer* +Nyx::theNPC () +{ + return NPC; +} +NeutrinoParticleContainer* +Nyx::theVirtNPC () +{ + return VirtNPC; +} +NeutrinoParticleContainer* +Nyx::theGhostNPC () +{ + return GhostNPC; +} +#endif +#endif + +DarkMatterParticleContainer* +Nyx::theShellPC () +{ + return ShellPC; +} + +void +Nyx::read_particle_params () +{ + ParmParse pp("nyx"); + pp.query("do_dm_particles", do_dm_particles); + if(!do_dm_particles) + return; +#ifdef AGN + pp.get("particle_init_type", particle_init_type); +#else + if (do_dm_particles) + { + pp.get("particle_init_type", particle_init_type); + pp.query("init_with_sph_particles", init_with_sph_particles); + } +#endif + + pp.query("particle_initrandom_serialize", particle_initrandom_serialize); + pp.query("particle_initrandom_count", particle_initrandom_count); + pp.query("particle_initrandom_count_per_box", particle_initrandom_count_per_box); + pp.query("particle_initrandom_mass", particle_initrandom_mass); + pp.query("particle_initrandom_mass_total", particle_initrandom_mass_total); + pp.query("particle_initrandom_iseed", particle_initrandom_iseed); + pp.query("particle_skip_factor", particle_skip_factor); + pp.query("ascii_particle_file", ascii_particle_file); + pp.query("particle_inituniform_mass", particle_inituniform_mass); + pp.query("particle_inituniform_vx", particle_inituniform_vx); + pp.query("particle_inituniform_vy", particle_inituniform_vy); + pp.query("particle_inituniform_vz", particle_inituniform_vz); + + pp.query("restart_particle_file", restart_particle_file); + + // Input error check + if (do_dm_particles && !ascii_particle_file.empty() && particle_init_type != "AsciiFile") + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not AsciiFile but you specified ascii_particle_file" << std::endl;; + amrex::Error(); + } + + pp.query("sph_particle_file", sph_particle_file); + + // Input error check + if (init_with_sph_particles != 1 && !sph_particle_file.empty()) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::init_with_sph_particles is not 1 but you specified sph_particle_file" << std::endl;; + amrex::Error(); + } + + // Input error check + if (init_with_sph_particles == 1 && sph_particle_file.empty()) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::init_with_sph_particles is 1 but you did not specify sph_particle_file" << std::endl;; + amrex::Error(); + } + + pp.query("binary_particle_file", binary_particle_file); + + // Input error check + if (!binary_particle_file.empty() && (particle_init_type != "BinaryFile" && + particle_init_type != "BinaryMetaFile" && + particle_init_type != "BinaryMortonFile")) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not BinaryFile, BinaryMetaFile, or BinaryMortonFile but you specified binary_particle_file" << std::endl; + amrex::Error(); + } + +#ifdef AGN + pp.query("agn_particle_file", agn_particle_file); + if (!agn_particle_file.empty() && particle_init_type != "AsciiFile") + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not AsciiFile but you specified agn_particle_file" << std::endl;; + amrex::Error(); + } +#endif + +#ifdef NEUTRINO_PARTICLES + pp.query("neutrino_particle_file", neutrino_particle_file); + if (!neutrino_particle_file.empty() && (particle_init_type != "AsciiFile"&& + particle_init_type != "BinaryMetaFile" && + particle_init_type != "BinaryFile" )) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not AsciiFile or BinaryFile but you specified neutrino_particle_file" << std::endl;; + amrex::Error(); + } +#endif + + pp.query("write_particle_density_at_init", write_particle_density_at_init); + + if((particle_initrandom_mass <= 0 && particle_initrandom_mass_total <= 0) + && (particle_init_type == "Random" || + particle_init_type == "RandomPerBox" || + particle_init_type == "RandomPerCell" )) + amrex::Abort("Starting random intialization with particles of non-positive mass"); + + // + // Control the verbosity of the Particle class + // + ParmParse ppp("particles"); + ppp.query("v", particle_verbose); + + // + // Set the cfl for particle motion (fraction of cell that a particle can + // move in a timestep). + // + ppp.query("cfl", particle_cfl); +#ifdef NEUTRINO_PARTICLES + ppp.query("neutrino_cfl", neutrino_cfl); +#endif +} + +void +Nyx::init_particles () +{ + BL_PROFILE("Nyx::init_particles()"); + + if (level > 0) + return; + + // + // Need to initialize particles before defining gravity. + // + if (do_dm_particles) + { + BL_ASSERT (DMPC == 0); + DMPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(DMPC); + if (false)//do_lightcone + ShellPC = new DarkMatterParticleContainer(parent); + + if (init_with_sph_particles == 1) + SPHPC = new DarkMatterParticleContainer(parent); + + if (parent->subCycle()) + { + VirtPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtPC); + + GhostPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostPC); } + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + DMPC->SetVerbose(particle_verbose); + + // Check particle mass for random setups +#ifdef AMREX_USE_SINGLE_PRECISION_PARTICLES + Real tol = 1e-6; +#else + Real tol = 1e-12; +#endif + + Real number_of_cells = Geom().ProbLength(0)/Geom().CellSize(0) * + Geom().ProbLength(1)/Geom().CellSize(1) * + Geom().ProbLength(2)/Geom().CellSize(2); + if(particle_initrandom_mass <= 0) { + particle_initrandom_mass = particle_initrandom_mass_total / number_of_cells; + if(verbose) + amrex::Print()<<"... setting particle_initrandom_mass to "< tol + && (particle_init_type == "Random" || + particle_init_type == "RandomPerBox" || + particle_init_type == "RandomPerCell" )) + amrex::Abort("Starting random intialization with particle_initrandom_mass_total / (n_cell)^3 != particle_initrandom_mass"); + + + DarkMatterParticleContainer::ParticleInitData pdata = {{particle_initrandom_mass}, {}, {}, {}}; + + if (particle_init_type == "Random") + { + if (particle_initrandom_count <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_count must be > 0"); + } + if (particle_initrandom_iseed <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_iseed must be > 0"); + } + + if (verbose) + { + amrex::Print() << "\nInitializing DM with cloud of " + << particle_initrandom_count + << " random particles with initial seed: " + << particle_initrandom_iseed << "\n\n"; + } + { + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitRandom(particle_initrandom_count, + particle_initrandom_iseed, pdata, + particle_initrandom_serialize); + } + + } + else if (particle_init_type == "RandomPerBox") + { + if (particle_initrandom_count_per_box <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_count_per_box must be > 0"); + } + if (particle_initrandom_iseed <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_iseed must be > 0"); + } + + if (verbose) + amrex::Print() << "\nInitializing DM with of " << particle_initrandom_count_per_box + << " random particles per box with initial seed: " + << particle_initrandom_iseed << "\n\n"; + + DMPC->InitRandomPerBox(particle_initrandom_count_per_box, + particle_initrandom_iseed, pdata); + + } + else if (particle_init_type == "RandomPerCell") + { + if (verbose) + amrex::Print() << "\nInitializing DM with 1 random particle per cell " << "\n"; + + int n_per_cell = 1; + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitNRandomPerCell(n_per_cell, pdata); + amrex::Gpu::Device::synchronize(); + + } + else if (particle_init_type == "OnePerCell") + { + if (verbose) + amrex::Print() << "\nInitializing DM with 1 uniform particle per cell " << "\n"; + + // int n_per_cell = 1; + DarkMatterParticleContainer::ParticleInitData pdata_vel = {{particle_inituniform_mass, particle_inituniform_vx, particle_inituniform_vy, particle_inituniform_vz},{},{},{}}; + DMPC->InitOnePerCell(0.5, 0.5, 0.5, pdata_vel); + } + else if (particle_init_type == "AsciiFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from \"" + << ascii_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Print() << "\nInitializing SPH particles from ascii \"" + << sph_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass and velocity. + // + DMPC->InitFromAsciiFile(ascii_particle_file, AMREX_SPACEDIM + 1); + if (init_with_sph_particles == 1) + SPHPC->InitFromAsciiFile(sph_particle_file, AMREX_SPACEDIM + 1); + } + else if (particle_init_type == "BinaryFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from \"" + << binary_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Print() << "\nInitializing SPH particles from binary \"" + << sph_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass and velocity. + // + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitFromBinaryFile(binary_particle_file, AMREX_SPACEDIM + 1); + if (init_with_sph_particles == 1) + SPHPC->InitFromBinaryFile(sph_particle_file, AMREX_SPACEDIM + 1); + + } + else if (particle_init_type == "BinaryMetaFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from meta file\"" + << binary_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Print() << "\nInitializing SPH particles from meta file\"" + << sph_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]` in each of the binary particle files. + // Here we're reading in the particle mass and velocity. + // + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitFromBinaryMetaFile(binary_particle_file, AMREX_SPACEDIM + 1); + if (init_with_sph_particles == 1) + SPHPC->InitFromBinaryMetaFile(sph_particle_file, AMREX_SPACEDIM + 1); + } + else if (particle_init_type == "BinaryMortonFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from morton-ordered binary file\"" + << binary_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Error("Morton-ordered input is not supported for sph particles."); + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]` in each of the binary particle files. + // Here we're reading in the particle mass and velocity. + // + DMPC->InitFromBinaryMortonFile(binary_particle_file, + AMREX_SPACEDIM + 1, + particle_skip_factor); + } + else if (particle_init_type == "Restart") + { + DMPC->Restart(restart_particle_file, dm_chk_particle_file); + } + else + { + amrex::Error("not a valid input for nyx.particle_init_type"); + } + } +#ifdef AGN + { + // Note that we don't initialize any actual AGN particles here, we just create the container. + BL_ASSERT (APC == 0); + APC = new AGNParticleContainer(parent, num_particle_ghosts); + ActiveParticles.push_back(APC); + + if (parent->subCycle()) + { + VirtAPC = new AGNParticleContainer(parent, num_particle_ghosts); + VirtualParticles.push_back(VirtAPC); + + GhostAPC = new AGNParticleContainer(parent, num_particle_ghosts); + GhostParticles.push_back(GhostAPC); + } + // + // 2 gives more stuff than 1. + // + APC->SetVerbose(particle_verbose); + } +#endif +#ifdef NEUTRINO_PARTICLES + { + BL_ASSERT (NPC == 0); +#ifdef NEUTRINO_DARK_PARTICLES + NPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(NPC); +#else + NPC = new NeutrinoParticleContainer(parent); + ActiveParticles.push_back(NPC); + + // Set the relativistic flag to 1 so that the density will be gamma-weighted + // when returned by AssignDensity calls. + NPC->SetRelativistic(1); + + // We must set the value for csquared which is used in computing gamma = 1 / sqrt(1-vsq/csq) + // Obviously this value is just a place-holder for now. + NPC->SetCSquared(1.); + +#endif + + if (parent->subCycle()) + { +#ifdef NEUTRINO_DARK_PARTICLES + VirtNPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtNPC); +#else + VirtNPC = new NeutrinoParticleContainer(parent); + VirtualParticles.push_back(VirtNPC); + + // Set the relativistic flag to 1 so that the density will be gamma-weighted + // when returned by AssignDensity calls. + VirtNPC->SetRelativistic(1); + +#endif + +#ifdef NEUTRINO_DARK_PARTICLES + GhostNPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostNPC); + +#else + GhostNPC = new NeutrinoParticleContainer(parent); + GhostParticles.push_back(GhostNPC); + + // Set the relativistic flag to 1 so that the density will be gamma-weighted + // when returned by AssignDensity calls. + GhostNPC->SetRelativistic(1); +#endif + + } + // + // Make sure to call RemoveParticlesOnExit() on exit. + // (if do_dm_particles then we have already called ExecOnFinalize) + // + if (!do_dm_particles) + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + NPC->SetVerbose(particle_verbose); + if (particle_init_type == "AsciiFile") + { + if (verbose) + amrex::Print() << "\nInitializing Neutrino particles from \"" + << neutrino_particle_file << "\" ...\n\n"; + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass, velocity and angles. + // +#ifdef NEUTRINO_DARK_PARTICLES + NPC->InitFromAsciiFile(neutrino_particle_file, AMREX_SPACEDIM + 1); +#else + NPC->InitFromAsciiFile(neutrino_particle_file, 2*AMREX_SPACEDIM + 1); +#endif + } + else if (particle_init_type == "BinaryFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing Neutrino particles from \"" + << neutrino_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass and velocity. + // + NPC->InitFromBinaryFile(neutrino_particle_file, AMREX_SPACEDIM + 1); + } + else if (particle_init_type == "BinaryMetaFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing NPC particles from meta file\"" + << neutrino_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]` in each of the binary particle files. + // Here we're reading in the particle mass and velocity. + // + NPC->InitFromBinaryMetaFile(neutrino_particle_file, AMREX_SPACEDIM + 1); + } + else + { + amrex::Error("for right now we only init Neutrino particles with ascii or binary"); + } + } + + if (write_particle_density_at_init == 1) + { + + Vector > particle_mf;//(new MultiFab(grids,dmap,1,1)); + + DMPC->AssignDensity(particle_mf,0,1,0,0); + + writeMultiFabAsPlotFile("ParticleDensity", *particle_mf[0], "density"); + +#ifdef NEUTRINO_PARTICLES + Vector > particle_npc_mf;//(new MultiFab(grids,dmap,1,1)); + // DMPC->AssignDensitySingleLevel(particle_mf,0,1,0,0); + + NPC->AssignDensity(particle_npc_mf,0,1,0,0); + + writeMultiFabAsPlotFile("ParticleNPCDensity", *particle_npc_mf[0], "density"); +#endif + exit(0); + } + +#endif +} + +#ifndef NO_HYDRO +void +Nyx::init_santa_barbara (int init_sb_vels) +{ + BL_PROFILE("Nyx::init_santa_barbara()"); + Real cur_time = state[State_Type].curTime(); + Real a = old_a; + + BL_PROFILE_VAR("Nyx::init_santa_barbara()::part", CA_part); + amrex::Print() << "... time and comoving a when data is initialized at level " + << level << " " << cur_time << " " << a << '\n'; + + if (level == 0) + { + Real frac_for_hydro = comoving_OmB / comoving_OmM; + Real omfrac = 1.0 - frac_for_hydro; + + if ( (init_with_sph_particles == 0) && (frac_for_hydro != 1.0) ) { + DMPC->MultiplyParticleMass(level, omfrac); + } + + Vector > particle_mf(1); + if (init_sb_vels == 1) + { + if (init_with_sph_particles == 1) { + SPHPC->AssignDensityAndVels(particle_mf); + } else { + DMPC->AssignDensityAndVels(particle_mf); + } + + } else { + if (init_with_sph_particles == 1) { + SPHPC->AssignDensity(particle_mf); + } else { + DMPC->AssignDensity(particle_mf); + } + } + + // As soon as we have used the SPH particles to define the density + // and velocity on the grid, we can go ahead and destroy them. + if (init_with_sph_particles == 1) { + delete SPHPC; + } + + BL_PROFILE_VAR_STOP(CA_part); + BL_PROFILE_VAR("Nyx::init_santa_barbara()::avg", CA_avg); + + for (int lev = parent->finestLevel()-1; lev >= 0; lev--) + { + amrex::average_down(*particle_mf[lev+1], *particle_mf[lev], + parent->Geom(lev+1), parent->Geom(lev), 0, 1, + parent->refRatio(lev)); + } + BL_PROFILE_VAR_STOP(CA_avg); + BL_PROFILE_VAR("Nyx::init_santa_barbara()::partmf", CA_partmf); + // Only multiply the density, not the velocities + if (init_with_sph_particles == 0) + { + if (frac_for_hydro == 1.0) + { + particle_mf[level]->mult(0,0,1); + } + else + { + particle_mf[level]->mult(frac_for_hydro / omfrac,0,1); + } + } + BL_PROFILE_VAR_STOP(CA_partmf); + + BL_PROFILE_VAR("Nyx::init_santa_barbara()::init", CA_init); + const auto geomdata = geom.data(); + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + // Temp unused for GammaLaw, set it here so that pltfiles have + // defined numbers + D_new.setVal(0, Temp_comp); + D_new.setVal(0, Ne_comp); + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + const auto fab_S_new=S_new.array(mfi); + const auto fab_D_new=D_new.array(mfi); + + GpuArray prob_param; + prob_param_fill(prob_param); + prob_param_special_fill(prob_param); + comoving_type=int(std::round(prob_param[comoving_type_comp])); + + prob_initdata_on_box(bx, fab_S_new, fab_D_new, geomdata, prob_param); + +// amrex::ParallelFor( +// bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { +// prob_initdata +// (i, j ,k, fab_S_new, fab_D_new, geomdata,prob_param); +// }); + } + + amrex::Gpu::Device::streamSynchronize(); + + if (inhomo_reion) init_zhi(); + + BL_PROFILE_VAR_STOP(CA_init); + + // Add the particle density to the gas density + MultiFab::Add(S_new, *particle_mf[level], 0, Density_comp, 1, 1); + + if (init_sb_vels == 1) + { + // Convert velocity to momentum + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + MultiFab::Multiply(*particle_mf[level], *particle_mf[level], 0, 1+i, 1, 0); + } + + // Add the particle momenta to the gas momenta (initially zero) + MultiFab::Add(S_new, *particle_mf[level], 1, Xmom_comp, AMREX_SPACEDIM, S_new.nGrow()); + } + + enforce_minimum_density_floor(S_new, new_a); + } else { + + MultiFab& S_new = get_new_data(State_Type); + FillCoarsePatch(S_new, 0, cur_time, State_Type, 0, S_new.nComp()); + + MultiFab& D_new = get_new_data(DiagEOS_Type); + FillCoarsePatch(D_new, 0, cur_time, DiagEOS_Type, 0, D_new.nComp()); + + MultiFab& Phi_new = get_new_data(PhiGrav_Type); + FillCoarsePatch(Phi_new, 0, cur_time, PhiGrav_Type, 0, Phi_new.nComp()); + +#ifndef CONST_SPECIES + // Convert (rho X)_i to X_i before calling init_e_from_T + for (int i = 0; i < NumSpec; ++i) + MultiFab::Divide(S_new, S_new, Density_comp, FirstSpec_comp+i, 1, 0); +#endif + } + + init_e_from_T(a); + +#ifndef CONST_SPECIES + // Convert X_i to (rho X)_i + MultiFab& S_new = get_new_data(State_Type); + for (int i = 0; i < NumSpec; ++i) + MultiFab::Multiply(S_new, S_new, Density_comp, FirstSpec_comp+i, 1, 0); +#endif +} +#endif + +void +Nyx::particle_post_restart (const std::string& restart_file, bool is_checkpoint) +{ + BL_PROFILE("Nyx::particle_post_restart()"); + + if (level > 0) + return; + + if (do_dm_particles) + { + BL_ASSERT(DMPC == 0); + DMPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(DMPC); + + if (parent->subCycle()) + { + VirtPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtPC); + + GhostPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostPC); + } + + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + + // + // 2 gives more stuff than 1. + // + DMPC->SetVerbose(particle_verbose); + + { + DMPC->Restart(restart_file, dm_chk_particle_file, is_checkpoint); + amrex::Gpu::Device::streamSynchronize(); + } + // + // We want the ability to write the particles out to an ascii file. + // + ParmParse pp("particles"); + + std::string dm_particle_output_file; + pp.query("dm_particle_output_file", dm_particle_output_file); + + if (!dm_particle_output_file.empty()) + { + DMPC->WriteAsciiFile(dm_particle_output_file); + } + } +#ifdef AGN + { + BL_ASSERT(APC == 0); + APC = new AGNParticleContainer(parent, num_particle_ghosts); + ActiveParticles.push_back(APC); + + if (parent->subCycle()) + { + VirtAPC = new AGNParticleContainer(parent, num_particle_ghosts); + VirtualParticles.push_back(VirtAPC); + + GhostAPC = new AGNParticleContainer(parent, num_particle_ghosts); + GhostParticles.push_back(GhostAPC); + } + + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + APC->SetVerbose(particle_verbose); + APC->Restart(restart_file, agn_chk_particle_file, is_checkpoint); + // + // We want the ability to write the particles out to an ascii file. + // + ParmParse pp("particles"); + + std::string agn_particle_output_file; + pp.query("agn_particle_output_file", agn_particle_output_file); + + if (!agn_particle_output_file.empty()) + { + APC->WriteAsciiFile(agn_particle_output_file); + } + } +#endif + +#ifdef NEUTRINO_DARK_PARTICLES + { + BL_ASSERT(NPC == 0); + NPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(NPC); + + if (parent->subCycle()) + { + VirtNPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtNPC); + + GhostNPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostNPC); + } + + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + NPC->SetVerbose(particle_verbose); + NPC->Restart(restart_file, npc_chk_particle_file, is_checkpoint); + // + // We want the ability to write the particles out to an ascii file. + // + ParmParse pp("particles"); + + std::string npc_particle_output_file; + pp.query("npc_particle_output_file", npc_particle_output_file); + + if (!npc_particle_output_file.empty()) + { + NPC->WriteAsciiFile(npc_particle_output_file); + } + } +#endif +} + +void +Nyx::particle_est_time_step (Real& est_dt) +{ + BL_PROFILE("Nyx::particle_est_time_step()"); + const Real cur_time = state[PhiGrav_Type].curTime(); + const Real a = get_comoving_a(cur_time); + MultiFab& grav = get_new_data(Gravity_Type); + const Real est_dt_particle = DMPC->estTimestep(grav, a, level, particle_cfl); + + if (est_dt_particle > 0) { + est_dt = std::min(est_dt, est_dt_particle); + } + +#ifdef NEUTRINO_PARTICLES + const Real est_dt_neutrino = NPC->estTimestep(grav, a, level, neutrino_cfl); + if (est_dt_neutrino > 0) { + est_dt = std::min(est_dt, est_dt_neutrino); + } +#endif + + if (verbose) + { + if (est_dt_particle > 0) + { + amrex::Print() << "...estdt from particles at level " + << level << ": " << est_dt_particle << '\n'; + } + else + { + amrex::Print() << "...there are no particles at level " + << level << '\n'; + } +#ifdef NEUTRINO_PARTICLES + if (est_dt_neutrino > 0) + { + amrex::Print() << "...estdt from neutrinos at level " + << level << ": " << est_dt_neutrino << '\n'; + } +#endif + } +} + +void +Nyx::particle_redistribute (int lbase, bool my_init) +{ + BL_PROFILE("Nyx::particle_redistribute()"); + if (DMPC) + { + + + // + // If we are calling with my_init = true, then we want to force the redistribute + // without checking whether the grids have changed. + // + if (my_init) + { + DMPC->Redistribute(lbase); + return; + } + + // + // These are usually the BoxArray and DMap from the last regridding. + // + static Vector ba; + static Vector dm; + + bool changed = false; + bool dm_changed = false; + bool ba_changed = false; + bool ba_size_changed = false; + + int flev = parent->finestLevel(); + + while ( parent->getAmrLevels()[flev] == nullptr ) { + flev--; + } + + if (ba.size() != flev+1) + { + amrex::Print() << "BA SIZE " << ba.size() << std::endl; + amrex::Print() << "FLEV " << flev << std::endl; + ba.resize(flev+1); + dm.resize(flev+1); + changed = true; + ba_size_changed = true; + } + else + { + for (int i = 0; i <= flev && !changed; i++) + { + if (ba[i] != parent->boxArray(i)) + { + // + // The BoxArrays have changed in the regridding. + // + changed = true; + ba_changed = true; + } + + if ( ! changed) + { + if (dm[i] != parent->getLevel(i).get_new_data(0).DistributionMap()) + { + // + // The DistributionMaps have changed in the regridding. + // + changed = true; + dm_changed = true; + } + } + } + } + + if (changed) + { + // + // We only need to call Redistribute if the BoxArrays or DistMaps have changed. + // We also only call it for particles >= lbase. This is + // because of we called redistribute during a subcycle, there may be particles not in + // the proper position on coarser levels. + // + if (verbose) + { + if (ba_size_changed) + amrex::Print() << "Calling redistribute because the size of BoxArray changed " << '\n'; + else if (ba_changed) + amrex::Print() << "Calling redistribute because BoxArray changed " << '\n'; + else if (dm_changed) + amrex::Print() << "Calling redistribute because DistMap changed " << '\n'; + } + + int iteration = 1; + for (int i = 0; i < theActiveParticles().size(); i++) + { + theActiveParticles()[i]->Redistribute(lbase, + theActiveParticles()[i]->finestLevel(), + iteration); + + } + + // + // Use the new BoxArray and DistMap to define ba and dm for next time. + // + for (int i = 0; i <= flev; i++) + { + ba[i] = parent->boxArray(i); + dm[i] = parent->getLevel(i).get_new_data(0).DistributionMap(); + } + } + else + { + if (verbose) + amrex::Print() << "NOT calling redistribute because NOT changed " << '\n'; + } + } +} + +void +Nyx::setup_virtual_particles() +{ + BL_PROFILE("Nyx::setup_virtual_particles()"); + + if(Nyx::theDMPC() != 0 && !virtual_particles_set) + { + DarkMatterParticleContainer::AoS virts; + if (level < parent->finestLevel()) + { + get_level(level + 1).setup_virtual_particles(); + Nyx::theVirtPC()->CreateVirtualParticles(level+1, virts); + Nyx::theVirtPC()->AddParticlesAtLevel(virts, level); + Nyx::theDMPC()->CreateVirtualParticles(level+1, virts); + Nyx::theVirtPC()->AddParticlesAtLevel(virts, level); + } + virtual_particles_set = true; + } +} + +void +Nyx::remove_virtual_particles() +{ + BL_PROFILE("Nyx::remove_virtual_particles()"); + + for (int i = 0; i < VirtualParticles.size(); i++) + { + if (VirtualParticles[i] != 0) + VirtualParticles[i]->RemoveParticlesAtLevel(level); + virtual_particles_set = false; + } +} + +void +Nyx::setup_ghost_particles(int ngrow) +{ + BL_PROFILE("Nyx::setup_ghost_particles()"); + BL_ASSERT(level < parent->finestLevel()); + + if(Nyx::theDMPC() != 0) + { + DarkMatterParticleContainer::AoS ghosts; + Nyx::theDMPC()->CreateGhostParticles(level, ngrow, ghosts); + Nyx::theGhostPC()->AddParticlesAtLevel(ghosts, level+1, ngrow); + } +#ifdef AGN + if(Nyx::theAPC() != 0) + { + AGNParticleContainer::AoS ghosts; + Nyx::theAPC()->CreateGhostParticles(level, ngrow, ghosts); + Nyx::theGhostAPC()->AddParticlesAtLevel(ghosts, level+1, ngrow); + } +#endif +#ifdef NEUTRINO_PARTICLES + if(Nyx::theNPC() != 0) + { +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer::AoS ghosts; +#else + NeutrinoParticleContainer::AoS ghosts; +#endif + Nyx::theNPC()->CreateGhostParticles(level, ngrow, ghosts); + Nyx::theGhostNPC()->AddParticlesAtLevel(ghosts, level+1, ngrow); + } +#endif + amrex::Gpu::Device::streamSynchronize(); +} + +void +Nyx::remove_ghost_particles() +{ + BL_PROFILE("Nyx::remove_ghost_particles()"); + + for (int i = 0; i < GhostParticles.size(); i++) + { + if (GhostParticles[i] != 0) + GhostParticles[i]->RemoveParticlesAtLevel(level); + } +} + +//NyxParticleContainerBase::~NyxParticleContainerBase() {} diff --git a/Exec/HaloFinder/Nyx_advance.cpp b/Exec/HaloFinder/Nyx_advance.cpp new file mode 100644 index 000000000..06e959f0b --- /dev/null +++ b/Exec/HaloFinder/Nyx_advance.cpp @@ -0,0 +1,631 @@ + +#include +#include +#include +#include +#include + +using namespace amrex; + +using std::string; + +std::vector shell_particles; + +Real +Nyx::advance (Real time, + Real dt, + int iteration, + int ncycle) + + // Arguments: + // time : the current simulation time + // dt : the timestep to advance (e.g., go from time to + // time + dt) + // iteration : where we are in the current AMR subcycle. Each + // level will take a number of steps to reach the + // final time of the coarser level below it. This + // counter starts at 1 + // ncycle : the number of subcycles at this level + +{ + + const std::string region_name = ncycle > 1 ? "R::Nyx::advance" : "R::Nyx::advance::STEP1"; + BL_PROFILE_REGION(region_name); + MultiFab::RegionTag amrlevel_tag("AmrLevel_Level_" + std::to_string(level)); + +#ifdef NO_HYDRO + + return advance_particles_only(time, dt, iteration, ncycle); + +#else + if (do_hydro) + { + if (Nyx::theActiveParticles().size() > 0) + { +#ifndef AGN + if (do_dm_particles) +#endif + { + return advance_hydro_plus_particles(time, dt, iteration, ncycle); + } + } + else + { + return advance_hydro(time, dt, iteration, ncycle); + } + } + else if(do_dm_particles) + return advance_particles_only(time, dt, iteration, ncycle); + else + return advance_heatcool(time, dt, iteration, ncycle); +#endif + return 0; +} + +// +// This will advance the Nyx AmrLevel. +// If no subcycling is used, this will be a full multilevel advance at +// at level 0 and a finer levels will be skipped over. +// If subcycling is used, this will check if finer levels are subcycled +// relative to this level. All levels that are not subcycled will be +// advanced in a multilevel advance with this level and be skipped over +// subsequently. If the next level is subcycled relative to this one, +// then this will only be a single level advance. +// +#ifndef NO_HYDRO +#ifdef AMREX_PARTICLES +Real +Nyx::advance_hydro_plus_particles (Real time, + Real dt, + int iteration, + int ncycle) +{ + + // A particle in cell (i) can affect cell values in (i-1) to (i+1) + int stencil_deposition_width = 1; + + // A particle in cell (i) may need information from cell values in (i-1) to (i+1) + // to update its position (typically via interpolation of the acceleration from the grid) + int stencil_interpolation_width = 1; + + // A particle that starts in cell (i + ncycle) can reach + // cell (i) in ncycle number of steps .. after "iteration" steps + // the particle has to be within (i + ncycle+1-iteration) to reach cell (i) + // in the remaining (ncycle-iteration) steps + + // *** ghost_width *** is used + // *) to set how many cells are used to hold ghost particles i.e copies of particles + // that live on (level-1) can affect the grid over all of the ncycle steps. + // We define ghost cells at the coarser level to cover all iterations so + // we can't reduce this number as iteration increases. + + int ghost_width = ncycle + stencil_deposition_width; + + // *** where_width *** is used + // *) to set how many cells the Where call in moveKickDrift tests = + // ghost_width + (1-iteration) - 1: + // the minus 1 arises because this occurs *after* the move + + int where_width = ghost_width + (1-iteration) - 1; + + // *** grav_n_grow *** is used + // *) to determine how many ghost cells we need to fill in the MultiFab from + // which the particle interpolates its acceleration + // *) to set how many cells the Where call in moveKickDrift tests = (grav.nGrow()-2). + // *) the (1-iteration) arises because the ghost particles are created on the coarser + // level which means in iteration 2 the ghost particles may have moved 1 additional cell along + + int grav_n_grow = ghost_width + (1-iteration) + (iteration-1) + + stencil_interpolation_width ; + + { + BL_PROFILE_REGION("R::Nyx::advance_hydro_plus_particles"); + BL_PROFILE("Nyx::advance_hydro_plus_particles()"); + // Sanity checks + if (!do_hydro) + amrex::Abort("In `advance_hydro_plus_particles` but `do_hydro` not true"); + + if (Nyx::theActiveParticles().size() <= 0) + amrex::Abort("In `advance_hydro_plus_particles` but no active particles"); + + const int finest_level = parent->finestLevel(); + int finest_level_to_advance; + bool nosub = !parent->subCycle(); + + if (nosub) + { + if (level > 0) + return dt; + + finest_level_to_advance = finest_level; + } + else + { + // This level was advanced by a previous multilevel advance. + if (level > 0 && ncycle == 1) + return dt; + + // Find the finest level to advance + int lev = level; + while(lev < finest_level && parent->nCycle(lev+1) == 1) + lev++; + finest_level_to_advance = lev; + + // We must setup virtual and Ghost Particles + // + // Setup the virtual particles that represent finer level particles + // + setup_virtual_particles(); + // + // Setup ghost particles for use in finer levels. Note that Ghost particles + // that will be used by this level have already been created, the + // particles being set here are only used by finer levels. + // + for(int lev = level; lev <= finest_level_to_advance && lev < finest_level; lev++) + { + get_level(lev).setup_ghost_particles(ghost_width); + } + } + + Real dt_lev; + + // + // Move current data to previous, clear current. + // Don't do this if a coarser level has done this already. + // + if (level == 0 || iteration > 1) + { + + for (int lev = level; lev <= finest_level; lev++) + { + dt_lev = parent->dtLevel(lev); + for (int k = 0; k < NUM_STATE_TYPE; k++) + { + get_level(lev).state[k].allocOldData(); + get_level(lev).state[k].swapTimeLevels(dt_lev); + } + } + } + + const Real prev_time = state[State_Type].prevTime(); + const Real cur_time = state[State_Type].curTime(); + const Real a_old = get_comoving_a(prev_time); + const Real a_new = get_comoving_a(cur_time); + + if (do_grav) { + // + // We now do a multilevel solve for old Gravity. This goes to the + // finest level regardless of subcycling behavior. Consequentially, + // If we are subcycling we skip this step on the first iteration of + // finer levels. + BL_PROFILE_VAR("solve_for_old_phi", solve_for_old_phi); + if (level == 0 || iteration > 1) + { + + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + + // fix fluxes on finer grids + if (do_reflux) + { + for (int lev = level; lev < finest_level; lev++) + { + gravity->zero_phi_flux_reg(lev + 1); + } + } + + // swap grav data + for (int lev = level; lev <= finest_level; lev++) + get_level(lev).gravity->swap_time_levels(lev); + + // + // Solve for phi using the previous phi as a guess. + // + int use_previous_phi_as_guess = 1; + int ngrow_for_solve = iteration + stencil_deposition_width; + gravity->multilevel_solve_for_old_phi(level, finest_level, + ngrow_for_solve, + use_previous_phi_as_guess); + } + BL_PROFILE_VAR_STOP(solve_for_old_phi); + + { + // + // Advance Particles + // + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities to the half-time and the positions to the new time + // We use the cell-centered gravity to correctly interpolate onto particle locations + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKickDrift ... updating particle positions and velocity\n"; + + MultiFab::RegionTag amrMoveKickDrift_tag("MoveKickDrift_" + std::to_string(level)); + + shell_particles.clear(); + shell_particles.shrink_to_fit(); + + std::string filename=Concatenate("lightcone_", int(100*(1/a_old-1)), 7); + std::string filename_vtk="./Output/LightCones/VTK/"+filename+".vtk"; + std::string filename_bin="./Output/LightCones/SimpleBinary/"+filename+".bin"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + // We need grav_n_grow grow cells to track boundary particles + const auto& ba = get_level(lev).get_new_data(State_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(State_Type).DistributionMap(); + MultiFab grav_vec_old(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_old_grav_vector(lev, grav_vec_old, time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) { + Real radius_inner = -1.e34; + Real radius_outer = -1.e34; + Real z_old = 1/a_old-1; + if(z_old=lightcone_end_z) { + integrate_distance_given_a(a_old, 1.0, radius_outer); + integrate_distance_given_a(a_new, 1.0, radius_inner); + Print()<moveKickDrift(grav_vec_old, lev, time, dt, a_old, a_half, where_width, radius_inner, radius_outer); + } + Real z_old = 1/a_old-1; + if(z_old=lightcone_end_z) { + //writeBinaryVTK(filename_vtk, shell_particles); + writeBinarySimple(filename_bin, shell_particles); + } + + + // Only need the coarsest virtual particles here. + if (lev == level && level < finest_level) + for (int i = 0; i < Nyx::theVirtualParticles().size(); i++) + Nyx::theVirtualParticles()[i]->moveKickDrift(grav_vec_old, lev, time, dt, a_old, a_half, where_width); + + // Miiiight need all Ghosts + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKickDrift(grav_vec_old, lev, time, dt, a_old, a_half, where_width); +#ifdef NEUTRINO_DARK_PARTICLES + for (int i = 0; i < Nyx::theActiveParticles().size() && neutrino_cfl >= 1; i++) + Nyx::theActiveParticles()[i]->Redistribute(); +#endif + } + } // if active particles + } // lsg + } // if (do_grav) + + // + // Call the hydro advance at each level to be advanced + // + BL_PROFILE_VAR("just_the_hydro", just_the_hydro); + { + + MultiFab::RegionTag amrhydro_tag("Hydro_" + std::to_string(level)); + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { +#ifdef SDC + if (sdc_split > 0 && (strang_restart_from_sdc == 0)) { + get_level(lev).sdc_hydro(time, dt, a_old, a_new); + } else { + get_level(lev).strang_hydro(time, dt, a_old, a_new); + } +#else + get_level(lev).strang_hydro(time, dt, a_old, a_new); +#endif + } + } + BL_PROFILE_VAR_STOP(just_the_hydro); + + { + // + // We must reflux before doing the next gravity solve + // + if (do_reflux) + { + for (int lev = level; lev < finest_level_to_advance; lev++) + { + get_level(lev).reflux(); + } + } + + // Always average down the new state from finer to coarser. + for (int lev = finest_level_to_advance-1; lev >= level; lev--) + { + get_level(lev).average_down( State_Type); + get_level(lev).average_down(DiagEOS_Type); + } + + // + // Here we use the "old" phi from the current time step as a guess for this + // solve + // + if (do_grav) + { + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::Copy(parent->getLevel(lev).get_new_data(PhiGrav_Type), + parent->getLevel(lev).get_old_data(PhiGrav_Type), + 0, 0, 1, 0); + } + + // Solve for new Gravity + BL_PROFILE_VAR("solve_for_new_phi", solve_for_new_phi); + int use_previous_phi_as_guess = 1; + if (finest_level_to_advance > level) + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + // The particle may be as many as "iteration" ghost cells out + int ngrow_for_solve = iteration + stencil_deposition_width; + gravity->multilevel_solve_for_new_phi(level, finest_level_to_advance, + ngrow_for_solve, + use_previous_phi_as_guess); + } + else + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + int fill_interior = 0; + gravity->solve_for_new_phi(level,get_new_data(PhiGrav_Type), + gravity->get_grad_phi_curr(level), + fill_interior, grav_n_grow); + } + BL_PROFILE_VAR_STOP(solve_for_new_phi); + + // Reflux + if (do_reflux) + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + gravity->add_to_fluxes(lev, iteration, ncycle); + } + } + + // + // Now do corrector part of source term update + // + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(lev)); + + // Now do corrector part of source term update + correct_gsrc(lev,time,prev_time,cur_time,dt); + + MultiFab& S_new = get_level(lev).get_new_data(State_Type); + MultiFab& D_new = get_level(lev).get_new_data(DiagEOS_Type); + + // First reset internal energy before call to compute_temp + MultiFab reset_e_src(S_new.boxArray(), S_new.DistributionMap(), 1, NUM_GROW); + reset_e_src.setVal(0.0); + get_level(lev).reset_internal_energy(S_new,D_new,reset_e_src); + + get_level(lev).compute_new_temp(S_new,D_new); + } + + // Must average down again after doing the gravity correction; + // always average down from finer to coarser. + // Here we average down both the new state and phi and gravity. + for (int lev = finest_level_to_advance-1; lev >= level; lev--) + get_level(lev).average_down(); + + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities by dt/2 to the new time. We use the + // cell-centered gravity to correctly interpolate onto particle + // locations. + MultiFab::RegionTag amrMoveKickDrift_tag("MoveKick_" + std::to_string(level)); + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKick ... updating velocity only\n"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + const auto& ba = get_level(lev).get_new_data(State_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(State_Type).DistributionMap(); + MultiFab grav_vec_new(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_new_grav_vector(lev, grav_vec_new, cur_time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) + Nyx::theActiveParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + + // Virtual particles will be recreated, so we need not kick them. + + // Ghost particles need to be kicked except during the final iteration. + if (iteration != ncycle) + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + } + } + } // do_grav + + // + // Synchronize Energies + // + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::RegionTag amrReset_tag("Reset_" + std::to_string(lev)); + MultiFab& S_new = get_level(lev).get_new_data(State_Type); + MultiFab& D_new = get_level(lev).get_new_data(DiagEOS_Type); + + get_level(lev).reset_internal_energy_nostore(S_new,D_new); + + } + } + } + // BL_PROFILE_REGION_STOP("R::Nyx::advance_hydro_plus_particles"); + + // Redistribution happens in post_timestep + return dt; +} +#endif + +Real +Nyx::advance_heatcool (Real time, + Real dt, + int iteration, + int ncycle) +{ + BL_PROFILE("Nyx::advance_heatcool()"); +#ifdef HEATCOOL + amrex::Print()<<"Using advance_heatcool since hydro and dm_particles are both off"<finestLevel(); + if (do_reflux && level < finest_level) + gravity->zero_phi_flux_reg(level + 1); + gravity->swap_time_levels(level); + } + + if (do_forcing) + forcing->evolve(dt); + + // Call the hydro advance itself + BL_PROFILE_VAR("just_the_hydro", just_the_hydro); +#ifdef SDC + if (sdc_split > 0) + { + sdc_hydro(time, dt, a_old, a_new); + } else { + strang_hydro(time, dt, a_old, a_new); + } +#else + strang_hydro(time, dt, a_old, a_new); +#endif + BL_PROFILE_VAR_STOP(just_the_hydro); + + if (do_grav) + { + if (verbose && ParallelDescriptor::IOProcessor()) + std::cout << "\n... new-time level solve at level " << level << '\n'; + + // + // Solve for new phi + // Here we use the "old" phi from the current time step as a guess for this solve + // + MultiFab::Copy(get_new_data(PhiGrav_Type),get_old_data(PhiGrav_Type),0,0,1,0); + int fill_interior = 0; + gravity->solve_for_new_phi(level,get_new_data(PhiGrav_Type), + gravity->get_grad_phi_curr(level),fill_interior); + if (do_reflux) + gravity->add_to_fluxes(level,iteration,ncycle); + + // Now do corrector part of source term update + correct_gsrc(level,time,prev_time,cur_time,dt); + } + + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + MultiFab reset_e_src(S_new.boxArray(), S_new.DistributionMap(), 1, NUM_GROW); + reset_e_src.setVal(0.0); + + // First reset internal energy before call to compute_temp + reset_internal_energy(S_new,D_new,reset_e_src); + compute_new_temp(S_new,D_new); + + return dt; +} +#endif diff --git a/Exec/HaloFinder/Nyx_advance_particles.cpp b/Exec/HaloFinder/Nyx_advance_particles.cpp new file mode 100644 index 000000000..f3095ddc0 --- /dev/null +++ b/Exec/HaloFinder/Nyx_advance_particles.cpp @@ -0,0 +1,282 @@ +#include +#include +#include + +using namespace amrex; + +using std::string; + +Real +Nyx::advance_particles_only (Real time, + Real dt, + int iteration, + int ncycle) + + // Arguments: + // time : the current simulation time + // dt : the timestep to advance (e.g., go from time to + // time + dt) + // iteration : where we are in the current AMR subcycle. Each + // level will take a number of steps to reach the + // final time of the coarser level below it. This + // counter starts at 1 + // ncycle : the number of subcycles at this level + +{ + BL_PROFILE("Nyx::advance_particles_only()"); + + // A particle in cell (i) can affect cell values in (i-1) to (i+1) + int stencil_deposition_width = 1; + + // A particle in cell (i) may need information from cell values in (i-1) to (i+1) + // to update its position (typically via interpolation of the acceleration from the grid) + int stencil_interpolation_width = 1; + + // A particle that starts in cell (i + ncycle) can reach + // cell (i) in ncycle number of steps .. after "iteration" steps + // the particle has to be within (i + ncycle+1-iteration) to reach cell (i) + // in the remaining (ncycle-iteration) steps + + // *** ghost_width *** is used + // *) to set how many cells are used to hold ghost particles i.e copies of particles + // that live on (level-1) can affect the grid over all of the ncycle steps. + // We define ghost cells at the coarser level to cover all iterations so + // we can't reduce this number as iteration increases. + + int ghost_width = ncycle + stencil_deposition_width; + + // *** where_width *** is used + // *) to set how many cells the Where call in moveKickDrift tests = + // ghost_width + (1-iteration) - 1: + // the minus 1 arises because this occurs *after* the move + + int where_width = ghost_width + (1-iteration) - 1; + + // *** grav_n_grow *** is used + // *) to determine how many ghost cells we need to fill in the MultiFab from + // which the particle interpolates its acceleration + // *) to set how many cells the Where call in moveKickDrift tests = (grav.nGrow()-2). + // *) the (1-iteration) arises because the ghost particles are created on the coarser + // level which means in iteration 2 the ghost particles may have moved 1 additional cell along + + int grav_n_grow = ghost_width + (1-iteration) + (iteration-1) + + stencil_interpolation_width ; + + // Sanity checks + if (do_hydro) + amrex::Abort("In `advance_particles_only` but `do_hydro` is true"); + + if (!do_grav) + amrex::Abort("In `advance_particles_only` but `do_grav` not true"); + const int finest_level = parent->finestLevel(); + int finest_level_to_advance; + bool nosub = !parent->subCycle(); + + if (nosub) + { + if (level > 0) + return dt; + + finest_level_to_advance = finest_level; + } + else + { + // This level was advanced by a previous multilevel advance. + if (level > 0 && ncycle == 1) + return dt; + + // Find the finest level to advance + int lev = level; + while(lev < finest_level && parent->nCycle(lev+1) == 1) + lev++; + finest_level_to_advance = lev; + + // We must setup virtual and Ghost Particles + // + // Setup the virtual particles that represent finer level particles + // + setup_virtual_particles(); + // + // Setup ghost particles for use in finer levels. Note that Ghost particles + // that will be used by this level have already been created, the + // particles being set here are only used by finer levels. + // + for(int lev = level; lev <= finest_level_to_advance && lev < finest_level; lev++) + { + get_level(lev).setup_ghost_particles(ghost_width); + } + } + + Real dt_lev; + + // + // Move current data to previous, clear current. + // Don't do this if a coarser level has done this already. + // + if (level == 0 || iteration > 1) + { + for (int lev = level; lev <= finest_level; lev++) + { + dt_lev = parent->dtLevel(lev); + for (int k = 0; k < NUM_STATE_TYPE; k++) + { + get_level(lev).state[k].allocOldData(); + get_level(lev).state[k].swapTimeLevels(dt_lev); + } +#ifndef NO_HYDRO + if(do_hydro) + { + MultiFab& S_old = get_level(lev).get_old_data(State_Type); + MultiFab& S_new = get_level(lev).get_new_data(State_Type); + MultiFab::Copy(S_new, S_old, 0, 0, S_old.nComp(), 0); + } +#endif + } + } + + const Real prev_time = state[PhiGrav_Type].prevTime(); + const Real cur_time = state[PhiGrav_Type].curTime(); + + const Real a_old = get_comoving_a(prev_time); + const Real a_new = get_comoving_a(cur_time); + + // + // We now do a multilevel solve for old Gravity. This goes to the + // finest level regardless of subcycling behavior. Consequentially, + // If we are subcycling we skip this step on the first iteration of + // finer levels. + if (level == 0 || iteration > 1) + { + // fix fluxes on finer grids + if (do_reflux) + { + for (int lev = level; lev < finest_level; lev++) + { + gravity->zero_phi_flux_reg(lev + 1); + } + } + + // swap grav data + for (int lev = level; lev <= finest_level; lev++) + get_level(lev).gravity->swap_time_levels(lev); + + // + // Solve for phi + // If a single-level calculation we can still use the previous phi as a guess. + // TODO: Check this. + int use_previous_phi_as_guess = 1; + gravity->multilevel_solve_for_old_phi(level, finest_level, + use_previous_phi_as_guess); + } + + { + // + // Advance Particles + // + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities to the half-time and the positions to the new time + // We use the cell-centered gravity to correctly interpolate onto particle locations + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKickDrift ... updating particle positions and velocity\n"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + // We need grav_n_grow grow cells to track boundary particles + const auto& ba = get_level(lev).get_new_data(PhiGrav_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(PhiGrav_Type).DistributionMap(); + MultiFab grav_vec_old(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_old_grav_vector(lev, grav_vec_old, time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) { + Real radius_inner = -1.e34; + Real radius_outer = -1.e34; + Real z_old = 1/a_old-1; + if(z_old=lightcone_end_z) { + integrate_distance_given_a(a_old, 1.0, radius_outer); + integrate_distance_given_a(a_new, 1.0, radius_inner); + Print()<moveKickDrift(grav_vec_old, lev, time, dt, a_old, a_half, where_width, radius_inner, radius_outer); + } + + // Only need the coarsest virtual particles here. + if (lev == level && level < finest_level) + for (int i = 0; i < Nyx::theVirtualParticles().size(); i++) + Nyx::theVirtualParticles()[i]->moveKickDrift(grav_vec_old, level, time, dt, a_old, a_half, where_width); + + // Miiiight need all Ghosts + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKickDrift(grav_vec_old, lev, time, dt, a_new, a_half, where_width); + +#ifdef NEUTRINO_DARK_PARTICLES + for (int i = 0; i < Nyx::theActiveParticles().size() && neutrino_cfl >= 1; i++) + Nyx::theActiveParticles()[i]->Redistribute(); +#endif + } + } + + // + // Here we use the "old" phi from the current time step as a guess for this + // solve + // + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::Copy(parent->getLevel(lev).get_new_data(PhiGrav_Type), + parent->getLevel(lev).get_old_data(PhiGrav_Type), + 0, 0, 1, 0); + } + } + + // Solve for new Gravity + int use_previous_phi_as_guess = 1; + if (finest_level_to_advance > level) + { + gravity->multilevel_solve_for_new_phi(level, finest_level_to_advance, + use_previous_phi_as_guess); + } + else + { + int fill_interior = 0; + gravity->solve_for_new_phi(level,get_new_data(PhiGrav_Type), + gravity->get_grad_phi_curr(level), + fill_interior, grav_n_grow); + } + + { + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities by dt/2 to the new time. We use the + // cell-centered gravity to correctly interpolate onto particle + // locations. + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKick ... updating velocity only\n"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + const auto& ba = get_level(lev).get_new_data(PhiGrav_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(PhiGrav_Type).DistributionMap(); + MultiFab grav_vec_new(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_new_grav_vector(lev, grav_vec_new, cur_time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) + Nyx::theActiveParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + + // Virtual particles will be recreated, so we need not kick them. + + // Ghost particles need to be kicked except during the final iteration. + if (iteration != ncycle) + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + } + } + } + + // Redistribution happens in post_timestep + return dt; +} diff --git a/Exec/HaloFinder/Prob.cpp b/Exec/HaloFinder/Prob.cpp new file mode 100644 index 000000000..221f7bf74 --- /dev/null +++ b/Exec/HaloFinder/Prob.cpp @@ -0,0 +1,97 @@ +#include "Nyx.H" +#include "Prob.H" + +void prob_param_special_fill(amrex::GpuArray& prob_param) +{} + +#ifndef NO_HYDRO +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void prob_initdata_state(const int i, + const int j, + const int k, + amrex::Array4 const& state, + amrex::GeometryData const& geomdata, + const amrex::GpuArray& prob_param) +{ + // This is the case where we have compiled with states defined + // but they have only one component each so we fill them this way. + if (state.nComp() == 1) + { + //Could be replaced with setVal + state(i,j,k,0) = 0.00; + } +#ifndef NO_HYDRO + // This is the regular case with NO_HYDRO = FALSE + else if (state.nComp() > 1) + { + state(i,j,k,Density_comp) = 0.00; //1.5d0 * small_dens + state(i,j,k,Xmom_comp) = 0.00; + state(i,j,k,Ymom_comp) = 0.00; + state(i,j,k,Zmom_comp) = 0.00; + + // These will both be set later in the call to init_e. + state(i,j,k,Eint_comp) = 0.0; + state(i,j,k,Eden_comp) = 0.0; + +#ifndef CONST_SPECIES + state(i,j,k,FirstSpec_comp ) = prob_param[ h_species_comp]; + state(i,j,k,FirstSpec_comp+1) = prob_param[he_species_comp]; +#endif + } +#endif +} + +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void prob_initdata(const int i, + const int j, + const int k, + amrex::Array4 const& state, + amrex::Array4 const& diag_eos, + amrex::GeometryData const& geomdata, + const amrex::GpuArray& prob_param) +{ + // This is the case where we have compiled with states defined + // but they have only one component each so we fill them this way. + if (state.nComp() == 1 && diag_eos.nComp() == 1) + { + //Could be replaced with setVal + diag_eos(i,j,k,0) = 0.00; + } +#ifndef NO_HYDRO + // This is the regular case with NO_HYDRO = FALSE + else if (state.nComp() > 1 && diag_eos.nComp() >= 2) + { + diag_eos(i,j,k,Temp_comp) = 0.0210*(1.00 + prob_param[z_in_comp])*(1.00 + prob_param[z_in_comp]); + diag_eos(i,j,k, Ne_comp) = 0.0; + + //Should be equivalent to inhomo_reion > 0 Nyx_setup.cpp + if (diag_eos.nComp() > 2) + diag_eos(i,j,k, Zhi_comp) = 7.5; + } +#endif +} + +void prob_initdata_on_box(const Box& bx, + Array4 const& state, + Array4 const& diag_eos, + GeometryData const& geomdata, + const GpuArray& prob_param) +{ + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + prob_initdata_state(i, j ,k, state, geomdata, prob_param); + prob_initdata (i, j ,k, state, diag_eos, geomdata, prob_param); + }); +} + +void prob_initdata_state_on_box(const Box& bx, + Array4 const& state, + GeometryData const& geomdata, + const GpuArray& prob_param) +{ + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + prob_initdata_state(i, j ,k, state, geomdata, prob_param); + }); +} +#endif diff --git a/Exec/HaloFinder/Prob_error.cpp b/Exec/HaloFinder/Prob_error.cpp new file mode 100644 index 000000000..e0e0503d3 --- /dev/null +++ b/Exec/HaloFinder/Prob_error.cpp @@ -0,0 +1,11 @@ +#include "Nyx.H" +#include "Prob.H" + +void prob_errtags_default(amrex::Vector& errtags) +{ + //Only include default tagging if NO_HYDRO=FALSE +#ifndef NO_HYDRO + AMRErrorTagInfo info; + errtags.push_back(AMRErrorTag(3.5e9,AMRErrorTag::GREATER,"denvol",info)); +#endif +} diff --git a/Exec/HaloFinder/Read_SimpleBinary_Halos.py b/Exec/HaloFinder/Read_SimpleBinary_Halos.py new file mode 100644 index 000000000..d781003b7 --- /dev/null +++ b/Exec/HaloFinder/Read_SimpleBinary_Halos.py @@ -0,0 +1,57 @@ +import numpy as np + +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D + +def read_binary_points(filename): + with open(filename, 'rb') as file: + # Read the binary data directly, assuming big-endian float32 + data = np.fromfile(file, dtype='>f4') # '>f4' indicates big-endian float32 + + # Reshape the data to Nx3 (x, y, z for each point) + if len(data) % 5 != 0: + raise ValueError("Data size is not a multiple of 5. The file might be corrupted or improperly formatted.") + + points = data.reshape((-1, 5)) + + num_points = len(data) // 5 + print(f"Total number of points: {num_points}") + + return points, num_points + +def plot_points(x, y, z): + fig = plt.figure(figsize=(10, 7)) + ax = fig.add_subplot(111, projection='3d') + + # Scatter plot for points + ax.scatter(x, y, z, c='blue', marker='o', s=1, alpha=0.8) + + # Set labels for the axes + ax.set_xlabel('X Coordinate') + ax.set_ylabel('Y Coordinate') + ax.set_zlabel('Z Coordinate') + + ax.set_title('Lightcone shell') + plt.show() + + +# Example usage +filename = "./Output/Halos/SimpleBinary/reeber_halos_0000177.bin" +points, num_points = read_binary_points(filename) + +# Display the first few points +x = np.zeros(num_points); +y = np.zeros(num_points); +z = np.zeros(num_points); +xcen = 3850.0; +ycen = 3850.0; +zcen = 3850.0; +for i, point in enumerate(points[:num_points]): # Adjust the slice as needed + x[i], y[i], z[i], mass, n_cells = point + #rad = np.sqrt((x[i] - xcen) * (x[i] - xcen) + (y[i] - ycen) * (y[i] - ycen) + (z[i] - zcen) * (z[i] - zcen)); + print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {mass:.15g}, {n_cells:.15g}") + #print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {vx:.15g}, {vy:.15g}, {vz:.15g}") + +#plot_points(x, y, z) + + diff --git a/Exec/HaloFinder/Read_SimpleBinary_Lightcones.py b/Exec/HaloFinder/Read_SimpleBinary_Lightcones.py new file mode 100755 index 000000000..d2109f8b1 --- /dev/null +++ b/Exec/HaloFinder/Read_SimpleBinary_Lightcones.py @@ -0,0 +1,57 @@ +import numpy as np + +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D + +def read_binary_points(filename): + with open(filename, 'rb') as file: + # Read the binary data directly, assuming big-endian float32 + data = np.fromfile(file, dtype='>f4') # '>f4' indicates big-endian float32 + + # Reshape the data to Nx3 (x, y, z for each point) + if len(data) % 6 != 0: + raise ValueError("Data size is not a multiple of 6. The file might be corrupted or improperly formatted.") + + points = data.reshape((-1, 6)) + + num_points = len(data) // 6 + print(f"Total number of points: {num_points}") + + return points, num_points + +def plot_points(x, y, z): + fig = plt.figure(figsize=(10, 7)) + ax = fig.add_subplot(111, projection='3d') + + # Scatter plot for points + ax.scatter(x, y, z, c='blue', marker='o', s=1, alpha=0.8) + + # Set labels for the axes + ax.set_xlabel('X Coordinate') + ax.set_ylabel('Y Coordinate') + ax.set_zlabel('Z Coordinate') + + ax.set_title('Lightcone shell') + plt.show() + + +# Example usage +filename = "lightcone_0000010.bin" +points, num_points = read_binary_points(filename) + +# Display the first few points +x = np.zeros(num_points); +y = np.zeros(num_points); +z = np.zeros(num_points); +xcen = 3850.0; +ycen = 3850.0; +zcen = 3850.0; +for i, point in enumerate(points[:num_points]): # Adjust the slice as needed + x[i], y[i], z[i], vx, vy, vz = point + rad = np.sqrt((x[i] - xcen) * (x[i] - xcen) + (y[i] - ycen) * (y[i] - ycen) + (z[i] - zcen) * (z[i] - zcen)); + print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {vx:.15g}, {vy:.15g}, {vz:.15g}, {rad:.15g}") + #print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {vx:.15g}, {vy:.15g}, {vz:.15g}") + +plot_points(x, y, z) + + diff --git a/Exec/HaloFinder/TREECOOL_middle b/Exec/HaloFinder/TREECOOL_middle new file mode 100644 index 000000000..c548b2ec3 --- /dev/null +++ b/Exec/HaloFinder/TREECOOL_middle @@ -0,0 +1,301 @@ +0.000000 5.700000e-14 3.100000e-14 1.121650e-16 3.560837e-25 4.486095e-25 5.008400e-27 +0.021189 7.131077e-14 3.942314e-14 1.290508e-16 4.465957e-25 5.631802e-25 5.728569e-27 +0.041393 8.817069e-14 4.881653e-14 1.564290e-16 5.546459e-25 6.943841e-25 6.874023e-27 +0.060698 1.080520e-13 6.036742e-14 1.892055e-16 6.806021e-25 8.499327e-25 8.214962e-27 +0.079181 1.313927e-13 7.381091e-14 2.281519e-16 8.287477e-25 1.030083e-24 9.775263e-27 +0.096910 1.574751e-13 8.920400e-14 2.740300e-16 9.950685e-25 1.237482e-24 1.157747e-26 +0.113943 1.870916e-13 1.066446e-13 3.274889e-16 1.184974e-24 1.471890e-24 1.363982e-26 +0.130334 2.201403e-13 1.260925e-13 3.893120e-16 1.397618e-24 1.732952e-24 1.598488e-26 +0.146128 2.558537e-13 1.472511e-13 4.603516e-16 1.627616e-24 2.017452e-24 1.863681e-26 +0.161368 2.977649e-13 1.718511e-13 5.410824e-16 1.893292e-24 2.347358e-24 2.160698e-26 +0.176091 3.428995e-13 1.987396e-13 6.320781e-16 2.183324e-24 2.708677e-24 2.490966e-26 +0.190332 3.912293e-13 2.276695e-13 7.337759e-16 2.493622e-24 3.095318e-24 2.855453e-26 +0.204120 4.463107e-13 2.603846e-13 8.467451e-16 2.838601e-24 3.523196e-24 3.255534e-26 +0.217484 5.046292e-13 2.943905e-13 9.710171e-16 3.208211e-24 3.984673e-24 3.690837e-26 +0.230449 5.642777e-13 3.291591e-13 1.106241e-15 3.588006e-24 4.460281e-24 4.159783e-26 +0.243038 6.309768e-13 3.700881e-13 1.252896e-15 4.013032e-24 4.994599e-24 4.663438e-26 +0.255273 7.003212e-13 4.118066e-13 1.409787e-15 4.451985e-24 5.544954e-24 5.197583e-26 +0.267172 7.734476e-13 4.548221e-13 1.577197e-15 4.911441e-24 6.119276e-24 5.762626e-26 +0.278754 8.509889e-13 5.012864e-13 1.753469e-15 5.402579e-24 6.737064e-24 6.352996e-26 +0.290035 9.292096e-13 5.484886e-13 1.937662e-15 5.899552e-24 7.363681e-24 6.965391e-26 +0.301030 1.014903e-12 6.006439e-13 2.128940e-15 6.455638e-24 8.031408e-24 7.596822e-26 +0.311754 1.100000e-12 6.524997e-13 2.325220e-15 7.009520e-24 8.691806e-24 8.240606e-26 +0.322219 1.122007e-12 6.649298e-13 2.524680e-15 7.152800e-24 8.855698e-24 8.890910e-26 +0.332438 1.137809e-12 6.737611e-13 2.725227e-15 7.256136e-24 8.971874e-24 9.541154e-26 +0.342423 1.145552e-12 6.779476e-13 2.924501e-15 7.292276e-24 9.018775e-24 1.018411e-25 +0.352183 1.154360e-12 6.828133e-13 3.119812e-15 7.336837e-24 9.075818e-24 1.081175e-25 +0.361728 1.162594e-12 6.880686e-13 3.308180e-15 7.389919e-24 9.129358e-24 1.141534e-25 +0.371068 1.173550e-12 6.950557e-13 3.486392e-15 7.462784e-24 9.205408e-24 1.198566e-25 +0.380211 1.187979e-12 7.039418e-13 3.651000e-15 7.551576e-24 9.306058e-24 1.251293e-25 +0.389166 1.198634e-12 7.104930e-13 3.797474e-15 7.612901e-24 9.375911e-24 1.298449e-25 +0.397940 1.207275e-12 7.158272e-13 3.924431e-15 7.664101e-24 9.427544e-24 1.339643e-25 +0.406540 1.213105e-12 7.194667e-13 4.033722e-15 7.705948e-24 9.444720e-24 1.375349e-25 +0.414973 1.216342e-12 7.215532e-13 4.127912e-15 7.730928e-24 9.444007e-24 1.406245e-25 +0.423246 1.219054e-12 7.229159e-13 4.207614e-15 7.750458e-24 9.437258e-24 1.432369e-25 +0.431364 1.219149e-12 7.226487e-13 4.274178e-15 7.752770e-24 9.411277e-24 1.454154e-25 +0.439333 1.221622e-12 7.238018e-13 4.326827e-15 7.768263e-24 9.401333e-24 1.471344e-25 +0.447158 1.218366e-12 7.215375e-13 4.364575e-15 7.740295e-24 9.333136e-24 1.483636e-25 +0.454845 1.222231e-12 7.235103e-13 4.388926e-15 7.757998e-24 9.322051e-24 1.491523e-25 +0.462398 1.220561e-12 7.219019e-13 4.397603e-15 7.745418e-24 9.272144e-24 1.494253e-25 +0.469822 1.209901e-12 7.148017e-13 4.393478e-15 7.678796e-24 9.156752e-24 1.492791e-25 +0.477121 1.199432e-12 7.078630e-13 4.374113e-15 7.613336e-24 9.044882e-24 1.486322e-25 +0.484300 1.193909e-12 7.040435e-13 4.342413e-15 7.575579e-24 8.958947e-24 1.475807e-25 +0.491362 1.190000e-12 7.012368e-13 4.297156e-15 7.547275e-24 8.883588e-24 1.460851e-25 +0.498311 1.178629e-12 6.940542e-13 4.239794e-15 7.471795e-24 8.754437e-24 1.441924e-25 +0.505150 1.160855e-12 6.828258e-13 4.171646e-15 7.359487e-24 8.569961e-24 1.419469e-25 +0.511883 1.150522e-12 6.759325e-13 4.091439e-15 7.295230e-24 8.440136e-24 1.393048e-25 +0.518514 1.136042e-12 6.666341e-13 4.002124e-15 7.204632e-24 8.281918e-24 1.363635e-25 +0.525045 1.123528e-12 6.584897e-13 3.904306e-15 7.121947e-24 8.137490e-24 1.331418e-25 +0.531479 1.115864e-12 6.531880e-13 3.797318e-15 7.067108e-24 8.027796e-24 1.296183e-25 +0.537819 1.095126e-12 6.402568e-13 3.683704e-15 6.929647e-24 7.825678e-24 1.258756e-25 +0.544068 1.072765e-12 6.264631e-13 3.564253e-15 6.785092e-24 7.612582e-24 1.219386e-25 +0.550228 1.061742e-12 6.193867e-13 3.439655e-15 6.716654e-24 7.478913e-24 1.178297e-25 +0.556303 1.046184e-12 6.096776e-13 3.310320e-15 6.619508e-24 7.314394e-24 1.135611e-25 +0.562293 1.036077e-12 6.031568e-13 3.177940e-15 6.556823e-24 7.189026e-24 1.091889e-25 +0.568202 1.022056e-12 5.945533e-13 3.043303e-15 6.470033e-24 7.035863e-24 1.047366e-25 +0.574031 1.005956e-12 5.847470e-13 2.907188e-15 6.370046e-24 6.869274e-24 1.002312e-25 +0.579784 9.912223e-13 5.757424e-13 2.770379e-15 6.278681e-24 6.712955e-24 9.569790e-26 +0.585461 9.680384e-13 5.618682e-13 2.633502e-15 6.132892e-24 6.504294e-24 9.115552e-26 +0.591065 9.544885e-13 5.536352e-13 2.497385e-15 6.046844e-24 6.366674e-24 8.663305e-26 +0.596597 9.306970e-13 5.394692e-13 2.362745e-15 5.895920e-24 6.161646e-24 8.215325e-26 +0.602060 9.143101e-13 5.296034e-13 2.230116e-15 5.791908e-24 6.006705e-24 7.773418e-26 +0.607455 8.991008e-13 5.202923e-13 6.307590e-16 5.694157e-24 5.863423e-24 3.786485e-26 +0.612784 8.823489e-13 5.100061e-13 3.181031e-16 5.585858e-24 5.712746e-24 2.016526e-26 +0.618048 8.751899e-13 5.052654e-13 2.134829e-16 5.538291e-24 5.624235e-24 1.429888e-26 +0.623249 8.637173e-13 4.980317e-13 1.608417e-16 5.463416e-24 5.507808e-24 1.139137e-26 +0.628389 8.528019e-13 4.915248e-13 1.289660e-16 5.396334e-24 5.407943e-24 1.052143e-26 +0.633468 8.428606e-13 4.856804e-13 1.037051e-16 5.336487e-24 5.317448e-24 9.806648e-27 +0.638489 8.382619e-13 4.829132e-13 8.621439e-17 5.310497e-24 5.260379e-24 8.760179e-27 +0.643453 8.336445e-13 4.801335e-13 7.325023e-17 5.284443e-24 5.202720e-24 8.019994e-27 +0.648360 8.267178e-13 4.759967e-13 6.313902e-17 5.244351e-24 5.141382e-24 7.475048e-27 +0.653213 8.196424e-13 4.717560e-13 5.492792e-17 5.203728e-24 5.086392e-24 6.126590e-27 +0.658011 8.135775e-13 4.680948e-13 5.019306e-17 5.169572e-24 5.037542e-24 5.049079e-27 +0.662758 8.065499e-13 4.638777e-13 4.592255e-17 5.129351e-24 4.982595e-24 4.934280e-27 +0.667453 8.027916e-13 4.615450e-13 4.201102e-17 5.109119e-24 4.948612e-24 4.841405e-27 +0.672098 7.961321e-13 4.575691e-13 3.837703e-17 5.066971e-24 4.900394e-24 4.766620e-27 +0.676694 7.886928e-13 4.531439e-13 3.495599e-17 5.019863e-24 4.847315e-24 3.977646e-27 +0.681241 7.815721e-13 4.489005e-13 3.301013e-17 4.974785e-24 4.796134e-24 3.250959e-27 +0.685742 7.747680e-13 4.448374e-13 3.108054e-17 4.931723e-24 4.746826e-24 3.256814e-27 +0.690196 7.690636e-13 4.415922e-13 2.915607e-17 4.898378e-24 4.709738e-24 3.265612e-27 +0.694605 7.642091e-13 4.389615e-13 2.722633e-17 4.872297e-24 4.681562e-24 3.277108e-27 +0.698970 7.569034e-13 4.349242e-13 2.528152e-17 4.830632e-24 4.638383e-24 2.779312e-27 +0.703291 7.496390e-13 4.309116e-13 2.415028e-17 4.789259e-24 4.595466e-24 2.292495e-27 +0.707570 7.426188e-13 4.270404e-13 2.297740e-17 4.749480e-24 4.554057e-24 2.318147e-27 +0.711807 7.358775e-13 4.234541e-13 2.175971e-17 4.710133e-24 4.516890e-24 2.344472e-27 +0.716003 7.254846e-13 4.177967e-13 2.049395e-17 4.647062e-24 4.457929e-24 2.371451e-27 +0.720159 7.152903e-13 4.122517e-13 1.917677e-17 4.585242e-24 4.400158e-24 2.060401e-27 +0.724276 7.051901e-13 4.067590e-13 1.834401e-17 4.524005e-24 4.342938e-24 1.747728e-27 +0.728354 6.953706e-13 4.014265e-13 1.746429e-17 4.464550e-24 4.287421e-24 1.774039e-27 +0.732394 6.824911e-13 3.941022e-13 1.653568e-17 4.386103e-24 4.210623e-24 1.800666e-27 +0.736397 6.764605e-13 3.906762e-13 1.555619e-17 4.351803e-24 4.175464e-24 1.827609e-27 +0.740363 6.671800e-13 3.853730e-13 1.452373e-17 4.296577e-24 4.120238e-24 1.582864e-27 +0.744293 6.580179e-13 3.801378e-13 1.387060e-17 4.242072e-24 4.065723e-24 1.333318e-27 +0.748188 6.544022e-13 3.781067e-13 1.317574e-17 4.223321e-24 4.045477e-24 1.355995e-27 +0.752048 6.481107e-13 3.742073e-13 1.243778e-17 4.185266e-24 4.006934e-24 1.378892e-27 +0.755875 6.441856e-13 3.714595e-13 1.165528e-17 4.161120e-24 3.981845e-24 1.402009e-27 +0.759668 6.390398e-13 3.680078e-13 1.082679e-17 4.129089e-24 3.949210e-24 1.255493e-27 +0.763428 6.353209e-13 3.653779e-13 1.022687e-17 4.106278e-24 3.925395e-24 1.104912e-27 +0.767156 6.297119e-13 3.616611e-13 9.589602e-18 4.071249e-24 3.889899e-24 1.124369e-27 +0.770852 6.265970e-13 3.595083e-13 8.913869e-18 4.052667e-24 3.870932e-24 1.144009e-27 +0.774517 6.224705e-13 3.573013e-13 8.198515e-18 4.028825e-24 3.850199e-24 1.163833e-27 +0.778151 6.193681e-13 3.556826e-13 7.442358e-18 4.011619e-24 3.835816e-24 1.005099e-27 +0.781755 6.051062e-13 3.476532e-13 6.941549e-18 3.922097e-24 3.752257e-24 8.413854e-28 +0.785330 5.877877e-13 3.378617e-13 6.410901e-18 3.812656e-24 3.649568e-24 8.565593e-28 +0.788875 5.676714e-13 3.264544e-13 5.849590e-18 3.684930e-24 3.529277e-24 8.718795e-28 +0.792392 5.495041e-13 3.167119e-13 5.256778e-18 3.571577e-24 3.427850e-24 8.873461e-28 +0.795880 5.363643e-13 3.106727e-13 4.631610e-18 3.493560e-24 3.367904e-24 7.118943e-28 +0.799341 5.169618e-13 3.009645e-13 4.299132e-18 3.374552e-24 3.268043e-24 5.304775e-28 +0.802774 4.995822e-13 2.923781e-13 3.947483e-18 3.268479e-24 3.180164e-24 5.402246e-28 +0.806180 4.877927e-13 2.870282e-13 3.576173e-18 3.198812e-24 3.127367e-24 5.500706e-28 +0.809560 4.759080e-13 2.816036e-13 3.184706e-18 3.128429e-24 3.073691e-24 5.600157e-28 +0.812913 4.623742e-13 2.752620e-13 2.772580e-18 3.047217e-24 3.011775e-24 4.368394e-28 +0.816241 4.520541e-13 2.711678e-13 2.571902e-18 2.987705e-24 2.981882e-24 3.092253e-28 +0.819544 4.443370e-13 2.686641e-13 2.360065e-18 2.945518e-24 2.969691e-24 3.148907e-28 +0.822822 4.345795e-13 2.649609e-13 2.136800e-18 2.889947e-24 2.944484e-24 3.206155e-28 +0.826075 4.272450e-13 2.627744e-13 1.901832e-18 2.850653e-24 2.936413e-24 3.263997e-28 +0.829304 4.200690e-13 2.607429e-13 1.654885e-18 2.812654e-24 2.930479e-24 2.576582e-28 +0.832509 4.130429e-13 2.588692e-13 1.528682e-18 2.775929e-24 2.926771e-24 1.863881e-28 +0.835691 4.062667e-13 2.568159e-13 1.395797e-18 2.739122e-24 2.916111e-24 1.897383e-28 +0.838849 3.995945e-13 2.547881e-13 1.256074e-18 2.702749e-24 2.904734e-24 1.931235e-28 +0.841985 3.930652e-13 2.529187e-13 1.109356e-18 2.667606e-24 2.895517e-24 1.965440e-28 +0.845098 3.847176e-13 2.499429e-13 9.554854e-19 2.620375e-24 2.873967e-24 1.518516e-28 +0.848189 3.667928e-13 2.407400e-13 8.818350e-19 2.507890e-24 2.780786e-24 1.055345e-28 +0.851258 3.528143e-13 2.340834e-13 8.044452e-19 2.422216e-24 2.716796e-24 1.073845e-28 +0.854306 3.393126e-13 2.275320e-13 7.232325e-19 2.338935e-24 2.654234e-24 1.092536e-28 +0.857332 3.274144e-13 2.212584e-13 6.381125e-19 2.263375e-24 2.596005e-24 1.111419e-28 +0.860338 3.155981e-13 2.150280e-13 5.490001e-19 2.188337e-24 2.538177e-24 8.680585e-29 +0.863323 3.038634e-13 2.088406e-13 5.043857e-19 2.113816e-24 2.480749e-24 6.160051e-29 +0.866287 2.922076e-13 2.026948e-13 4.576116e-19 2.039797e-24 2.423706e-24 6.264938e-29 +0.869232 2.806318e-13 1.965913e-13 4.086313e-19 1.966287e-24 2.367056e-24 6.370885e-29 +0.872156 2.691342e-13 1.905289e-13 3.573978e-19 1.893271e-24 2.310788e-24 6.477896e-29 +0.875061 2.581002e-13 1.846475e-13 3.038639e-19 1.822827e-24 2.254911e-24 4.829715e-29 +0.877947 2.486829e-13 1.793655e-13 2.810482e-19 1.761161e-24 2.199478e-24 3.124579e-29 +0.880814 2.393272e-13 1.741182e-13 2.571572e-19 1.699900e-24 2.144408e-24 3.176211e-29 +0.883661 2.300330e-13 1.689053e-13 2.321684e-19 1.639041e-24 2.089699e-24 3.228351e-29 +0.886491 2.207996e-13 1.637267e-13 2.060592e-19 1.578580e-24 2.035351e-24 3.281002e-29 +0.889302 2.116261e-13 1.585815e-13 1.788069e-19 1.518513e-24 1.981352e-24 2.619701e-29 +0.892095 2.025115e-13 1.534694e-13 1.640604e-19 1.458830e-24 1.927701e-24 1.935953e-29 +0.894870 1.934552e-13 1.483901e-13 1.486533e-19 1.399530e-24 1.874395e-24 1.966965e-29 +0.897627 1.860132e-13 1.437805e-13 1.325724e-19 1.349538e-24 1.824350e-24 1.998274e-29 +0.900367 1.790063e-13 1.393093e-13 1.158044e-19 1.302091e-24 1.775350e-24 2.029880e-29 +0.903090 1.720436e-13 1.348662e-13 9.833555e-20 1.254941e-24 1.726661e-24 2.061784e-29 +0.905796 1.651244e-13 1.304507e-13 8.015240e-20 1.208085e-24 1.678274e-24 2.093989e-29 +0.908485 1.582476e-13 1.260626e-13 6.124118e-20 1.161519e-24 1.630186e-24 2.126493e-29 +0.911158 1.514133e-13 1.217015e-13 4.158803e-20 1.115240e-24 1.582395e-24 2.159299e-29 +0.913814 1.446210e-13 1.173673e-13 2.117900e-20 1.069246e-24 1.534897e-24 2.192407e-29 +0.916454 1.378694e-13 1.130588e-13 0.000000e+00 1.023525e-24 1.487683e-24 0.000000e+00 +0.919078 1.326935e-13 1.093997e-13 0.000000e+00 9.870744e-25 1.444549e-24 0.000000e+00 +0.921686 1.275485e-13 1.057624e-13 0.000000e+00 9.508408e-25 1.401672e-24 0.000000e+00 +0.924279 1.224342e-13 1.021468e-13 0.000000e+00 9.148230e-25 1.359051e-24 0.000000e+00 +0.926857 1.173502e-13 9.855261e-14 0.000000e+00 8.790185e-25 1.316683e-24 0.000000e+00 +0.929419 1.122961e-13 9.497961e-14 0.000000e+00 8.434253e-25 1.274565e-24 0.000000e+00 +0.931966 1.072719e-13 9.142766e-14 0.000000e+00 8.080418e-25 1.232694e-24 0.000000e+00 +0.934498 1.022771e-13 8.789657e-14 0.000000e+00 7.728660e-25 1.191069e-24 0.000000e+00 +0.937016 9.731088e-14 8.438562e-14 0.000000e+00 7.378909e-25 1.149682e-24 0.000000e+00 +0.939519 9.371704e-14 8.160445e-14 0.000000e+00 7.115605e-25 1.114961e-24 0.000000e+00 +0.942008 9.014364e-14 7.883910e-14 0.000000e+00 6.853799e-25 1.080437e-24 0.000000e+00 +0.944483 8.659053e-14 7.608948e-14 0.000000e+00 6.593480e-25 1.046108e-24 0.000000e+00 +0.946943 8.305762e-14 7.335546e-14 0.000000e+00 6.334640e-25 1.011975e-24 0.000000e+00 +0.949390 7.954452e-14 7.063678e-14 0.000000e+00 6.077250e-25 9.780345e-25 0.000000e+00 +0.951823 7.605111e-14 6.793334e-14 0.000000e+00 5.821304e-25 9.442834e-25 0.000000e+00 +0.954243 7.257733e-14 6.524508e-14 0.000000e+00 5.566797e-25 9.107219e-25 0.000000e+00 +0.956649 6.912279e-14 6.257172e-14 0.000000e+00 5.313699e-25 8.773464e-25 0.000000e+00 +0.959041 6.628733e-14 6.031097e-14 0.000000e+00 5.105714e-25 8.483019e-25 0.000000e+00 +0.961421 6.386646e-14 5.832717e-14 0.000000e+00 4.927940e-25 8.221747e-25 0.000000e+00 +0.963788 6.145870e-14 5.635417e-14 0.000000e+00 4.751132e-25 7.961892e-25 0.000000e+00 +0.966142 5.906407e-14 5.439189e-14 0.000000e+00 4.575287e-25 7.703451e-25 0.000000e+00 +0.968483 5.668231e-14 5.244018e-14 0.000000e+00 4.400386e-25 7.446403e-25 0.000000e+00 +0.970812 3.834510e-14 4.402700e-14 0.000000e+00 2.595247e-25 0.000000e+00 0.000000e+00 +0.973128 4.188557e-15 4.802124e-15 0.000000e+00 6.018460e-26 0.000000e+00 0.000000e+00 +0.975432 1.865695e-15 2.134670e-15 0.000000e+00 3.180310e-26 0.000000e+00 0.000000e+00 +0.977724 1.174019e-15 1.340458e-15 0.000000e+00 2.064471e-26 0.000000e+00 0.000000e+00 +0.980003 8.471102e-16 9.651459e-16 0.000000e+00 1.519086e-26 0.000000e+00 0.000000e+00 +0.982271 6.565596e-16 7.464321e-16 0.000000e+00 1.200737e-26 0.000000e+00 0.000000e+00 +0.984527 5.315749e-16 6.030196e-16 0.000000e+00 9.923108e-27 0.000000e+00 0.000000e+00 +0.986772 4.431324e-16 5.015781e-16 0.000000e+00 8.451805e-27 0.000000e+00 0.000000e+00 +0.989005 3.771366e-16 4.259206e-16 0.000000e+00 7.356993e-27 0.000000e+00 0.000000e+00 +0.991226 3.259163e-16 3.672376e-16 0.000000e+00 6.509960e-27 0.000000e+00 0.000000e+00 +0.993436 2.849390e-16 3.203243e-16 0.000000e+00 5.834653e-27 0.000000e+00 0.000000e+00 +0.995635 2.513551e-16 2.819082e-16 0.000000e+00 5.283248e-27 0.000000e+00 0.000000e+00 +0.997823 2.232830e-16 2.498288e-16 0.000000e+00 4.824163e-27 0.000000e+00 0.000000e+00 +1.000000 1.994308e-16 2.226022e-16 0.000000e+00 4.435703e-27 0.000000e+00 0.000000e+00 +1.002166 1.788819e-16 1.991761e-16 0.000000e+00 4.102474e-27 0.000000e+00 0.000000e+00 +1.004321 1.609682e-16 1.787834e-16 0.000000e+00 3.813247e-27 0.000000e+00 0.000000e+00 +1.006466 1.451916e-16 1.608518e-16 0.000000e+00 3.559640e-27 0.000000e+00 0.000000e+00 +1.008600 1.311728e-16 1.449461e-16 0.000000e+00 3.335270e-27 0.000000e+00 0.000000e+00 +1.010724 1.186187e-16 1.307295e-16 0.000000e+00 3.135191e-27 0.000000e+00 0.000000e+00 +1.012837 1.072987e-16 1.179375e-16 0.000000e+00 2.955509e-27 0.000000e+00 0.000000e+00 +1.014940 9.702979e-17 1.063595e-16 0.000000e+00 2.743582e-27 0.000000e+00 0.000000e+00 +1.017033 8.802750e-17 9.623278e-17 0.000000e+00 2.495225e-27 0.000000e+00 0.000000e+00 +1.019116 8.019337e-17 8.744034e-17 0.000000e+00 2.264206e-27 0.000000e+00 0.000000e+00 +1.021189 7.332056e-17 7.974478e-17 0.000000e+00 2.063345e-27 0.000000e+00 0.000000e+00 +1.023252 6.724845e-17 7.296182e-17 0.000000e+00 1.887445e-27 0.000000e+00 0.000000e+00 +1.025306 6.185044e-17 6.694622e-17 0.000000e+00 1.732417e-27 0.000000e+00 0.000000e+00 +1.027350 5.702533e-17 6.158191e-17 0.000000e+00 1.595002e-27 0.000000e+00 0.000000e+00 +1.029384 5.269128e-17 5.677508e-17 0.000000e+00 1.472571e-27 0.000000e+00 0.000000e+00 +1.031408 4.878134e-17 5.244901e-17 0.000000e+00 1.362982e-27 0.000000e+00 0.000000e+00 +1.033424 4.524024e-17 4.854037e-17 0.000000e+00 1.264471e-27 0.000000e+00 0.000000e+00 +1.035430 4.202191e-17 4.499644e-17 0.000000e+00 1.175576e-27 0.000000e+00 0.000000e+00 +1.037426 3.908766e-17 4.177292e-17 0.000000e+00 1.095070e-27 0.000000e+00 0.000000e+00 +1.039414 3.640472e-17 3.883237e-17 0.000000e+00 1.021924e-27 0.000000e+00 0.000000e+00 +1.041393 2.545887e-17 2.710715e-17 0.000000e+00 9.552614e-28 0.000000e+00 0.000000e+00 +1.043362 2.376379e-17 2.525780e-17 0.000000e+00 8.943373e-28 0.000000e+00 0.000000e+00 +1.045323 2.220278e-17 2.355856e-17 0.000000e+00 8.385110e-28 0.000000e+00 0.000000e+00 +1.047275 2.076240e-17 2.199408e-17 0.000000e+00 7.872305e-28 0.000000e+00 0.000000e+00 +1.049218 1.943092e-17 2.055102e-17 0.000000e+00 7.400175e-28 0.000000e+00 0.000000e+00 +1.051153 1.819806e-17 1.921769e-17 0.000000e+00 6.964559e-28 0.000000e+00 0.000000e+00 +1.053078 1.705478e-17 1.798381e-17 0.000000e+00 6.561819e-28 0.000000e+00 0.000000e+00 +1.054996 1.599309e-17 1.684033e-17 0.000000e+00 6.188765e-28 0.000000e+00 0.000000e+00 +1.056905 1.500588e-17 1.577919e-17 0.000000e+00 5.842588e-28 0.000000e+00 0.000000e+00 +1.058805 1.408684e-17 1.479325e-17 0.000000e+00 5.520808e-28 0.000000e+00 0.000000e+00 +1.060698 1.323032e-17 1.387614e-17 0.000000e+00 5.221225e-28 0.000000e+00 0.000000e+00 +1.062582 1.243127e-17 1.302215e-17 0.000000e+00 4.941885e-28 0.000000e+00 0.000000e+00 +1.064458 1.168513e-17 1.222615e-17 0.000000e+00 4.681045e-28 0.000000e+00 0.000000e+00 +1.066326 1.098779e-17 1.148352e-17 0.000000e+00 4.437148e-28 0.000000e+00 0.000000e+00 +1.068186 1.033555e-17 1.079010e-17 0.000000e+00 4.208798e-28 0.000000e+00 0.000000e+00 +1.070038 9.725022e-18 1.014211e-17 0.000000e+00 3.994742e-28 0.000000e+00 0.000000e+00 +1.071882 9.153161e-18 9.536141e-18 0.000000e+00 3.793849e-28 0.000000e+00 0.000000e+00 +1.073718 8.617175e-18 8.969074e-18 0.000000e+00 3.605101e-28 0.000000e+00 0.000000e+00 +1.075547 8.114517e-18 8.438075e-18 0.000000e+00 3.427577e-28 0.000000e+00 0.000000e+00 +1.077368 7.642860e-18 7.940556e-18 0.000000e+00 3.260441e-28 0.000000e+00 0.000000e+00 +1.079181 7.200066e-18 7.474148e-18 0.000000e+00 3.102934e-28 0.000000e+00 0.000000e+00 +1.080987 6.784177e-18 7.036681e-18 0.000000e+00 2.954367e-28 0.000000e+00 0.000000e+00 +1.082785 6.393389e-18 6.626162e-18 0.000000e+00 2.814110e-28 0.000000e+00 0.000000e+00 +1.084576 6.026038e-18 6.240760e-18 0.000000e+00 2.681590e-28 0.000000e+00 0.000000e+00 +1.086360 5.680592e-18 5.878787e-18 0.000000e+00 2.556281e-28 0.000000e+00 0.000000e+00 +1.088136 5.355632e-18 5.538686e-18 0.000000e+00 2.437701e-28 0.000000e+00 0.000000e+00 +1.089905 5.049847e-18 5.219021e-18 0.000000e+00 2.325410e-28 0.000000e+00 0.000000e+00 +1.091667 4.762020e-18 4.918463e-18 0.000000e+00 2.219002e-28 0.000000e+00 0.000000e+00 +1.093422 4.491023e-18 4.635780e-18 0.000000e+00 2.118103e-28 0.000000e+00 0.000000e+00 +1.095169 4.235808e-18 4.369832e-18 0.000000e+00 2.022369e-28 0.000000e+00 0.000000e+00 +1.096910 3.995401e-18 4.119560e-18 0.000000e+00 1.931483e-28 0.000000e+00 0.000000e+00 +1.098644 3.768894e-18 3.883981e-18 0.000000e+00 1.845150e-28 0.000000e+00 0.000000e+00 +1.100371 3.555441e-18 3.662180e-18 0.000000e+00 1.763099e-28 0.000000e+00 0.000000e+00 +1.102091 3.354254e-18 3.453305e-18 0.000000e+00 1.685079e-28 0.000000e+00 0.000000e+00 +1.103804 3.164596e-18 3.256564e-18 0.000000e+00 1.610855e-28 0.000000e+00 0.000000e+00 +1.105510 2.985780e-18 3.071217e-18 0.000000e+00 1.540212e-28 0.000000e+00 0.000000e+00 +1.107210 2.817162e-18 2.896573e-18 0.000000e+00 1.472945e-28 0.000000e+00 0.000000e+00 +1.108903 2.658141e-18 2.731988e-18 0.000000e+00 1.408869e-28 0.000000e+00 0.000000e+00 +1.110590 2.508153e-18 2.576860e-18 0.000000e+00 1.347807e-28 0.000000e+00 0.000000e+00 +1.112270 2.366670e-18 2.430625e-18 0.000000e+00 1.289597e-28 0.000000e+00 0.000000e+00 +1.113943 2.233198e-18 2.292757e-18 0.000000e+00 1.234084e-28 0.000000e+00 0.000000e+00 +1.115611 2.107271e-18 2.162761e-18 0.000000e+00 1.181128e-28 0.000000e+00 0.000000e+00 +1.117271 1.988454e-18 2.040175e-18 0.000000e+00 1.130593e-28 0.000000e+00 0.000000e+00 +1.118926 1.876339e-18 1.924565e-18 0.000000e+00 1.082354e-28 0.000000e+00 0.000000e+00 +1.120574 1.770540e-18 1.815525e-18 0.000000e+00 1.036294e-28 0.000000e+00 0.000000e+00 +1.122216 1.670697e-18 1.712674e-18 0.000000e+00 9.923029e-29 0.000000e+00 0.000000e+00 +1.123852 1.576469e-18 1.615653e-18 0.000000e+00 9.502763e-29 0.000000e+00 0.000000e+00 +1.125481 1.487538e-18 1.524125e-18 0.000000e+00 9.101169e-29 0.000000e+00 0.000000e+00 +1.127105 1.403603e-18 1.437776e-18 0.000000e+00 8.717329e-29 0.000000e+00 0.000000e+00 +1.128722 1.324380e-18 1.356307e-18 0.000000e+00 8.350379e-29 0.000000e+00 0.000000e+00 +1.130334 1.249604e-18 1.279439e-18 0.000000e+00 7.999501e-29 0.000000e+00 0.000000e+00 +1.131939 1.179024e-18 1.206910e-18 0.000000e+00 7.663928e-29 0.000000e+00 0.000000e+00 +1.133539 1.112403e-18 1.138472e-18 0.000000e+00 7.342931e-29 0.000000e+00 0.000000e+00 +1.135133 1.049518e-18 1.073893e-18 0.000000e+00 7.035825e-29 0.000000e+00 0.000000e+00 +1.136721 0.000000e+00 1.012954e-18 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.138303 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.139879 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.141450 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.143015 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.144574 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.146128 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.147676 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.149219 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.150756 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.152288 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.153815 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.155336 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.156852 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.158362 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.159868 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.161368 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.162863 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.164353 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.165838 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.167317 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.168792 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.170262 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.171726 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.173186 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.174641 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.176091 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.177536 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.178977 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.180413 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.181844 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.183270 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.184691 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.186108 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.187521 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.188928 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.190332 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.191730 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.193125 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.194514 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.195900 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.197281 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.198657 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.200029 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.201397 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.202761 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.204120 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 diff --git a/Exec/HaloFinder/ascent_actions.yaml b/Exec/HaloFinder/ascent_actions.yaml new file mode 100644 index 000000000..8fbab5703 --- /dev/null +++ b/Exec/HaloFinder/ascent_actions.yaml @@ -0,0 +1,35 @@ +--- +- + action: add_pipelines + pipelines: + slice: + f1: + params: + normal: + x: 0.0 + ? "y" + : 0.0 + z: 1.0 + point: + x: 0.0 + ? "y" + : 0.0 + z: 0.0 + topology: topo + type: exaslice +- + action: add_extracts + extracts: + e1: + params: + fields: + - Density + - Xmom + - Ymom + - Zmom + - rho_E + - rho_e + path: the_slice + protocol: blueprint/mesh/hdf5 + pipeline: slice + type: relay diff --git a/Exec/HaloFinder/ascent_actions_nohydro.yaml b/Exec/HaloFinder/ascent_actions_nohydro.yaml new file mode 100644 index 000000000..597c9446b --- /dev/null +++ b/Exec/HaloFinder/ascent_actions_nohydro.yaml @@ -0,0 +1,25 @@ +- + action: "add_scenes" + scenes: + scene1: + image_prefix: "ascent_render_mesh_particle_x_velocity_" + plots: + plt1: + type: "pseudocolor" + field: "particle_x_velocity" + scene2: + image_prefix: "ascent_render_particles_xvel_" + plots: + plt1: + type: "pseudocolor" + field: "particle_xvel" + scene3: + image_prefix: "ascent_render_both_" + plots: + plt1: + type: "pseudocolor" + field: "particle_x_velocity" + plt2: + type: "pseudocolor" + field: "particle_xvel" + diff --git a/Exec/HaloFinder/ascent_actions_slicefile.yaml b/Exec/HaloFinder/ascent_actions_slicefile.yaml new file mode 100644 index 000000000..bbb174af3 --- /dev/null +++ b/Exec/HaloFinder/ascent_actions_slicefile.yaml @@ -0,0 +1,35 @@ +--- +- + action: add_pipelines + pipelines: + slice: + f1: + params: + normal: + x: 0.0 + ? "y" + : 0.0 + z: 1.0 + point: + x: 0.0 + ? "y" + : 0.0 + z: 0.0 + topology: topo + type: exaslice +- + action: add_extracts + extracts: + e1: + params: + fields: + - Density + - Xmom + - Ymom + - Zmom + - rho_E + - rho_e + path: the_slice + pipeline: slice + protocol: blueprint/mesh/hdf5 + type: relay diff --git a/Exec/HaloFinder/comoving.cpp b/Exec/HaloFinder/comoving.cpp new file mode 100644 index 000000000..70d5071e8 --- /dev/null +++ b/Exec/HaloFinder/comoving.cpp @@ -0,0 +1,969 @@ +#include +#include +#include + +using namespace amrex; + +Real Nyx::initial_z = -1.0; +Real Nyx::final_a = -1.0; +Real Nyx::final_z = -1.0; +Real Nyx::relative_max_change_a = 0.01; +Real Nyx::absolute_max_change_a = -1.0; +Real Nyx::dt_binpow = -1.0; +Real Nyx::initial_time = -1.0; +Real Nyx::final_time = -1.0; + +void +Nyx::read_comoving_params () +{ + ParmParse pp("nyx"); + + pp.get("comoving_OmB" , comoving_OmB); + pp.get("comoving_OmM" , comoving_OmM); + pp.get("comoving_h" , comoving_h); + pp.query("comoving_OmR" , comoving_OmR); + + pp.query("initial_z", initial_z); + pp.query("final_a", final_a); + pp.query("final_z", final_z); + + if (final_z >= 0) + { + if (final_a > 0) + { + std::cerr << "ERROR::dont specify both final_a and final_z\n"; + amrex::Error(); + } + else + { + final_a = 1 / (1 + final_z); + } + } + + pp.query("relative_max_change_a", relative_max_change_a); + pp.query("absolute_max_change_a", absolute_max_change_a); + pp.query("dt_binpow", dt_binpow); + + + // for shrinking box tests, initial_z < 0 is ok + if (initial_z < 0) + { + std::cerr << "ERROR::Need to specify non-negative initial redshift \n"; + amrex::Error(); + } + + if (comoving_h > 0) + { + // save start/end times, for reporting purposes + Real a0 = 0.0, a1 = 1.0/(1.0+initial_z); + integrate_time_given_a(a0, a1, initial_time); + + if (final_z >= 0) { + a1 = 1.0/(1.0+final_z); + } else { + a1 = final_a; + } + integrate_time_given_a(a0, a1, final_time); + } else { + // These are just defaults so the values are defined + initial_time = 0.0; + final_time = 0.0; + } +} + +void +Nyx::integrate_comoving_a_to_a(const Real old_a_local, const Real a_value, Real& dt) +{ + Real H_0, OmL; + Real Delta_t; + Real start_a, end_a, start_slope, end_slope; + int j, nsteps; + + if (comoving_h == 0.0) + amrex::Abort("fort_integrate_comoving_a_to_z: Shouldn't be setting plot_z_values if not evolving a"); + + H_0 = comoving_h * Hubble_const; + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + // Use lots of steps if we want to nail the z_value + // nsteps = 1024 + + // Use enough steps if we want to be close to the a_value + nsteps = 2048; + + // We integrate a, but stop when a = a_value (or close enough) + Delta_t = dt/nsteps; + end_a = old_a_local; + for(j = 1; j <= nsteps; j++) + { + // This uses RK2 to integrate the ODE: + // da / dt = H_0 * sqrt(OmM/a + OmR/a^2 + OmL*a^2) + start_a = end_a; + + // Compute the slope at the old time + if (comoving_type > 0) + start_slope = H_0*std::sqrt(comoving_OmM/start_a + comoving_OmR/(start_a*start_a) + OmL*start_a*start_a); + else + start_slope = comoving_h; + + // Compute a provisional value of ln(a) at the new time + end_a = start_a + start_slope * Delta_t; + + // Compute the slope at the new time + if (comoving_type > 0) + end_slope = H_0*std::sqrt(comoving_OmM/end_a + comoving_OmR/(end_a*end_a) + OmL*end_a*end_a); + else + end_slope = comoving_h; + + // Now recompute a at the new time using the average of the two slopes + end_a = start_a + 0.5e0 * (start_slope + end_slope) * Delta_t; + + // We have crossed from a too small to a too big in this step + if ( (end_a - a_value) * (start_a - a_value) < 0) + { + dt = ( ( end_a - a_value) * double(j ) + + (a_value - start_a) * double(j+1) ) / (end_a - start_a) * Delta_t; + break; + } + + } +} + +void +Nyx::integrate_comoving_a_to_z(const Real old_a_local, const Real z_value, Real& dt) +{ + Real H_0, OmL; + Real Delta_t; + Real start_a, end_a, start_slope, end_slope; + Real a_value; + int j, nsteps; + + if (comoving_h == 0.0) + amrex::Abort("fort_integrate_comoving_a_to_z: Shouldn't be setting plot_z_values if not evolving a"); + + H_0 = comoving_h * Hubble_const; + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + // Translate the target "z" into a target "a" + a_value = 1.e0 / (1.e0 + z_value); + + // Use lots of steps if we want to nail the z_value + nsteps = 1024; + + // We integrate a, but stop when a = a_value (or close enough) + Delta_t = dt/nsteps; + end_a = old_a_local; + for(j = 1; j <= nsteps; j++) + { + // This uses RK2 to integrate the ODE: + // da / dt = H_0 * sqrt(OmM/a + OmR/a^2 + OmL*a^2) + start_a = end_a; + + // Compute the slope at the old time + if (comoving_type > 0) + start_slope = H_0*std::sqrt(comoving_OmM/start_a + comoving_OmR/(start_a*start_a) + OmL*start_a*start_a); + else + start_slope = comoving_h; + + // Compute a provisional value of ln(a) at the new time + end_a = start_a + start_slope * Delta_t; + + // Compute the slope at the new time + if (comoving_type > 0) + end_slope = H_0*std::sqrt(comoving_OmM/end_a + comoving_OmR/(end_a*end_a) + OmL*end_a*end_a); + else + end_slope = comoving_h; + + // Now recompute a at the new time using the average of the two slopes + end_a = start_a + 0.5e0 * (start_slope + end_slope) * Delta_t; + + // We have crossed from a too small to a too big in this step + if ( (end_a - a_value) * (start_a - a_value) < 0) + { + dt = ( ( end_a - a_value) * double(j ) + + (a_value - start_a) * double(j+1) ) / (end_a - start_a) * Delta_t; + break; + } + + } +} + +void +Nyx::est_maxdt_comoving_a(const Real old_a_local, Real & dt) +{ + Real H_0, OmL; + Real max_dt; + + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + // This subroutine computes dt based on not changing a by more than 5% + // if we use forward Euler integration + // d(ln(a)) / dt = H_0 * sqrt(OmM/a^3 + OmL) + + H_0 = comoving_h * Hubble_const; + + if (H_0 != 0.0) + { + if (comoving_type > 0) + max_dt = (0.05) / H_0 / std::sqrt( comoving_OmM/(old_a_local * old_a_local *old_a_local) + +comoving_OmR/(old_a_local * old_a_local *old_a_local * old_a_local) + OmL); + else + max_dt = (0.05) / std::abs(comoving_h); + dt = amrex::min(dt,max_dt); + } +} + +// This might only work for t=0=> a=.00625, although constant canceled +void +Nyx::est_lindt_comoving_a(const Real old_a_local, const Real new_a_local, Real& dt) +{ + Real H_0; + Real lin_dt; + + // This subroutine computes dt based on not changing a by more than 5% + // if we use forward Euler integration + // OmL = 1.e0 - comoving_OmM - comoving_OmR; + // d(ln(a)) / dt = H_0 * sqrt(OmM/a^3 + OmR/a^4 + OmL) + + H_0 = comoving_h * Hubble_const; + + // Could definately be optimized better + if (H_0 != 0.0) + { + lin_dt= ( pow((new_a_local/(pow(.75,(2_rt/3_rt)))),(.75)) + -pow((old_a_local/(pow(.75,(2_rt/3_rt)))),(.75)) ) / H_0; + dt=lin_dt; + } +} + +void +Nyx::estdt_comoving_a(const Real old_a_local, Real& new_a_local, + Real& dt, const Real change_allowed, + const Real fixed_da, const Real final_a_in, int& dt_modified) +{ + if (comoving_h != 0.0e0) + { + if( fixed_da <= 0.0e0) + { + // First call this to make sure dt that we send to integration routine isnt outrageous + est_maxdt_comoving_a(old_a_local,dt); + + // Initial call to see if existing dt will work + integrate_comoving_a(old_a_local,new_a_local,dt); + + // Make sure a isn't growing too fast + enforce_percent_change(old_a_local,new_a_local,dt,change_allowed); + } + else + { + // First call this to make sure dt that we send to integration routine isnt outrageous + new_a_local = (old_a_local + fixed_da); + est_lindt_comoving_a(old_a_local,new_a_local,dt); + est_maxdt_comoving_a(old_a_local,dt); + + // Then integrate old_a_local to a_value using dt as a guess for the maximum dt + // output dt is based on a fraction of the input dt + integrate_comoving_a_to_a(old_a_local,new_a_local,dt); + } + // Make sure we don't go past final_a_in (if final_a_in is set) + if (final_a_in > 0.0e0) + enforce_final_a(old_a_local,new_a_local,dt,final_a_in); + + dt_modified = 1; + } + else + { + // dt is unchanged by this call + + dt_modified = 0; + } +} + +void +Nyx::enforce_percent_change(const Real old_a_local, Real& new_a_local, + Real& dt,const Real change_allowed) +{ + + int i; + Real factor = ( (new_a_local - old_a_local) / old_a_local ) / change_allowed; + + // Only go into this process if percent change exceeds change_allowed + + if(factor > 1.0) + { + for(i = 1; i <= 100; i++) + { + factor = ( (new_a_local - old_a_local) / old_a_local ) / change_allowed; + + // Note: 0.99 is just a fudge factor so we don't get bogged down. + if(factor > 1.0) + { + dt = (1.0 / factor) * dt * 0.99; + integrate_comoving_a(old_a_local,new_a_local,dt); + } + else if (i < 100) + { + integrate_comoving_a(old_a_local,new_a_local,dt); + // We're done + return; + } + else + amrex::Abort("Too many iterations in enforce_percent_change"); + + } + } + else + return; +} + +void +Nyx::enforce_final_a(const Real old_a_local, Real& new_a_local, + Real& dt, const Real final_a_in) +{ + int i; + Real factor; + const Real eps = 1.e-10; + + if (old_a_local > final_a_in) + amrex::Abort("Oops -- old_a > final_a_in"); + + // Only go into this process if new_a is past final_a_in + if (new_a_local > final_a_in) + { + for(i = 1; i <= 100; i++) + { + if ( (new_a_local > (final_a_in+eps)) || (new_a_local < final_a_in) ) + { + factor = (final_a_in - old_a_local) / (new_a_local - old_a_local); + dt = dt * factor; + integrate_comoving_a(old_a_local,new_a_local,dt); + } + else if (i<100) + return; + else + amrex::Abort("Too many iterations in enforce_final_a"); + } + } + else + // We don't need to do anything + return; +} + + +void +Nyx::comoving_est_time_step (Real& cur_time, Real& estdt) +{ + Real change_allowed = relative_max_change_a; + Real fixed_da = absolute_max_change_a; + Real dt = estdt; + Real new_dummy_a; + int dt_modified; + + if ( std::abs(cur_time - new_a_time) <= 1.e-12 * cur_time) + { + + // Initial guess -- note that we send in "new_a" because we haven't yet swapped + // "old_a" and "new_a" -- we can't do that until after we compute dt and then + // integrate a forward. + estdt_comoving_a + (new_a, new_dummy_a, dt, change_allowed, fixed_da, final_a, dt_modified); + + if(dt_binpow >= 0) + { + if(estdt>=dt) + estdt=dt; + else if(estdt>.5*dt) + { + estdt=.5*dt; + // std::cout << "Lavel = 1" <.25*dt) + { + estdt=.25*dt; + // std::cout << "Lavel = 2" <.125*dt) + { + estdt=.125*dt; + // std::cout << "Lavel = 3" <.0625*dt) + { + estdt=.0625*dt; + // std::cout << "Lavel = 4" < 4" <= 0) + { + if(estdt>=dt) + estdt=dt; + else if(estdt>.5*dt) + { + estdt=.5*dt; + // std::cout << "Lavel = 1" <.25*dt) + { + estdt=.25*dt; + // std::cout << "Lavel = 2" <.125*dt) + { + estdt=.125*dt; + // std::cout << "Lavel = 3" <.0625*dt) + { + estdt=.0625*dt; + // std::cout << "Lavel = 4" < 4" < old_a_time - eps && time < old_a_time + eps) + { + return old_a; + } + else if (time > new_a_time - eps && time < new_a_time + eps) + { + return new_a; + } + else if (time > old_a_time && time < new_a_time) + { + Real frac = (time - old_a_time) / (new_a_time - old_a_time); + Real a = frac*new_a + (1.0-frac)*old_a; + return a; + } + else + { + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "Invalid:get comoving_a at " << time << std::endl; + std::cout << "Old / new a_time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new a " << old_a << " " << new_a << std::endl; + } + amrex::Error("get_comoving_a: invalid time"); + return 0; + } +} + +void +Nyx::integrate_comoving_a (Real time,Real dt) +{ + if (level > 0) + return; + + bool first; + + if ( std::abs(time-new_a_time) <= (1.e-10 * time) ) + { + first = true; + } else { + first = false; + } + + if (first) + { + + // Update a + old_a = new_a; + integrate_comoving_a(old_a, new_a, dt); + + // Update the times + old_a_time = new_a_time; + new_a_time = old_a_time + dt; + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "Integrating a from time " << time << " by dt = " << dt << '\n'; + std::cout << "Old / new A time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new A " << old_a << " " << new_a << std::endl; + std::cout << "Old / new z " << 1./old_a-1.<< " " << 1./new_a-1. << std::endl; + } + } + else if (std::abs(time-old_a_time) <= 1.e-10 * time) + { + // Leave old_a and old_a_time alone -- we have already swapped them + integrate_comoving_a(old_a, new_a, dt); + + // Update the new time only + new_a_time = old_a_time + dt; + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "Re-integrating a from time " << time << " by dt = " << dt << '\n'; + std::cout << "Old / new A time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new A " << old_a << " " << new_a << std::endl; + std::cout << "Old / new z " << 1./old_a-1.<< " " << 1./new_a-1. << std::endl; + } + } + else + { + std::cout << "Time passed to integrate_comoving_a " << time << std::endl; + std::cout << "Old / new A time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new A " << old_a << " " << new_a << std::endl; + std::cerr << "ERROR::dont know what to do in integrate_comoving_a" << std::endl; + amrex::Error(); + } +} + +Real invEz(Real H0, Real Om, Real Or, Real a) +{ + // invEz = + return 1.0e0 / ( H0*std::sqrt(Om/a + Or/(a*a) + (1.0e0-Om-Or)*a*a) ); +} + +void +Nyx::integrate_time_given_a(const Real a0, const Real a1, Real& dt) +{ + const Real small_dt_fac = 1.0e-6; + Real H0, OmM, OmR, prev_soln, h; + int iter, n, j; + + H0 = comoving_h*Hubble_const; + OmM = comoving_OmM; + OmR = comoving_OmR; + + prev_soln = -1.0e0; + // trapezoidal integration + for(iter = 1; iter<= 20; iter++) + { + n = static_cast(std::round(std::pow(2,iter))); + + h = (a1-a0)/(n-1); + + if (a0 < 1.0e-10) // prevent division by zero in invEz + dt = 0.5*invEz(H0, OmM, OmR, a1); + else + dt = 0.5*(invEz(H0, OmM, OmR, a0) + invEz(H0, OmM, OmR, a1)); + + for(j = 1; j <= n-2; j++) + { + dt = dt + invEz(H0, OmM, OmR, a0+j*h); + } + dt = dt*h; + + if (iter > 4) + if (std::abs(dt-prev_soln) < small_dt_fac*std::abs(prev_soln)) + return; + + prev_soln = dt; + } +} + +void +Nyx::integrate_distance_given_a(const Real a0, const Real a1, Real& dt) +{ + const Real small_dt_fac = 1.0e-6; + Real H0, OmM, OmR, prev_soln, h; + int iter, n, j; + + H0 = comoving_h*Hubble_const; + OmM = comoving_OmM; + OmR = comoving_OmR; + + prev_soln = -1.0e0; + // trapezoidal integration + for(iter = 1; iter<= 20; iter++) + { + n = static_cast(std::round(std::pow(2,iter))); + + h = (a1-a0)/(n-1); + + if (a0 < 1.0e-10) // prevent division by zero in invEz + dt = 0.5*invEz(H0, OmM, OmR, a1)/a1; + else + dt = 0.5*(invEz(H0, OmM, OmR, a0)/a0 + invEz(H0, OmM, OmR, a1)/a1); + + for(j = 1; j <= n-2; j++) + { + dt = dt + invEz(H0, OmM, OmR, a0+j*h)/(a0+j*h); + } + dt = dt*h*c_light; + + if (iter > 4) + if (std::abs(dt-prev_soln) < small_dt_fac*std::abs(prev_soln)) + return; + + prev_soln = dt; + } +} + +void +Nyx::integrate_comoving_a (const Real old_a_local, Real& new_a_local, const Real dt) +{ + + const Real small_a_fac = 1.0e-8; + Real H_0, OmL, Delta_t, prev_soln; + Real start_a, end_a, start_slope, end_slope; + int iter, j, nsteps; + + if (comoving_h == 0.0) + { + new_a_local = old_a_local; + return; + } + + H_0 = comoving_h * Hubble_const; + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + prev_soln = 2.0e0; // 0(std::round(std::pow(2,iter-1))); + + Delta_t = dt/nsteps; + end_a = old_a_local; + + for(j = 1; j <= nsteps; j++) + { + // This uses RK2 to integrate the ODE: + // da / dt = H_0 * sqrt(OmM/a + OmR/a^2 + OmL*a^2) + start_a = end_a; + + // Compute the slope at the old time + if (comoving_type > 0) + start_slope = H_0*std::sqrt(comoving_OmM/start_a + comoving_OmR/(start_a*start_a) + OmL*start_a*start_a); + else + start_slope = comoving_h; + + // Compute a provisional value of ln(a) at the new time + end_a = start_a + start_slope * Delta_t; + + // Compute the slope at the new time + if (comoving_type > 0) + end_slope = H_0*std::sqrt(comoving_OmM/end_a + comoving_OmR/(end_a*end_a) + OmL*end_a*end_a); + else + end_slope = comoving_h; + + // Now recompute a at the new time using the average of the two slopes + end_a = start_a + 0.5e0 * (start_slope + end_slope) * Delta_t; + + } + + new_a_local = end_a; + + if (std::abs(1.0e0-new_a_local/prev_soln) <= small_a_fac) + return; + prev_soln = new_a_local; + + } +} + +void +Nyx::comoving_a_post_restart (const std::string& restart_file) +{ + if (level > 0) + return; + + if (ParallelDescriptor::IOProcessor()) + { + std::string FileName = restart_file + "/comoving_a"; + std::ifstream File; + File.open(FileName.c_str(),std::ios::in); + if (!File.good()) + amrex::FileOpenFailed(FileName); + File >> old_a; + } + ParallelDescriptor::Bcast(&old_a, 1, ParallelDescriptor::IOProcessorNumber()); + + new_a = old_a; + +#ifdef NO_HYDRO + old_a_time = state[PhiGrav_Type].curTime(); +#else + old_a_time = state[State_Type].curTime(); +#endif + new_a_time = old_a_time; + if (ParallelDescriptor::IOProcessor()) + { + std::cout << "...setting old_a_time to " << old_a_time << std::endl; + } + +} + +void +Nyx::plot_z_est_time_step (Real& dt_0, bool& dt_changed) +{ + Real dt = dt_0; + Real a_old, z_old, a_new, z_new; + Real z_value; + + // This is where we are now +#ifdef NO_HYDRO + Real cur_time = state[PhiGrav_Type].curTime(); +#else + Real cur_time = state[State_Type].curTime(); +#endif + a_old = get_comoving_a(cur_time); + z_old = (1. / a_old) - 1.; + + // ***************************************************** + // First test whether we are within dt of a plot_z value + // ***************************************************** + + // This is where we would be if we use the current dt_0 + Real new_time = cur_time + dt_0; + integrate_comoving_a (cur_time,dt_0); + a_new = get_comoving_a(new_time); + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the plot_z_values array + bool found_one = false; + for (int i = 0; i < plot_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - plot_z_values[i]) * (z_old - plot_z_values[i]) < 0 && !found_one) + { + z_value = plot_z_values[i]; + found_one = true; + } + } + + // Now that we know that dt_0 is too big and makes us pass z_value, + // we must figure out what value of dt < dt_0 makes us exactly reach z_value + if (found_one) + { + integrate_comoving_a_to_z(old_a, z_value, dt); + + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << dt << std::endl; + std::cout << " ... in order to write a plotfile at z = " << z_value << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = dt; + dt_changed = true; + } + else + { + + // ***************************************************** + // If not within one dt, now test whether we are within 2*dt of a plot_z value + // ***************************************************** + + // This is where we would be if we advance by twice the current dt_0 + Real two_dt = 2.0*dt_0; + + integrate_comoving_a(a_old, a_new, two_dt); + + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the plot_z_values array + found_one = false; + for (int i = 0; i < plot_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - plot_z_values[i]) * (z_old - plot_z_values[i]) < 0 && !found_one) + { + z_value = plot_z_values[i]; + found_one = true; + } + } + + // Now that we know that 2*dt_0 will make us pass z_value, we set the current dt + // as half the interval to reach that z_value + if (found_one) + { + integrate_comoving_a_to_z(old_a, z_value, two_dt); + + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << 0.5 * two_dt << std::endl; + std::cout << " ... in order to write a plotfile at z = " << z_value + << " two steps from now " << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = 0.5 * two_dt; + dt_changed = true; + } + + } +} + +void +Nyx::analysis_z_est_time_step (Real& dt_0, bool& dt_changed) +{ + Real dt = dt_0; + Real a_old, z_old, a_new, z_new, z_value; + + // This is where we are now +#ifdef NO_HYDRO + Real cur_time = state[PhiGrav_Type].curTime(); +#else + Real cur_time = state[State_Type].curTime(); +#endif + a_old = get_comoving_a(cur_time); + z_old = (1. / a_old) - 1.; + + // ***************************************************** + // First test whether we are within dt of a analysis_z value + // ***************************************************** + + // This is where we would be if we use the current dt_0 + Real new_time = cur_time + dt_0; + integrate_comoving_a (cur_time,dt_0); + a_new = get_comoving_a(new_time); + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the analysis_z_values array + bool found_one = false; + for (int i = 0; i < analysis_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - analysis_z_values[i]) * (z_old - analysis_z_values[i]) < 0 && !found_one) + { + z_value = analysis_z_values[i]; + found_one = true; + } + } + + // Now that we know that dt_0 is too big and makes us pass z_value, + // we must figure out what value of dt < dt_0 makes us exactly reach z_value + if (found_one) + { + integrate_comoving_a_to_z(old_a, z_value, dt); + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << dt << std::endl; + std::cout << " ... in order to do analysis at z = " << z_value << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = dt; + dt_changed = true; + } + else + { + + // ***************************************************** + // If not within one dt, now test whether we are within 2*dt of a analysis_z value + // ***************************************************** + + // This is where we would be if we advance by twice the current dt_0 + Real two_dt = 2.0*dt_0; + + integrate_comoving_a(a_old, a_new, two_dt); + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the analysis_z_values array + found_one = false; + for (int i = 0; i < analysis_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - analysis_z_values[i]) * (z_old - analysis_z_values[i]) < 0 && !found_one) + { + z_value = analysis_z_values[i]; + found_one = true; + } + } + + // Now that we know that 2*dt_0 will make us pass z_value, we set the current dt + // as half the interval to reach that z_value + if (found_one) + { + two_dt = 2.0*dt_0; + integrate_comoving_a_to_z(old_a, z_value, two_dt); + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << 0.5 * two_dt << std::endl; + std::cout << " ... in order to do analysis at z = " << z_value + << " two steps from now " << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = 0.5 * two_dt; + dt_changed = true; + } + + } +} diff --git a/Exec/HaloFinder/inputs_128x128x128 b/Exec/HaloFinder/inputs_128x128x128 new file mode 100644 index 000000000..46b980362 --- /dev/null +++ b/Exec/HaloFinder/inputs_128x128x128 @@ -0,0 +1,219 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +nyx.ppm_type = 1 +nyx.use_colglaz = 0 +nyx.corner_coupling = 1 + +nyx.strang_split = 0 +nyx.sdc_split = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 11 +#nyx.simd_width = 8 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +nyx.h_species=.7546 +nyx.he_species=.2454 + +nyx.small_dens = 1.e-2 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.ml_tol = 1.e-10 +gravity.sl_tol = 1.e-10 +gravity.mlmg_agglomeration=1 +gravity.mlmg_consolidation=1 +nyx.reuse_mlpoisson = 1 + +nyx.initial_z = 159.0 +nyx.final_z = 0.0 + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00100 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 28.49002849 28.49002849 28.49002849 + +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 128 8 8 +fabarray.mfiter_tile_size = 1024000 8 8 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COSMOLOGY +nyx.comoving_OmM = 0.275 +nyx.comoving_OmB = 0.046 +nyx.comoving_h = 0.702e0 + +# UVB and reionization +nyx.inhomo_reion = 0 +nyx.inhomo_zhi_file = "zhi.bin" +nyx.inhomo_grid = 512 +nyx.uvb_rates_file = "TREECOOL_middle" +nyx.uvb_density_A = 1.0 +nyx.uvb_density_B = 0.0 +nyx.reionization_zHI_flash = -1.0 +nyx.reionization_zHeII_flash = -1.0 +nyx.reionization_T_zHI = 2.0e4 +nyx.reionization_T_zHeII = 1.5e4 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryMetaFile +nyx.binary_particle_file = 64sssss_20mpc.nyx +particles.nparts_per_read = 2097152 + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.5 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 2.0 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Nyx.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 1 # verbosity in Amr.cpp +particles.v = 2 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 +#amr.nosub = 1 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = -1 +amr.check_file = chk +amr.check_int = 100 +amr.checkpoint_nfiles = 64 + +# PLOTFILES +amr.plot_files_output = -1 +amr.plot_file = plt +amr.plot_int = -1 + +amr.plot_vars = density xmom ymom zmom rho_e Temp phi_grav +amr.derive_plot_vars = particle_mass_density particle_count + +# Halo Finder +#nyx.analysis_z_values = 150 10 5 4 3 2 +reeber.halo_int = 1 +reeber.negate = 1 +reeber.halo_density_vars = particle_mass_density +reeber.halo_component_threshold = 3.0 +reeber.halo_extrema_threshold = 8.0 +#nyx.mass_halo_min = 1.e6 +#nyx.mass_seed = 1.e6 + +# ANALYSIS in situ +nyx.analysis_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 +insitu.int = 1 +insitu.start = 0 +insitu.reeber_int = 1 + +# SENSEI in situ +sensei.enabled = 0 +#sensei.config = write_vtk.xml +sensei.config = render_iso_catalyst_3d.xml +sensei.frequency = 2 + +#PROBIN FILENAME +amr.probin_file = "" +#TESTING FLAGS +amrex.abort_on_out_of_gpu_memory=1 +amrex.async_out=1 +max_step=500 +nyx.minimize_memory=1 +nyx.hydro_tile_size=64 64 64 +nyx.sundials_tile_size=64 64 64 +nyx.sundials_use_tiling=1 +amr.max_grid_size=128 128 128 +nyx.sundials_alloc_type=5 +amrex.max_gpu_streams=4 +amrex.the_arena_is_managed=1 +nyx.minimize_memory=0 +amrex.abort_on_out_of_gpu_memory=1 +amr.checkpoint_files_output=-1 +amr.plot_files_output=-1 +amrex.async_out=1 +amr.regrid_on_restart=0 +amr.max_grid_size=256 +tiny_profiler.device_synchronize_around_region=0 +amrex.use_profiler_syncs=0 +nyx.sundials_alloc_type=5 +amr.check_int=-1 +nyx.reuse_mlpoisson=1 +amrex.the_arena_release_threshold=35651584000 +amrex.the_pinned_arena_release_threshold=1258291200 +gravity.ml_tol=1e-08 +gravity.sl_tol=1e-08 +nyx.fix_random_seed=1 +amrex.use_gpu_aware_mpi=0 +amrex.async_out=1 +amr.n_cell=128 128 128 +amr.derive_plot_vars=particle_mass_density particle_count particle_x_velocity particle_y_velocity particle_z_velocity +amr.plot_int=-1 +nyx.write_lightcones_and_halos_vtk = 1 +nyx.write_lightcones_and_halos_simplebinary = 1 +#nyx.plot_z_values=200.0 8.0 7.5 7.0 6.6 6.3 6.0 5.8 5.6 5.4 5.2 5.0 4.8 4.6 4.4 4.2 4.0 3.8 3.6 3.4 3.2 3.0 2.8 2.6 2.4 2.2 2.0 1.0 0.5 0.0 +nyx.binary_particle_file=Filelist_128x128x128.txt +geometry.prob_hi=7700 7700 7700 +particles.nreaders=1 +nyx.comoving_h=0.68 +nyx.comoving_OmB=0.0493890 +nyx.comoving_OmM=.31 +particles.nparts_per_read=2097152 +nyx.initial_z=50 +amrex.max_gpu_streams=8 +nyx.sundials_reltol=1e-6 +nyx.sundials_abstol=1e-6 +DistributionMapping.strategy=ROUNDROBIN +nyx.load_balance_start_z=0.0 +nyx.load_balance_int=100 +nyx.load_balance_wgt_strategy=1 +nyx.load_balance_strategy=KNAPSACK +amr.max_grid_size_x=256 +amr.max_grid_size_y=256 +amr.max_grid_size_z=256 diff --git a/Exec/HaloFinder_GPU/32.nyx b/Exec/HaloFinder_GPU/32.nyx new file mode 120000 index 000000000..f35c108d2 --- /dev/null +++ b/Exec/HaloFinder_GPU/32.nyx @@ -0,0 +1 @@ +../LyA/32.nyx \ No newline at end of file diff --git a/Exec/HaloFinder_GPU/64sssss_20mpc.nyx b/Exec/HaloFinder_GPU/64sssss_20mpc.nyx new file mode 120000 index 000000000..3c7b0271b --- /dev/null +++ b/Exec/HaloFinder_GPU/64sssss_20mpc.nyx @@ -0,0 +1 @@ +../LyA/64sssss_20mpc.nyx \ No newline at end of file diff --git a/Exec/HaloFinder_GPU/CMakeLists.txt b/Exec/HaloFinder_GPU/CMakeLists.txt new file mode 100644 index 000000000..73ab32f95 --- /dev/null +++ b/Exec/HaloFinder_GPU/CMakeLists.txt @@ -0,0 +1,11 @@ +set(_sources Prob.cpp Prob_error.cpp) +set(_input_files inputs inputs.summit inputs_gimlet_in_transit.dsc) +list(APPEND _input_files inputs_nohydro.rt inputs.rt inputs.rt.garuda) +list(APPEND _input_files 32.nyx 64sssss_20mpc.nyx TREECOOL_middle) +list(APPEND _input_files ascent_actions_nohydro.yaml ascent_actions.yaml ascent_actions_slicefile.yaml) + +# setup_nyx_executable(_sources _input_files) +nyx_setup_executable(_sources _input_files) + +unset(_sources) +unset(_input_files) diff --git a/Exec/HaloFinder_GPU/DarkMatterParticleContainer.H b/Exec/HaloFinder_GPU/DarkMatterParticleContainer.H new file mode 100644 index 000000000..8de10ebf8 --- /dev/null +++ b/Exec/HaloFinder_GPU/DarkMatterParticleContainer.H @@ -0,0 +1,77 @@ + +#ifndef _DarkMatterParticleContainer_H_ +#define _DarkMatterParticleContainer_H_ + +#include +#include + +class DarkMatterParticleContainer + : public NyxParticleContainer<1+AMREX_SPACEDIM> +{ +public: + DarkMatterParticleContainer (amrex::Amr* amr) + : NyxParticleContainer<1+AMREX_SPACEDIM>(amr) + { + real_comp_names.clear(); + real_comp_names.push_back("mass"); + real_comp_names.push_back("xvel"); + real_comp_names.push_back("yvel"); + real_comp_names.push_back("zvel"); + } + + using MyParIter = amrex::ParIter<1+AMREX_SPACEDIM>; + using MyConstParIter = amrex::ParConstIter<1+AMREX_SPACEDIM>; + + virtual ~DarkMatterParticleContainer () {} + + void InitCosmo1ppcMultiLevel(amrex::MultiFab& mf, const amrex::Real disp_fac[], const amrex::Real vel_fac[], + const amrex::Real particleMass, int disp_idx, int vel_idx, + amrex::BoxArray &baWhereNot, int lev, int nlevs); + + void AssignDensityAndVels (amrex::Vector >& mf, int lev_min = 0) const; + + virtual void moveKickDrift (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_old = 1.0, amrex::Real a_half = 1.0, int where_width = 0, + amrex::Real radius_inner = -1.e34, amrex::Real radius_outer = -1.e34, + amrex::Vector* shell_particles=nullptr); + virtual void moveKick (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_new = 1.0, amrex::Real a_half = 1.0); + + void InitFromBinaryMortonFile(const std::string& particle_directory, int nextra, int skip_factor); + +}; + +AMREX_GPU_HOST_DEVICE AMREX_INLINE void update_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move); + +AMREX_GPU_HOST_DEVICE AMREX_INLINE void update_dm_particle_move_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move); + +AMREX_GPU_HOST_DEVICE AMREX_INLINE void store_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& phi, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move, + const amrex::Real& radius_inner, + const amrex::Real& radius_outer, + int* d_count, const amrex::Box& box, + const int idir, const int jdir, const int kdir, + const bool is_file_write=false, LightConeParticle* d_shell_particles = nullptr); + +#endif /* _DarkMatterParticleContainer_H_ */ + diff --git a/Exec/HaloFinder_GPU/DarkMatterParticleContainer.cpp b/Exec/HaloFinder_GPU/DarkMatterParticleContainer.cpp new file mode 100644 index 000000000..85db0fc82 --- /dev/null +++ b/Exec/HaloFinder_GPU/DarkMatterParticleContainer.cpp @@ -0,0 +1,920 @@ +#include + +#include +#ifdef AMREX_USE_HDF5 +#include +#endif + +using namespace amrex; +#include + +/// These are helper functions used when initializing from a morton-ordered +/// binary particle file. +namespace { + + inline uint64_t split(unsigned int a) { + uint64_t x = a & 0x1fffff; + x = (x | x << 32) & 0x1f00000000ffff; + x = (x | x << 16) & 0x1f0000ff0000ff; + x = (x | x << 8) & 0x100f00f00f00f00f; + x = (x | x << 4) & 0x10c30c30c30c30c3; + x = (x | x << 2) & 0x1249249249249249; + return x; + } + + inline uint64_t get_morton_index(unsigned int x, + unsigned int y, + unsigned int z) { + uint64_t morton_index = 0; + morton_index |= split(x) | ( split(y) << 1) | (split(z) << 2); + return morton_index; + } + + struct BoxMortonKey { + uint64_t morton_id; + int box_id; + }; + + struct by_morton_id { + bool operator()(const BoxMortonKey &a, const BoxMortonKey &b) { + return a.morton_id < b.morton_id; + } + }; + + std::string get_file_name(const std::string& base, int file_num) { + std::stringstream ss; + ss << base << file_num; + return ss.str(); + } + + struct ParticleMortonFileHeader { + long NP; + int DM; + int NX; + int SZ; + int NF; + }; + + void ReadHeader(const std::string& dir, + const std::string& file, + ParticleMortonFileHeader& hdr) { + std::string header_filename = dir; + header_filename += "/"; + header_filename += file; + + Vector fileCharPtr; + ParallelDescriptor::ReadAndBcastFile(header_filename, fileCharPtr); + std::string fileCharPtrString(fileCharPtr.dataPtr()); + std::istringstream HdrFile(fileCharPtrString, std::istringstream::in); + + HdrFile >> hdr.NP; + HdrFile >> hdr.DM; + HdrFile >> hdr.NX; + HdrFile >> hdr.SZ; + HdrFile >> hdr.NF; + } +} + +AMREX_GPU_HOST_DEVICE +bool checkBoxIntersectsShell(const amrex::GpuArray box_min, + const amrex::GpuArray box_max, + const amrex::GpuArray& center, + Real radius_inner, + Real radius_outer, + const amrex::GpuArray& dx_mesh) +{ + + // Compute squared distance from center to closest point on box + Real dist_sq_min = 0.0; + Real dist_sq_max = 0.0; + for (int dim = 0; dim < AMREX_SPACEDIM; ++dim) { + Real c = center[dim]; + + Real dmin = 0.0; + if (c < box_min[dim]) { + dmin = box_min[dim] - c; + } else if (c > box_max[dim]) { + dmin = c - box_max[dim]; + } else { + dmin = 0.0; + } + dist_sq_min += dmin * dmin; + + // Distance to farthest corner in this dimension (max distance) + Real dmax = std::max(std::abs(c - box_min[dim]), std::abs(c - box_max[dim])); + dist_sq_max += dmax * dmax; + } + + Real radius_inner_sq = radius_inner * radius_inner; + Real radius_outer_sq = radius_outer * radius_outer; + + // The shell is between radius_inner and radius_outer + // So if [min_dist, max_dist] overlaps with [radius_inner, radius_outer] then box intersects shell + bool intersects = !(dist_sq_min > radius_outer_sq || dist_sq_max < radius_inner_sq); + + return intersects; +} + +void +DarkMatterParticleContainer::moveKickDrift (amrex::MultiFab& acceleration, + int lev, + amrex::Real t, + amrex::Real dt, + amrex::Real a_old, + amrex::Real a_half, + int where_width, + amrex::Real radius_inner, + amrex::Real radius_outer, + amrex::Vector* shell_particles) +{ + BL_PROFILE("DarkMatterParticleContainer::moveKickDrift()"); + + //If there are no particles at this level + if (lev >= this->GetParticles().size()) + return; + const auto dxi = Geom(lev).InvCellSizeArray(); + const auto dx_mesh = Geom(lev).CellSizeArray(); + + amrex::MultiFab* ac_ptr; + if (this->OnSameGrids(lev, acceleration)) + { + ac_ptr = &acceleration; + } + else + { + const IntVect& ng = acceleration.nGrowVect(); + ac_ptr = new amrex::MultiFab(this->ParticleBoxArray(lev), + this->ParticleDistributionMap(lev), + acceleration.nComp(),acceleration.nGrow()); + ac_ptr->setVal(0.); + if(acceleration.boxArray() == ac_ptr->boxArray())//this->finestLevel() == 0) + { + ac_ptr->Redistribute(acceleration,0,0,acceleration.nComp(),ng); + ac_ptr->FillBoundary(); + } + else + { + ac_ptr->ParallelCopy(acceleration,0,0,acceleration.nComp(),ng,ng); + ac_ptr->FillBoundary(); + } + } + + const GpuArray plo = Geom(lev).ProbLoArray(); + + int do_move = 1; + + auto pc= this; + auto ShellPC = new ParticleContainer<10,0>(pc->Geom(lev), pc->ParticleDistributionMap(lev), pc->ParticleBoxArray(lev)); + ShellPC->resizeData(); + ParticleContainer<10,0>::ParticleInitData pdata = {{}, {}, {}, {}}; + ShellPC->InitNRandomPerCell(1,pdata); + // ShellPC->resize(pc.TotalNumberOfParticles()); + auto geom_test=pc->Geom(lev); + const GpuArray phi=geom_test.ProbHiArray(); + const GpuArray center({AMREX_D_DECL((phi[0]-plo[0])*0.5,(phi[1]-plo[1])*0.5,(phi[2]-plo[2])*0.5)}); + //const Real a_half = 0.5 * (a_old + a_new); + + auto domain=geom_test.Domain(); + auto z=1/a_old-1; + //From write_info.cpp + amrex::Real a_cur = 1.0 / a_half; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + AoS& particles = pti.GetArrayOfStructs(); + ParticleType* pstruct = particles().data(); + const long np = pti.numParticles(); + int grid = pti.index(); + + auto& ptile = ShellPC->DefineAndReturnParticleTile(lev, pti); + int old_np = ptile.size(); + int num_to_add = np; + int new_np = old_np + num_to_add; + ptile.resize(new_np); + + const FArrayBox& accel_fab= ((*ac_ptr)[grid]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + update_dm_particle_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_old, a_half,do_move); + }); + + } + + int h_count = 0; + int* d_count = nullptr; + d_count = (int*)amrex::The_Arena()->alloc(sizeof(int)); + amrex::Gpu::copy(amrex::Gpu::hostToDevice, &h_count, &h_count + 1, d_count); + + Real lenx = phi[0]-plo[0]; + Real leny = phi[1]-plo[1]; + Real lenz = phi[2]-plo[2]; + int maxind[3]; + maxind[0] = floor((radius_outer+lenx*0.5)/lenx); + maxind[1] = floor((radius_outer+leny*0.5)/leny); + maxind[2] = floor((radius_outer+lenz*0.5)/lenz); + + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + auto& particles = (this->ParticlesAt(lev,pti)).GetArrayOfStructs(); + + auto* pstruct = particles().data(); + + const long np = pti.numParticles(); + int grid = pti.index(); + + const FArrayBox& accel_fab= ((*ac_ptr)[0]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + + const amrex::Box& box = pti.validbox(); + + for(int idir=-maxind[0];idir<=maxind[0];idir++) { + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) { + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) { + + amrex::GpuArray box_min, box_max; + for (int dim = 0; dim < AMREX_SPACEDIM; ++dim) { + Real shift_val = 0.0; + if (dim == 0) shift_val = idir * lenx; // X-direction shift + if (dim == 1) shift_val = jdir * leny; // Y-direction shift + if (dim == 2) shift_val = kdir * lenz; // Z-direction shift + + box_min[dim] = plo[dim] + (box.smallEnd(dim) + 0.5) * dx_mesh[dim] + shift_val; + box_max[dim] = plo[dim] + (box.bigEnd(dim) + 0.5) * dx_mesh[dim] + shift_val; + } + + bool is_box_intersect = checkBoxIntersectsShell(box_min, box_max, center, radius_inner, radius_outer, dx_mesh); + + if(!is_box_intersect){ + continue; + } + + for(int i=0;i d_shell_particles(h_count); + LightConeParticle* d_shell_particles_ptr = d_shell_particles.data(); + + // Reset device counter + h_count = 0; + amrex::Gpu::copy(amrex::Gpu::hostToDevice, &h_count, &h_count + 1, d_count); + + //Fill the GPU lightconeparticle data + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + auto& particles = (this->ParticlesAt(lev,pti)).GetArrayOfStructs(); + + auto* pstruct = particles().data(); + + const long np = pti.numParticles(); + int grid = pti.index(); + + const FArrayBox& accel_fab= ((*ac_ptr)[0]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + const amrex::Box& box = pti.validbox(); + + for(int idir=-maxind[0];idir<=maxind[0];idir++) { + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) { + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) { + + amrex::GpuArray box_min, box_max; + for (int dim = 0; dim < AMREX_SPACEDIM; ++dim) { + Real shift_val = 0.0; + if (dim == 0) shift_val = idir * lenx; // X-direction shift + if (dim == 1) shift_val = jdir * leny; // Y-direction shift + if (dim == 2) shift_val = kdir * lenz; // Z-direction shift + + box_min[dim] = plo[dim] + (box.smallEnd(dim) + 0.5) * dx_mesh[dim] + shift_val; + box_max[dim] = plo[dim] + (box.bigEnd(dim) + 0.5) * dx_mesh[dim] + shift_val; + } + + bool is_box_intersect = checkBoxIntersectsShell(box_min, box_max, center, radius_inner, radius_outer, dx_mesh); + + if(!is_box_intersect){ + continue; + } + + for(int i=0;iresize(h_count); + amrex::Gpu::copy(amrex::Gpu::deviceToHost, + d_shell_particles.begin(), + d_shell_particles.end(), + shell_particles->begin()); + amrex::Gpu::streamSynchronize(); + } + // Clean up + amrex::The_Arena()->free(d_count); + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + AoS& particles = pti.GetArrayOfStructs(); + ParticleType* pstruct = particles().data(); + + const long np = pti.numParticles(); + int grid = pti.index(); + + const FArrayBox& accel_fab= ((*ac_ptr)[grid]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + update_dm_particle_move_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_old, a_half,do_move); + }); + + } + + auto dir=Concatenate("plt_light", int(100*(1/a_old-1)), 7); + auto name="ShellPC"; + amrex::Vector real_comp_names_shell; + real_comp_names_shell.clear(); + real_comp_names_shell.push_back("mass"); + real_comp_names_shell.push_back("xvel"); + real_comp_names_shell.push_back("yvel"); + real_comp_names_shell.push_back("zvel"); + real_comp_names_shell.push_back("xposold"); + real_comp_names_shell.push_back("yposold"); + real_comp_names_shell.push_back("zposold"); + real_comp_names_shell.push_back("xposvalid"); + real_comp_names_shell.push_back("yposvalid"); + real_comp_names_shell.push_back("zposvalid"); + std::string compression = "None@0"; + if(radius_inner>0&&radius_outer>radius_inner) { + int write_hdf5=0; +#ifdef AMREX_USE_HDF5 + write_hdf5=1; +#endif + if(write_hdf5!=1){ + //ShellPC->WritePlotFile(dir, name, real_comp_names_shell); + } +#ifdef AMREX_USE_HDF5 + else{ + ShellPC->WritePlotFileHDF5(dir, name, real_comp_names_shell, compression); + } +#endif + } + Print()<<"After write\t"<TotalNumberOfParticles()<<"\t"<amrex::ParticleContainer<7,0>::WritePlotFile(dir, name, real_comp_names_shell); + if (ac_ptr != &acceleration) delete ac_ptr; + + ParticleLevel& pmap = this->GetParticles(lev); + if (lev > 0 && sub_cycle) + { + if (! m_particle_locator.isValid(GetParGDB())) m_particle_locator.build(GetParGDB()); + m_particle_locator.setGeometry(GetParGDB()); + AmrAssignGrid> assign_grid = m_particle_locator.getGridAssignor(); + + amrex::ParticleLocData pld; + for (auto& kv : pmap) { + AoS& particles = kv.second.GetArrayOfStructs(); + ParticleType* pstruct = particles().data(); + const long np = particles.size(); + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + // amrex::ParticleContainer<4, 0>::SuperParticleType& p=pstruct[i]; + auto& p=pstruct[i]; + if(p.id()>0) { + const auto tup = assign_grid(p, lev, lev, where_width); + auto p_boxes = amrex::get<0>(tup); + auto p_levs = amrex::get<1>(tup); + if(p_boxes<0||p_levs<0) { + if (p.id() == amrex::GhostParticleID) + { + p.id() = -1; + } + else + { + amrex::Error("Trying to get rid of a non-ghost particle in moveKickDrift"); + } + } + } + }); + Gpu::streamSynchronize(); + } + } + delete ShellPC; +} + +void +DarkMatterParticleContainer::moveKick (MultiFab& acceleration, + int lev, + Real t, + Real dt, + Real a_new, + Real a_half) +{ + BL_PROFILE("DarkMatterParticleContainer::moveKick()"); + + const auto dxi = Geom(lev).InvCellSizeArray(); + + MultiFab* ac_ptr; + if (OnSameGrids(lev,acceleration)) + { + ac_ptr = &acceleration; + } + else + { + const IntVect& ng = acceleration.nGrowVect(); + ac_ptr = new amrex::MultiFab(this->ParticleBoxArray(lev), + this->ParticleDistributionMap(lev), + acceleration.nComp(),acceleration.nGrow()); + ac_ptr->setVal(0.); + if(acceleration.boxArray() == ac_ptr->boxArray())//this->finestLevel() == 0) + { + ac_ptr->Redistribute(acceleration,0,0,acceleration.nComp(),ng); + ac_ptr->FillBoundary(); + } + else + { + ac_ptr->ParallelCopy(acceleration,0,0,acceleration.nComp(),ng,ng); + ac_ptr->FillBoundary(); + } + } + + const GpuArray plo = Geom(lev).ProbLoArray(); + + int do_move = 0; + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { + + AoS& particles = pti.GetArrayOfStructs(); + ParticleType* pstruct = particles().data(); + const long np = pti.numParticles(); + int grid = pti.index(); + + const FArrayBox& accel_fab= ((*ac_ptr)[grid]); + Array4 accel= accel_fab.array(); + + int nc=AMREX_SPACEDIM; + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + update_dm_particle_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_half,a_new,do_move); + update_dm_particle_move_single(pstruct[i],nc, + accel, + plo,dxi,dt,a_half,a_new,do_move); + }); + } + + + if (ac_ptr != &acceleration) delete ac_ptr; +} + +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void update_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move) +{ + amrex::Real half_dt = 0.5 * dt; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; + + amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; + amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5; + amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; + + int i = static_cast(amrex::Math::floor(lx)); + int j = static_cast(amrex::Math::floor(ly)); + int k = static_cast(amrex::Math::floor(lz)); + + amrex::Real xint = lx - i; + amrex::Real yint = ly - j; + amrex::Real zint = lz - k; + + amrex::Real sx[] = {amrex::Real(1.)-xint, xint}; + amrex::Real sy[] = {amrex::Real(1.)-yint, yint}; + amrex::Real sz[] = {amrex::Real(1.)-zint, zint}; + + for (int d=0; d < AMREX_SPACEDIM; ++d) + { + amrex::Real val = 0.0; + for (int kk = 0; kk<=1; ++kk) + { + for (int jj = 0; jj <= 1; ++jj) + { + for (int ii = 0; ii <= 1; ++ii) + { + val += sx[amrex::Math::abs(ii-1)]* + sy[amrex::Math::abs(jj-1)]* + sz[amrex::Math::abs(kk-1)]*acc(i-ii,j-jj,k-kk,d); + } + } + } + + + p.rdata(d+1)=a_prev*p.rdata(d+1)+half_dt * val; + p.rdata(d+1)*=a_cur_inv; + } + +} + +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void store_dm_particle_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& phi, + amrex::GpuArray const& dx_mesh, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move, const Real& radius_inner, + const Real& radius_outer, int* d_count, const amrex::Box& box, + const int idir, const int jdir, const int kdir, + const bool is_file_write, LightConeParticle* d_shell_particles) +{ + amrex::Real half_dt = 0.5 * dt; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; + const GpuArray center({AMREX_D_DECL((phi[0]-plo[0])*0.5,(phi[1]-plo[1])*0.5,(phi[2]-plo[2])*0.5)}); + + if (do_move == 1) { + bool result=false; + + Real xlen, ylen, zlen; + // Looping over the periodic boxes + + + xlen = p.pos(0)+(idir)*(phi[0]-plo[0]) - center[0]; + ylen = p.pos(1)+(jdir)*(phi[1]-plo[1]) - center[1]; + zlen = p.pos(2)+(kdir)*(phi[2]-plo[2]) - center[2]; + Real mag = sqrt(xlen*xlen+ylen*ylen+zlen*zlen); + + if(mag>radius_inner && mag= 0) { + LightConeParticle tmp; + tmp.x = x1; tmp.y = y1; tmp.z = z1; + tmp.vx = vx; tmp.vy = vy; tmp.vz = vz; + d_shell_particles[idx] = tmp; + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void update_dm_particle_move_single (amrex::ParticleContainer<1+AMREX_SPACEDIM, 0>::SuperParticleType& p, + const int nc, + amrex::Array4 const& acc, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Real& dt, const amrex::Real& a_prev, + const amrex::Real& a_cur, const int& do_move) +{ + amrex::Real half_dt = 0.5 * dt; + amrex::Real a_cur_inv = 1.0 / a_cur; + amrex::Real dt_a_cur_inv = dt * a_cur_inv; + + if (do_move == 1) + { + for (int comp=0; comp < nc; ++comp) { + p.pos(comp) = p.pos(comp) + dt_a_cur_inv * p.rdata(comp+1); + } + } + +} + +void +DarkMatterParticleContainer::InitCosmo1ppcMultiLevel( + MultiFab& mf, const Real disp_fac[], const Real vel_fac[], + const Real particleMass, int disp_idx, int vel_idx, + BoxArray &baWhereNot, int lev, int nlevs) +{ + BL_PROFILE("DarkMatterParticleContainer::InitCosmo1ppcMultiLevel()"); + const int MyProc = ParallelDescriptor::MyProc(); + const Geometry& geom = m_gdb->Geom(lev); + const Real* dx = geom.CellSize(); + + static Vector calls; + + calls.resize(nlevs); + + calls[lev]++; + + if (calls[lev] > 1) return; + + Vector& particles = this->GetParticles(); + + particles.reserve(15); // So we don't ever have to do any copying on a resize. + + particles.resize(nlevs); + + ParticleType p; + Real disp[AMREX_SPACEDIM]; + Real vel[AMREX_SPACEDIM]; + + Real mean_disp[AMREX_SPACEDIM]={AMREX_D_DECL(0,0,0)}; + + + // + // The mf should be initialized according to the ics... + // + int outside_counter=0; + long outcount[3]={0,0,0}; + long outcountminus[3]={0,0,0}; + long totalcount=0; +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(mf,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + FArrayBox& myFab = mf[mfi]; + const Box& vbx = mfi.validbox(); + const int *fab_lo = vbx.loVect(); + const int *fab_hi = vbx.hiVect(); + ParticleLocData pld; + for (int kx = fab_lo[2]; kx <= fab_hi[2]; kx++) + { + for (int jx = fab_lo[1]; jx <= fab_hi[1]; jx++) + { + for (int ix = fab_lo[0]; ix <= fab_hi[0]; ix++) + { + IntVect indices(AMREX_D_DECL(ix, jx, kx)); + totalcount++; + if (baWhereNot.contains(indices)) + { + continue; + } + + for (int n = 0; n < AMREX_SPACEDIM; n++) + { + disp[n] = myFab(indices,disp_idx+n); + // + // Start with homogeneous distribution (for 1 p per cell in the center of the cell), + // + p.pos(n) = geom.ProbLo(n) + + (indices[n]+Real(0.5))*dx[n]; + if(disp[n]*disp_fac[n]>dx[n]/2.0) + outcount[n]++; + if(disp[n]*disp_fac[n]<-dx[n]/2.0) + outcountminus[n]++; + mean_disp[n]+=fabs(disp[n]); + // + // then add the displacement (input values weighted by domain length). + // + p.pos(n) += disp[n] * disp_fac[n]; + + // + // Set the velocities. + // + vel[n] = myFab(indices,vel_idx+n); + p.rdata(n+1) = vel[n] * vel_fac[n]; + } + // + // Set the mass of the particle from the input value. + // + p.rdata(0) = particleMass; + p.id() = ParticleType::NextID(); + p.cpu() = MyProc; + + if (!this->Where(p, pld)) + { + this->PeriodicShift(p); + + if (!this->Where(p, pld)) + amrex::Abort("DarkMatterParticleContainer::InitCosmo1ppcMultiLevel():invalid particle"); + } + + BL_ASSERT(pld.m_lev >= 0 && pld.m_lev <= m_gdb->finestLevel()); + //handle particles that ran out of this level into a finer one. + if (baWhereNot.contains(pld.m_cell)) + { + outside_counter++; + ParticleType newp[8]; + ParticleLocData new_pld; + for (int i=0;i<8;i++) + { + newp[i].rdata(0) = particleMass/8.0; + newp[i].id() = ParticleType::NextID(); + newp[i].cpu() = MyProc; + for (int dim=0;dimWhere(newp[i], new_pld)) + { + this->PeriodicShift(newp[i]); + + if (!this->Where(newp[i], new_pld)) + amrex::Abort("DarkMatterParticleContainer::InitCosmo1ppcMultiLevel():invalid particle"); + } + particles[new_pld.m_lev][std::make_pair(new_pld.m_grid, + new_pld.m_tile)].push_back(newp[i]); + } + + } + + // + // Add it to the appropriate PBox at the appropriate level. + // + else + particles[pld.m_lev][std::make_pair(pld.m_grid, pld.m_tile)].push_back(p); + } + } + } + } + Redistribute(); +} + +/* + Particle deposition +*/ + +void +DarkMatterParticleContainer::AssignDensityAndVels (Vector >& mf, int lev_min) const +{ + AssignDensity(mf, lev_min, AMREX_SPACEDIM+1); +} + +void +DarkMatterParticleContainer::InitFromBinaryMortonFile(const std::string& particle_directory, + int /*nextra*/, int skip_factor) { + BL_PROFILE("DarkMatterParticleContainer::InitFromBinaryMortonFile"); + + ParticleMortonFileHeader hdr; + ReadHeader(particle_directory, "Header", hdr); + + uint64_t num_parts = hdr.NP; + int DM = hdr.DM; + int NX = hdr.NX; + int float_size = hdr.SZ; + int num_files = hdr.NF; + size_t psize = (DM + NX) * float_size; + + std::string particle_file_base = particle_directory + "/particles."; + std::vector file_names; + for (int i = 0; i < num_files; ++i) + file_names.push_back(get_file_name(particle_file_base, i)); + + const int lev = 0; + const BoxArray& ba = ParticleBoxArray(lev); + int num_boxes = ba.size(); + uint64_t num_parts_per_box = num_parts / num_boxes; + uint64_t num_parts_per_file = num_parts / num_files; + uint64_t num_bytes_per_file = num_parts_per_file * psize; + + std::vector box_morton_keys(num_boxes); + for (int i = 0; i < num_boxes; ++i) { + const Box& box = ba[i]; + unsigned int x = box.smallEnd(0); + unsigned int y = box.smallEnd(1); + unsigned int z = box.smallEnd(2); + box_morton_keys[i].morton_id = get_morton_index(x, y, z); + box_morton_keys[i].box_id = i; + } + + std::sort(box_morton_keys.begin(), box_morton_keys.end(), by_morton_id()); + + std::vector file_indices(num_boxes); + for (int i = 0; i < num_boxes; ++i) + file_indices[box_morton_keys[i].box_id] = i; + + ParticleType p; + for (MFIter mfi = MakeMFIter(lev, false); mfi.isValid(); ++mfi) { // no tiling + const int grid = mfi.index(); + const int tile = mfi.LocalTileIndex(); + auto& particles = GetParticles(lev); + + uint64_t start = file_indices[grid]*num_parts_per_box; + uint64_t stop = start + num_parts_per_box; + + int file_num = start / num_parts_per_file; + uint64_t seek_pos = (start * psize ) % num_bytes_per_file; + std::string file_name = file_names[file_num]; + + std::ifstream ifs; + ifs.open(file_name.c_str(), std::ios::in|std::ios::binary); + if (!ifs ) { + amrex::Print() << "Failed to open file " << file_name << " for reading. \n"; + amrex::Abort(); + } + + ifs.seekg(seek_pos, std::ios::beg); + + for (uint64_t i = start; i < stop; ++i) { + int next_file = i / num_parts_per_file; + if (next_file != file_num) { + file_num = next_file; + file_name = file_names[file_num]; + ifs.close(); + ifs.open(file_name.c_str(), std::ios::in|std::ios::binary); + if (!ifs ) { + amrex::Print() << "Failed to open file " << file_name << " for reading. \n"; + amrex::Abort(); + } + } + + Vector fpos(DM); + Vector fextra(NX); + ifs.read((char*)&fpos[0], DM*sizeof(float)); + ifs.read((char*)&fextra[0], NX*sizeof(float)); + + if ( (i - start) % skip_factor == 0 ) { + AMREX_D_TERM(p.pos(0) = fpos[0];, + p.pos(1) = fpos[1];, + p.pos(2) = fpos[2];); + + for (int comp = 0; comp < NX; comp++) + p.rdata(AMREX_SPACEDIM+comp) = fextra[comp]; + + p.rdata(AMREX_SPACEDIM) *= skip_factor; + + p.id() = ParticleType::NextID(); + p.cpu() = ParallelDescriptor::MyProc(); + particles[std::make_pair(grid, tile)].push_back(p); + } + } + } + + Redistribute(); +} + diff --git a/Exec/HaloFinder_GPU/Filelist_128x128x128.txt b/Exec/HaloFinder_GPU/Filelist_128x128x128.txt new file mode 100644 index 000000000..935d96bce --- /dev/null +++ b/Exec/HaloFinder_GPU/Filelist_128x128x128.txt @@ -0,0 +1,4 @@ +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-0 +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-1 +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-2 +/global/cfs/cdirs/nyx/jmsexton/mockgen/shamik_output/output/mockgen4nyx_test_13579_Lbox-7700.0_N-128_proc-3 diff --git a/Exec/HaloFinder_GPU/Filelist_2048x2048x2048.txt b/Exec/HaloFinder_GPU/Filelist_2048x2048x2048.txt new file mode 100644 index 000000000..8054e81f8 --- /dev/null +++ b/Exec/HaloFinder_GPU/Filelist_2048x2048x2048.txt @@ -0,0 +1,128 @@ +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-0 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-1 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-10 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-100 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-101 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-102 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-103 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-104 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-105 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-106 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-107 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-108 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-109 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-11 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-110 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-111 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-112 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-113 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-114 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-115 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-116 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-117 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-118 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-119 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-12 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-120 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-121 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-122 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-123 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-124 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-125 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-126 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-127 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-13 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-14 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-15 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-16 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-17 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-18 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-19 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-2 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-20 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-21 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-22 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-23 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-24 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-25 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-26 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-27 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-28 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-29 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-3 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-30 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-31 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-32 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-33 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-34 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-35 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-36 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-37 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-38 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-39 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-4 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-40 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-41 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-42 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-43 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-44 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-45 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-46 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-47 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-48 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-49 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-5 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-50 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-51 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-52 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-53 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-54 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-55 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-56 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-57 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-58 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-59 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-6 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-60 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-61 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-62 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-63 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-64 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-65 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-66 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-67 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-68 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-69 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-7 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-70 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-71 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-72 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-73 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-74 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-75 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-76 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-77 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-78 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-79 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-8 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-80 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-81 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-82 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-83 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-84 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-85 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-86 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-87 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-88 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-89 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-9 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-90 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-91 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-92 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-93 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-94 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-95 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-96 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-97 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-98 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-2048/mockgen4nyx_hires_13579_Lbox-7700.0_N-2048_proc-99 diff --git a/Exec/HaloFinder_GPU/Filelist_4096x4096x4096.txt b/Exec/HaloFinder_GPU/Filelist_4096x4096x4096.txt new file mode 100755 index 000000000..88c971162 --- /dev/null +++ b/Exec/HaloFinder_GPU/Filelist_4096x4096x4096.txt @@ -0,0 +1,256 @@ +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-0 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-1 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-10 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-100 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-101 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-102 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-103 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-104 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-105 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-106 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-107 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-108 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-109 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-11 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-110 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-111 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-112 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-113 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-114 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-115 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-116 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-117 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-118 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-119 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-12 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-120 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-121 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-122 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-123 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-124 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-125 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-126 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-127 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-128 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-129 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-13 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-130 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-131 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-132 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-133 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-134 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-135 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-136 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-137 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-138 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-139 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-14 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-140 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-141 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-142 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-143 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-144 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-145 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-146 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-147 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-148 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-149 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-15 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-150 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-151 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-152 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-153 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-154 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-155 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-156 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-157 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-158 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-159 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-16 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-160 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-161 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-162 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-163 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-164 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-165 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-166 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-167 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-168 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-169 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-17 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-170 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-171 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-172 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-173 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-174 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-175 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-176 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-177 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-178 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-179 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-18 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-180 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-181 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-182 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-183 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-184 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-185 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-186 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-187 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-188 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-189 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-19 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-190 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-191 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-192 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-193 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-194 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-195 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-196 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-197 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-198 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-199 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-2 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-20 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-200 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-201 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-202 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-203 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-204 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-205 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-206 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-207 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-208 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-209 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-21 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-210 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-211 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-212 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-213 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-214 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-215 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-216 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-217 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-218 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-219 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-22 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-220 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-221 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-222 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-223 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-224 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-225 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-226 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-227 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-228 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-229 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-23 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-230 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-231 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-232 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-233 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-234 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-235 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-236 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-237 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-238 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-239 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-24 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-240 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-241 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-242 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-243 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-244 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-245 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-246 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-247 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-248 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-249 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-25 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-250 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-251 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-252 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-253 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-254 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-255 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-26 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-27 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-28 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-29 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-3 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-30 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-31 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-32 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-33 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-34 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-35 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-36 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-37 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-38 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-39 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-4 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-40 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-41 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-42 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-43 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-44 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-45 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-46 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-47 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-48 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-49 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-5 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-50 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-51 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-52 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-53 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-54 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-55 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-56 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-57 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-58 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-59 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-6 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-60 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-61 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-62 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-63 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-64 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-65 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-66 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-67 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-68 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-69 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-7 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-70 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-71 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-72 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-73 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-74 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-75 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-76 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-77 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-78 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-79 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-8 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-80 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-81 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-82 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-83 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-84 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-85 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-86 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-87 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-88 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-89 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-9 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-90 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-91 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-92 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-93 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-94 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-95 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-96 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-97 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-98 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/agora-cosmology_4096/agora-cosmology_4096-ICs_Lbox-7700.0_N-4096_2nlpt_part-99 diff --git a/Exec/HaloFinder_GPU/Filelist_768x768x768.txt b/Exec/HaloFinder_GPU/Filelist_768x768x768.txt new file mode 100644 index 000000000..dc979fed3 --- /dev/null +++ b/Exec/HaloFinder_GPU/Filelist_768x768x768.txt @@ -0,0 +1,4 @@ +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-768/mockgen4nyx_test_13579_Lbox-7700.0_N-768_proc-0 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-768/mockgen4nyx_test_13579_Lbox-7700.0_N-768_proc-1 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-768/mockgen4nyx_test_13579_Lbox-7700.0_N-768_proc-2 +/global/cfs/cdirs/nyx/ics4nyx-lightcones/gridnside-768/mockgen4nyx_test_13579_Lbox-7700.0_N-768_proc-3 diff --git a/Exec/HaloFinder_GPU/GNUmakefile b/Exec/HaloFinder_GPU/GNUmakefile new file mode 100644 index 000000000..47d57a401 --- /dev/null +++ b/Exec/HaloFinder_GPU/GNUmakefile @@ -0,0 +1,89 @@ +# AMREX_HOME defines the directory in which we will find all the AMReX code +AMREX_HOME ?= ../../subprojects/amrex + +# TOP defines the directory in which we will find Source, Exec, etc +TOP = ../.. + +# compilation options +COMP = gnu +USE_MPI = TRUE +# Use with Async IO +MPI_THREAD_MULTIPLE = TRUE +USE_OMP = FALSE +USE_CUDA = TRUE + +USE_SUNDIALS = TRUE +USE_FORT_ODE = FALSE +SUNDIALS_ROOT = /global/cfs/cdirs/m4106/jmsexton/sundials_shared/sundials/instdir +#SUNDIALS_ROOT ?= ${CURDIR}/$(TOP)/../sundials/instdir +USE_HDF5 = FALSE +HDF5_HOME = /opt/cray/pe/hdf5/1.12.2.9 + +COMMON_DIR = /global/common/software/nyx/nataraj2 +#ASCENT_HOME ?= ${CURDIR}/$(TOP)/../ascent/install-debug +ASCENT_HOME ?= ${COMMON_DIR}/ascent/install-debug + +USE_ASCENT_INSITU = TRUE +ifeq ($(USE_ASCENT_INSITU),TRUE) + ASCENT_HOME ?= NOT_SET + ifneq ($(ASCENT_HOME),NOT_SET) + include $(ASCENT_HOME)/share/ascent/ascent_config.mk + endif + USE_CONDUIT = TRUE + USE_ASCENT = TRUE +endif + + +PROFILE = FALSE +TRACE_PROFILE = FALSE +COMM_PROFILE = FALSE +TINY_PROFILE = FALSE + +PRECISION = DOUBLE +USE_SINGLE_PRECISION_PARTICLES = TRUE +DEBUG = FALSE + +# physics +DIM = 3 +USE_HEATCOOL = TRUE +USE_SAVE_REACT = FALSE +USE_AGN = FALSE +ifeq ($(NO_HYDRO),TRUE) +USE_SDC = FALSE +USE_SUNDIALS = FALSE +USE_FUSED = FALSE +else +ifeq ($(USE_HEATCOOL),TRUE) +USE_SDC = TRUE +USE_SUNDIALS = TRUE +ifeq ($(USE_HIP),TRUE) +USE_FUSED ?= $(USE_HIP) +endif +USE_FUSED ?= $(USE_CUDA) +else +USE_SDC = FALSE +USE_SUNDIALS = FALSE +USE_FUSED = FALSE +endif +endif + +USE_SDC = FALSE +USE_FUSED = TRUE +USE_HEATCOOL=FALSE + +USE_CONST_SPECIES = TRUE +NEUTRINO_PARTICLES = FALSE +NEUTRINO_DARK_PARTICLES = FALSE + +USE_OWN_BCS = FALSE + +# Halo finder +BOOST_INLUDE_DIR := /usr/include/boost +REEBER = TRUE +DIY_INCLUDE_DIR = /global/common/software/nyx/nataraj2/diy/include +REEBER_EXT_HOME = /global/common/software/nyx/nataraj2/reeber + +Bpack := ./Make.package +Blocs := . + +include $(TOP)/Exec/Make.Nyx diff --git a/Exec/HaloFinder_GPU/GNUmakefile_Frontier b/Exec/HaloFinder_GPU/GNUmakefile_Frontier new file mode 100644 index 000000000..fbf389dcd --- /dev/null +++ b/Exec/HaloFinder_GPU/GNUmakefile_Frontier @@ -0,0 +1,90 @@ +# AMREX_HOME defines the directory in which we will find all the AMReX code +AMREX_HOME ?= ../../subprojects/amrex + +# TOP defines the directory in which we will find Source, Exec, etc +TOP = ../.. + +# compilation options +COMP = HIP +USE_MPI = TRUE +# Use with Async IO +MPI_THREAD_MULTIPLE = TRUE +USE_OMP = FALSE +USE_CUDA = FALSE +USE_HIP = TRUE + +USE_SUNDIALS = TRUE +USE_FORT_ODE = FALSE +SUNDIALS_ROOT = /ccs/home/nataraj2/Softwares/sundials_GitHub/sundials/instdir +#SUNDIALS_ROOT ?= ${CURDIR}/$(TOP)/../sundials/instdir +USE_HDF5 = FALSE +HDF5_HOME = /opt/cray/pe/hdf5/1.12.2.9 + +COMMON_DIR = /ccs/home/nataraj2/Softwares +#ASCENT_HOME ?= ${CURDIR}/$(TOP)/../ascent/install-debug +ASCENT_HOME ?= ${COMMON_DIR}/ascent/install-debug + +USE_ASCENT_INSITU = FALSE +ifeq ($(USE_ASCENT_INSITU),TRUE) + ASCENT_HOME ?= NOT_SET + ifneq ($(ASCENT_HOME),NOT_SET) + include $(ASCENT_HOME)/share/ascent/ascent_config.mk + endif + USE_CONDUIT = TRUE + USE_ASCENT = TRUE +endif + + +PROFILE = FALSE +TRACE_PROFILE = FALSE +COMM_PROFILE = FALSE +TINY_PROFILE = FALSE + +PRECISION = DOUBLE +USE_SINGLE_PRECISION_PARTICLES = TRUE +DEBUG = FALSE + +# physics +DIM = 3 +USE_HEATCOOL = TRUE +USE_SAVE_REACT = FALSE +USE_AGN = FALSE +ifeq ($(NO_HYDRO),TRUE) +USE_SDC = FALSE +USE_SUNDIALS = FALSE +USE_FUSED = FALSE +else +ifeq ($(USE_HEATCOOL),TRUE) +USE_SDC = TRUE +USE_SUNDIALS = TRUE +ifeq ($(USE_HIP),TRUE) +USE_FUSED ?= $(USE_HIP) +endif +USE_FUSED ?= $(USE_CUDA) +else +USE_SDC = FALSE +USE_SUNDIALS = FALSE +USE_FUSED = FALSE +endif +endif + +USE_SDC = FALSE +USE_FUSED = FALSE +USE_HEATCOOL=FALSE + +USE_CONST_SPECIES = TRUE +NEUTRINO_PARTICLES = FALSE +NEUTRINO_DARK_PARTICLES = FALSE + +USE_OWN_BCS = FALSE + +# Halo finder +BOOST_INLUDE_DIR := /usr/include/boost +REEBER = TRUE +DIY_INCLUDE_DIR = /ccs/home/nataraj2/Softwares/diy/include +REEBER_EXT_HOME = /ccs/home/nataraj2/Softwares/reeber + +Bpack := ./Make.package +Blocs := . + +include $(TOP)/Exec/Make.Nyx diff --git a/Exec/HaloFinder_GPU/LightConeParticle.H b/Exec/HaloFinder_GPU/LightConeParticle.H new file mode 100644 index 000000000..eb0a9f244 --- /dev/null +++ b/Exec/HaloFinder_GPU/LightConeParticle.H @@ -0,0 +1,13 @@ +#ifndef _LightConeParticle_H_ +#define _LightConeParticle_H_ +class LightConeParticle { + public: + double x, y, z, vx, vy, vz; +}; +//extern std::vector shell_particles; + +void writeBinaryVTK(const std::string& filename, + const std::vector& shell_particles); +void writeBinarySimple(const std::string& filename, + const std::vector& shell_particles); +#endif diff --git a/Exec/HaloFinder_GPU/LightConeParticle.cpp b/Exec/HaloFinder_GPU/LightConeParticle.cpp new file mode 100644 index 000000000..7b38abaa1 --- /dev/null +++ b/Exec/HaloFinder_GPU/LightConeParticle.cpp @@ -0,0 +1,158 @@ +#include +#include +#include +#include +#include +#include + +#include + +void SwapEnd(float& val) { + // Swap endianess if necessary + char* bytes = reinterpret_cast(&val); + std::swap(bytes[0], bytes[3]); + std::swap(bytes[1], bytes[2]); +} + +void writeBinaryVTK(const std::string& filename, const std::vector& particles) { + + int rank = amrex::ParallelDescriptor::MyProc(); + int size = amrex::ParallelDescriptor::NProcs(); + + long int local_num_particles = particles.size(); + long int total_num_particles = local_num_particles; + + // Get total particles across all ranks + amrex::ParallelDescriptor::ReduceLongSum(total_num_particles); + + // Compute offset for this rank's data + size_t offset = 0; + MPI_Exscan(&local_num_particles, &offset, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD); + + // Header handling + size_t header_size = 0; + + if (rank == 0) { + std::ofstream file(filename, std::ios::binary); + if (!file) { + amrex::Abort("Error: Could not open file " + filename + "\n"); + } + + // Write the header + file << "# vtk DataFile Version 2.0\n"; + file << "Particle Cloud Data\n"; + file << "BINARY\n"; + file << "DATASET POLYDATA\n"; + file << "POINTS " << total_num_particles << " float\n"; + + // Determine header size + file.seekp(0, std::ios::end); + header_size = file.tellp(); + file.close(); + } + + // Broadcast the header size to all ranks + amrex::ParallelDescriptor::Bcast(&header_size, 1, 0); + + // Use MPI collective I/O for binary data + MPI_File mpi_file; + MPI_File_open(MPI_COMM_WORLD, filename.c_str(), + MPI_MODE_WRONLY | MPI_MODE_APPEND, MPI_INFO_NULL, &mpi_file); + + // Compute byte offset for this rank + size_t byte_offset = header_size + sizeof(float) * 3 * offset; + + // Prepare local data + std::vector data(3 * local_num_particles); + for (size_t i = 0; i < local_num_particles; ++i) { + data[3 * i] = particles[i].x; + data[3 * i + 1] = particles[i].y; + data[3 * i + 2] = particles[i].z; + + // Convert to big-endian if needed + SwapEnd(data[3 * i]); + SwapEnd(data[3 * i + 1]); + SwapEnd(data[3 * i + 2]); + } + + // Write particle data collectively + MPI_File_write_at_all(mpi_file, byte_offset, data.data(), data.size(), MPI_FLOAT, MPI_STATUS_IGNORE); + + MPI_File_close(&mpi_file); + + if (rank == 0) { + std::cout << "Successfully wrote VTK file: " << filename << "\n"; + } +} + +void writeBinarySimple(const std::string& filename, const std::vector& particles) { + + int rank = amrex::ParallelDescriptor::MyProc(); + int size = amrex::ParallelDescriptor::NProcs(); + + long int local_num_particles = particles.size(); + + long int total_num_particles = local_num_particles; + amrex::ParallelDescriptor::ReduceLongSum(total_num_particles); + + if (amrex::ParallelDescriptor::IOProcessor()) { + std::cout << "Total number of particles across all ranks: " << total_num_particles << "\n"; + } + + // Compute offset for this rank's data + size_t offset = 0; + MPI_Exscan(&local_num_particles, &offset, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD); + + // Header handling + unsigned long header_size = 0; + + if (rank == 0) { + std::ofstream file(filename, std::ios::binary); + if (!file) { + amrex::Abort("Error: Could not open file " + filename + "\n"); + } + file.seekp(0, std::ios::end); + header_size = file.tellp(); + file.close(); + } + + // Broadcast the header size to all ranks + amrex::ParallelDescriptor::Bcast(&header_size, 1, 0); + + // Use MPI collective I/O for binary data + MPI_File mpi_file; + MPI_File_open(MPI_COMM_WORLD, filename.c_str(), + MPI_MODE_WRONLY | MPI_MODE_APPEND, MPI_INFO_NULL, &mpi_file); + + // Compute byte offset for this rank + size_t byte_offset = header_size + sizeof(float) * 6 * offset; + + // Prepare local data + std::vector data(6 * local_num_particles); + for (size_t i = 0; i < local_num_particles; ++i) { + data[6 * i] = particles[i].x; + data[6 * i + 1] = particles[i].y; + data[6 * i + 2] = particles[i].z; + data[6 * i + 3] = particles[i].vx; + data[6 * i + 4] = particles[i].vy; + data[6 * i + 5] = particles[i].vz; + + // Convert to big-endian if needed + SwapEnd(data[6 * i]); + SwapEnd(data[6 * i + 1]); + SwapEnd(data[6 * i + 2]); + SwapEnd(data[6 * i + 3]); + SwapEnd(data[6 * i + 4]); + SwapEnd(data[6 * i + 5]); + } + + // Write particle data collectively + MPI_File_write_at_all(mpi_file, byte_offset, data.data(), data.size(), MPI_FLOAT, MPI_STATUS_IGNORE); + + MPI_File_close(&mpi_file); + + if (rank == 0) { + std::cout << "Successfully wrote VTK file: " << filename << "\n"; + } +} + diff --git a/Exec/HaloFinder_GPU/Make.package b/Exec/HaloFinder_GPU/Make.package new file mode 100644 index 000000000..7ac175cdc --- /dev/null +++ b/Exec/HaloFinder_GPU/Make.package @@ -0,0 +1,4 @@ +CEXE_sources += Prob.cpp +CEXE_sources += Prob_error.cpp +CEXE_sources += LightConeParticle.cpp +CEXE_headers += LightConeParticle.H diff --git a/Exec/HaloFinder_GPU/NeutrinoParticleContainer.H b/Exec/HaloFinder_GPU/NeutrinoParticleContainer.H new file mode 100644 index 000000000..1baa33743 --- /dev/null +++ b/Exec/HaloFinder_GPU/NeutrinoParticleContainer.H @@ -0,0 +1,49 @@ +#ifdef NEUTRINO_PARTICLES +#ifndef _NeutrinoParticleContainer_H_ +#define _NeutrinoParticleContainer_H_ + +#include + +class NeutrinoParticleContainer + : public NyxParticleContainer<2+AMREX_SPACEDIM> +{ + +private: + int m_relativistic; // if 1 then we weight the mass by gamma in AssignDensity* + amrex::Real m_csq; // the square of the speed of light -- used to compute relativistic effects + +public: + NeutrinoParticleContainer (amrex::Amr* amr) + : NyxParticleContainer<2+AMREX_SPACEDIM>(amr) + { } + + using MyParIter = amrex::ParIter<2+AMREX_SPACEDIM>; + using MyConstParIter = amrex::ParConstIter<2+AMREX_SPACEDIM>; + + virtual ~NeutrinoParticleContainer () {} + + void SetRelativistic (int relativistic) { m_relativistic = relativistic; } + + void SetCSquared (amrex::Real csq) { m_csq = csq; } + +#ifndef NEUTRINO_DARK_PARTICLES + void AssignDensity (amrex::Vector >& mf, int lev_min = 0, + int ncomp = 1, int finest_level = -1, int ngrow = 2) const + { AssignRelativisticDensity (mf,lev_min,ncomp,finest_level,ngrow); } + + void AssignRelativisticDensitySingleLevel (amrex::MultiFab& mf, int level, int ncomp=1, int particle_lvl_offset = 0) const; + + void AssignRelativisticDensity (amrex::Vector >& mf, + int lev_min = 0, int ncomp = 1, int finest_level = -1, int ngrow = 2) const; +#endif + + virtual void moveKickDrift (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_old = 1.0, amrex::Real a_half = 1.0, int where_width = 0); + virtual void moveKick (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_new = 1.0, amrex::Real a_half = 1.0); +}; + +#endif /*_NeutrinoParticleContainer_H_*/ +#endif /*NEUTRINO_PARTICLES*/ diff --git a/Exec/HaloFinder_GPU/Nyx.H b/Exec/HaloFinder_GPU/Nyx.H new file mode 100644 index 000000000..89b190141 --- /dev/null +++ b/Exec/HaloFinder_GPU/Nyx.H @@ -0,0 +1,1076 @@ + +#ifndef _Nyx_H_ +#define _Nyx_H_ + +#include +#include +#include +#include + +#ifdef AMREX_PARTICLES +#include +#include +#ifdef AGN +#include +#endif +#ifdef NEUTRINO_PARTICLES +#include +#endif +#else +#include +#endif + +#ifdef REEBER +#ifndef REEBER_HIST +#include +#endif +#endif + +#ifdef AMREX_USE_CONDUIT +#include +using namespace amrex; +using namespace conduit; + +#include +using namespace ascent; +#endif + +#include + +#ifdef BL_HDF5 +#include +#endif + +using std::istream; +using std::ostream; +#ifdef AMREX_PARTICLES +typedef NyxParticleContainer<1+AMREX_SPACEDIM+3> StellarParticleContainer; +#endif +#ifdef NO_HYDRO +enum StateType { + PhiGrav_Type = 0, + Gravity_Type, + NUM_STATE_TYPE +}; +static constexpr int max_prob_param = 5; +#endif + +#ifndef NO_HYDRO +enum StateType { + State_Type = 0, + DiagEOS_Type, + PhiGrav_Type, + Gravity_Type, +#ifdef SDC + SDC_IR_Type, +#endif + NUM_STATE_TYPE +}; +static constexpr int max_prob_param = 20; +#endif + +// +// AmrLevel-derived class for hyperbolic conservation equations for stellar +// media +// + +class Nyx + : + public amrex::AmrLevel +{ +public: + // + //Default constructor. Builds invalid object. + // + Nyx(); + + // + //The basic constructor. + // + Nyx(amrex::Amr& papa, int lev, const amrex::Geometry& level_geom, + const amrex::BoxArray& bl, const amrex::DistributionMapping& dm, + amrex::Real time); + + // + //The destructor. + // + virtual ~Nyx() override; + + // + //Restart from a checkpoint file. + // + virtual void restart(amrex::Amr& papa, istream& is, bool b_read_special=false) override; + + // + //Call amrex::AmrLevel::checkPoint and then add radiation info + // + virtual void checkPoint(const std::string& dir, std::ostream& os, + amrex::VisMF::How how, bool dump_old) override; + virtual void checkPointPre(const std::string& dir, std::ostream& os) override; + virtual void checkPointPost(const std::string& dir, std::ostream& os) override; + + // A string written as the first item in `write_plot_file()` at level zero. + // It is so we can distinguish between different types of plot files. For + // Nyx it has the form: Nyx-Vnnn. + virtual std::string thePlotFileType() const override; + + virtual void setPlotVariables() override; + + // + //Write a plotfile to specified directory. + // + virtual void writePlotFile(const std::string& dir, ostream& os, amrex::VisMF::How how) override; +#ifdef BL_HDF5 + virtual void writePlotFileHDF5(const std::string& dir, ostream& os, amrex::VisMF::How how) override; +#endif + virtual void writePlotFilePre(const std::string& dir, ostream& os) override; + virtual void writePlotFilePost(const std::string& dir, ostream& os) override; + + static void writeBuildInfo (); + void writeJobInfo (const std::string& dir); + +#ifdef AMREX_USE_CONDUIT + void blueprint_check_point (); +#endif + + // + //Write amrex::MultiFab as plot file + // + void writeMultiFabAsPlotFile(const std::string& pltfile, const amrex::MultiFab& mf, std::string componentName); + // + //Write all parameters into specified directory. + // + static int write_parameters_in_plotfile; + virtual void write_parameter_file(const std::string& dir); + static int write_skip_prepost; + static int write_hdf5; + + static int runlog_precision; + static int runlog_precision_terse; + + // + //Define data descriptors. + // + static void variable_setup(); + static void hydro_setup(); + static void heatcool_setup(); + static void no_hydro_setup(); + + // + //Define tagging functions. + // + static void error_setup(); + // + //Cleanup data descriptors at end of run. + // + static void variable_cleanup(); + + // + //Initialize grid data at problem start-up. + // + virtual void initData() override; + + // + //Initialize grid data from a plotfile at problem start-up. + // + void init_from_plotfile(); + + // + //Initialize grid data from a plotfile at problem start-up. + // + void ReadPlotFile(bool first, const std::string& plot_file_name, bool& rhoe_infile); + + // + // Read inputs related to comoving coordinates + // + static void read_comoving_params(); + + // + // Initial redshift + // + static amrex::Real initial_z; + + // + // Initial time in code units + // + static amrex::Real initial_time; + + // + // End time in code units + // + static amrex::Real final_time; + + // + // Final a -- used as stopping criterion if positive + // + static amrex::Real final_a; + + // + // Final z -- used as stopping criterion if positive + // + static amrex::Real final_z; + + // + // Relative change in a allowed in one timestep + // + static amrex::Real relative_max_change_a; + + // + // Absolute change in a allowed in one timestep for fixed delta_a + // + static amrex::Real absolute_max_change_a; + + // + // Positive number means use powers of 2 binning for relative dt + // + static amrex::Real dt_binpow; + + // + // Old and new times at which "old_a" and "new_a" are defined. + // + static amrex::Real old_a_time; + static amrex::Real new_a_time; + + // + // "a" at old_a_time and new_a_time + // + static amrex::Real old_a; + static amrex::Real new_a; + + // + // comoving parameters + // + static amrex::Real comoving_OmB; + static amrex::Real comoving_OmM; + static amrex::Real comoving_OmR; + static amrex::Real comoving_h; + static int comoving_type; + + // + // Get the comoving coordinate "a" + // + amrex::Real get_comoving_a(amrex::Real time); + static void integrate_time_given_a(const amrex::Real a0, const amrex::Real a1, amrex::Real& dt); + static void integrate_distance_given_a(const amrex::Real a0, const amrex::Real a1, amrex::Real& dt); + void integrate_comoving_a(const amrex::Real old_a, amrex::Real& new_a, const amrex::Real dt); + void integrate_comoving_a(amrex::Real time, amrex::Real dt); + + // + // Time step control based on "a" not growing too fast + // + void integrate_comoving_a_to_a(const amrex::Real old_a, const amrex::Real a_value, amrex::Real& dt); + void integrate_comoving_a_to_z(const amrex::Real old_a, const amrex::Real z_value, amrex::Real& dt); + void est_maxdt_comoving_a(const amrex::Real old_a, amrex::Real & dt); + void est_lindt_comoving_a(const amrex::Real old_a, const amrex::Real new_a, amrex::Real& dt); + void estdt_comoving_a(const amrex::Real old_a,amrex::Real& new_a,amrex::Real& dt,const amrex::Real change_allowed,const amrex::Real fixed_da,const amrex::Real final_a,int& dt_modified); + void enforce_percent_change(const amrex::Real old_a,amrex::Real& new_a,amrex::Real& dt,const amrex::Real change_allowed); + void enforce_final_a(const amrex::Real old_a,amrex::Real& new_a,amrex::Real& dt,const amrex::Real final_a); + void comoving_est_time_step(amrex::Real& cur_time, amrex::Real& est_dt); + + // + // Time step control based on "z" not passing one of the specified plot_z_values + // + void plot_z_est_time_step(amrex::Real& est_dt, bool& dt_changed); + + // + // Time step control based on "z" not passing one of the specified analysis_z_values + // + void analysis_z_est_time_step(amrex::Real& est_dt, bool& dt_changed); + + // + // How to initialize "a" at restart (from checkpoint or plotfile) + // + void comoving_a_post_restart(const std::string& restart_file); + + // + // How to initialize typical values at restart (from checkpoint or plotfile) + // + void typical_values_post_restart(const std::string& restart_file); + + // + // Initialize the zhi component of the EOS from a binary input file + // + void init_zhi (); + + // + // Initialize the Santa Barbara problem + // + void init_santa_barbara(int init_sb_vels); + + void check_initial_species(); + void init_e_from_T(const amrex::Real& a); + + // + // Initialize from MUSIC + // + void initcosmo(); + void icReadAndPrepareFab(std::string mfDirName, int nghost, amrex::MultiFab &mf); + + // + // Write forcing spectrum in checkpoint directories + // + void forcing_check_point (const std::string& dir); + + // + // Restore spectrum at restart + // + void forcing_post_restart(const std::string& restart_file); + + void integrate_state_force( + amrex::Box const& bx, + amrex::Array4 const& state, + amrex::Array4 const& diag_eos, + const amrex::Real* dx, + const amrex::Real time, + const amrex::Real a, + const amrex::Real half_dt); + + void ext_src_force( + amrex::Box const& bx, + amrex::Array4 const& old_state, + amrex::Array4 const& new_state, + amrex::Array4 const& old_diag, + amrex::Array4 const& new_diag, + amrex::Array4 const& src, + const amrex::Real* problo, + const amrex::Real* dx, + const amrex::Real time, + const amrex::Real z, + const amrex::Real dt); + + // + // Read particle-related inputs + // + static void read_particle_params(); + + // + // Read initialization-related inputs + // + static void read_init_params(); + + // + // Read hydro-related inputs + // +#ifndef NO_HYDRO + static void read_hydro_params(); +#endif + + // + // Write particles in checkpoint directories + // + void particle_check_point(const std::string& dir); + +#ifdef AMREX_PARTICLES + // + // Write particles in plotfile directories + // + void particle_plot_file(const std::string& dir); + + // + // How to initialize at restart + // + void particle_post_restart(const std::string& restart_file, bool is_checkpoint = true); + + // + // Redistribute + // + // override for sanity + void particle_redistribute(int lbase = 0, bool init = false) override; + + // + // Initialize particle locations and velocities (and strengths if relevant) + // + virtual void init_particles(); + + // + // Setup virtual particles if necessary + // + void setup_virtual_particles(); + + // + // Remove virtual particles if necessary + // + void remove_virtual_particles(); + // + // Setup ghost particles (for finer levels) if necessary + // + void setup_ghost_particles(int ngrow); + + // + // Remove ghost particles (for this level) if necessary + // + void remove_ghost_particles(); + + // + // Time step control based on particles + // + void particle_est_time_step(amrex::Real& est_dt); + + // + // Default verbosity of Particle class + // + static int particle_verbose; + + // + // Default cfl of particles in Particle class + // + static amrex::Real particle_cfl; +#ifdef NEUTRINO_PARTICLES + static amrex::Real neutrino_cfl; +#endif + + // + // Shall we write the initial single-level particle density into a multifab + // called "ParticleDensity"? + // + static int write_particle_density_at_init; +#endif + + // + // Derived quantities associated with particles + // + std::unique_ptr particle_derive (const std::string& name, amrex::Real time, int ngrow); + + // + //Set time levels of state data. + // + virtual void setTimeLevel (amrex::Real time, amrex::Real dt_old, amrex::Real dt_new) override; + + // + //Initialize data on this level from another Nyx (during regrid). + // + virtual void init(amrex::AmrLevel& old) override; + + // + // Initialize data on this level after regridding if old level did not + // previously exist + // + virtual void init() override; + + // + // Proceed with next timestep? + // + virtual int okToContinue() override; + + // + // Tell amrex::Amr to write a plotfile now + // + bool writePlotNow() override; + + // + // Tell Nyx to do analysis now + // + bool doAnalysisNow(); + + int updateInSitu(); + + // + // Advance grids at this level in time. + // + virtual amrex::Real advance(amrex::Real time, amrex::Real dt, int iteration, int ncycle) override; + + amrex::Real advance_hydro(amrex::Real time, amrex::Real dt, int iteration, int ncycle); + amrex::Real advance_heatcool(amrex::Real time, amrex::Real dt, int iteration, int ncycle); + amrex::Real advance_no_hydro(amrex::Real time, amrex::Real dt, int iteration, int ncycle); + amrex::Real advance_hydro_plus_particles(amrex::Real time, amrex::Real dt, int iteration, + int ncycle); + + void strang_hydro(amrex::Real time, amrex::Real dt, amrex::Real a_old, amrex::Real a_new); +#ifdef SDC + void sdc_hydro(amrex::Real time, amrex::Real dt, amrex::Real a_old, amrex::Real a_new); +#endif + + void correct_gsrc(int lev, amrex::Real time, amrex::Real prev_time, amrex::Real cur_time, + amrex::Real dt); + +/// +/// this constructs the hydrodynamic source (essentially the flux +/// divergence) using the CTU framework for unsplit hydrodynamics +/// +/// @param time current time +/// @param dt timestep +/// + void construct_ctu_hydro_source(amrex::Real time, amrex::Real dt, amrex::Real a_old, amrex::Real a_new, + amrex::MultiFab& S_border, amrex::MultiFab& D_border, + amrex::MultiFab& ext_src_old, amrex::MultiFab& hydro_src, + amrex::MultiFab& grav, + bool init_flux_register, bool add_to_flux_register); + +/// +/// Hydrodynamic (and radiation) fluxes. +/// + amrex::Vector > fluxes; +/// +/// Return the n'th fluxes MultiFab. +/// +/// @param dir direction in which to find the fluxes along +/// + amrex::MultiFab& Fluxes (int dir); + + amrex::Vector > mass_fluxes; + + void construct_hydro_source(const amrex::MultiFab& S, + amrex::MultiFab& sources_for_hydro, + amrex::MultiFab& hydro_source, + amrex::MultiFab& grav, + amrex::Real a_old, + amrex::Real a_new, + amrex::Real dt, + bool init_flux_register, bool add_to_flux_register); + + void update_state_with_sources( amrex::MultiFab& S_old , amrex::MultiFab& S_new, + amrex::MultiFab& ext_src_old, amrex::MultiFab& hydro_src, + amrex::MultiFab& grav , +#ifdef SDC + amrex::MultiFab& reset_e_src, +#endif + amrex::Real dt, amrex::Real a_old, amrex::Real a_new); + + void enforce_minimum_density ( amrex::MultiFab& S_old , amrex::MultiFab& S_new, +#ifdef SDC + amrex::MultiFab& hydro_source, + amrex::MultiFab& reset_e_src, +#endif + amrex::Real a_new); +#ifdef SDC + void enforce_minimum_density_cons ( amrex::MultiFab& S_old , amrex::MultiFab& S_new, amrex::MultiFab& reset_e_src); +#else + void enforce_minimum_density_cons ( amrex::MultiFab& S_old , amrex::MultiFab& S_new); +#endif + void enforce_minimum_density_floor ( amrex::MultiFab& S_new, amrex::Real a_new); + + void strang_first_step (amrex::Real time, amrex::Real dt, amrex::MultiFab& state, amrex::MultiFab& dstate); + void strang_second_step (amrex::Real time, amrex::Real dt, amrex::MultiFab& state, amrex::MultiFab& dstate); + + void sdc_reactions ( amrex::MultiFab& state_old, amrex::MultiFab& state_new, amrex::MultiFab& dstate, + amrex::MultiFab& hydro_src, amrex::MultiFab& IR, amrex::MultiFab& reset_e_src, + amrex::Real dt, amrex::Real a_old, amrex::Real a_new, + int sdc_iter); + + int integrate_state_vec(amrex::MultiFab &state, amrex::MultiFab &diag_eos, const amrex::Real& a, const amrex::Real& delta_time); + int integrate_state_grownvec(amrex::MultiFab &state, amrex::MultiFab &diag_eos, const amrex::Real& a, const amrex::Real& delta_time); + int integrate_state_vec_mfin(amrex::Array4const& state4, amrex::Array4const& diag_eos4,const amrex::Box& tbx, const amrex::Real& a, const amrex::Real& delta_time, long int& old_max_steps, long int& new_max_steps); + +int integrate_state_struct + (amrex::MultiFab &S_old, + amrex::MultiFab &S_new, + amrex::MultiFab &D_old, + amrex::MultiFab &hydro_src, + amrex::MultiFab &IR, + amrex::MultiFab &reset_src, + const amrex::Real& a, const amrex::Real& a_end, + const amrex::Real& delta_time, + const int sdc_iter); + + int integrate_state_struct_mfin + (amrex::Array4 const& state4, + amrex::Array4 const& diag_eos4, + amrex::Array4 const& state_n4, + amrex::Array4 const& hydro_src4, + amrex::Array4 const& reset_src4, + amrex::Array4 const& IR4, +#ifdef SAVE_REACT + amrex::Array4 const& react_in_arr, + amrex::Array4 const& react_out_arr, + amrex::Array4 const& react_out_work_arr, +#endif + const amrex::Box& tbx, + const amrex::Real& a, const amrex::Real& a_end, + const amrex::Real& delta_time, + long int& old_max_steps, long int& new_max_steps, + const int sdc_iter); + + int integrate_state_cell(amrex::MultiFab &state, amrex::MultiFab &diag_eos, const amrex::Real& a, const amrex::Real& delta_time); + int integrate_state_growncell(amrex::MultiFab &state, amrex::MultiFab &diag_eos, const amrex::Real& a, const amrex::Real& delta_time); + + amrex::Real advance_particles_only (amrex::Real time, amrex::Real dt, int iteration, int ncycle); + + void moveKickDriftExact(amrex::Real dt); + void moveKickExact(amrex::Real dt); + void time_center_source_terms(amrex::MultiFab& S_new, amrex::MultiFab& ext_src_old, + amrex::MultiFab& ext_src_new, amrex::Real dt); + + void conserved_to_primitive(amrex::MultiFab& state); + void primitive_to_conserved(amrex::MultiFab& state); + +#ifdef REEBER + void halo_find(amrex::Real dt, amrex::Vector& reeber_halos); + void halo_print(amrex::Vector& reeber_halos); + void SwapEnd(float& val); + void writeHaloBinaryVTK(const std::string& filename_vtk, + const amrex::Vector& reeber_halos, + const amrex::Real& radius_outer, + const amrex::Real& radius_inner); + void writeHaloSimpleBinary(const std::string& filename_bin, + const amrex::Vector& reeber_halos, + const amrex::Real& radius_outer, + const amrex::Real& radius_inner); + +#endif + void agn_halo_find(amrex::Real dt); + void agn_halo_merge(); + void agn_halo_accrete(amrex::Real dt); + +#ifdef REEBER + void runReeberAnalysis(amrex::Vector& new_state, + amrex::Vector >& particle_mf, + const amrex::Geometry Geom, const amrex::Vector& level_refinements, + int nStep, bool do_analysis, std::vector& reeber_halos); +#endif + + void Lya_statistics(); + + // + // Estimate time step. + // + amrex::Real est_time_step(amrex::Real dt_old); + + // + // Compute initial time step. + // + amrex::Real initial_time_step(); + + // + // Compute initial `dt'. + // + virtual void computeInitialDt(int finest_level, int sub_cycle, + amrex::Vector& n_cycle, + const amrex::Vector& ref_ratio, + amrex::Vector& dt_level, amrex::Real stop_time) override; + // + // Compute new `dt'. + // + virtual void computeNewDt(int finest_level, int sub_cycle, + amrex::Vector& n_cycle, + const amrex::Vector& ref_ratio, + amrex::Vector& dt_min, amrex::Vector& dt_level, + amrex::Real stop_time, int post_regrid_flag) override; + + // + // Print information about energy budget. + // + void do_energy_diagnostics(); + + // + // Do work after timestep(). + // + virtual void post_timestep(int iteration) override; + + // + // Contains operations to be done only after a full coarse timestep. + // + virtual void postCoarseTimeStep(amrex::Real cumtime) override; + + // + // Do work after `regrid()`. + // + virtual void post_regrid(int lbase, int new_finest) override; + + // + // Do work after a `restart()`. + // + virtual void post_restart() override; + + // + // Do work after `init()`. + // + virtual void post_init(amrex::Real stop_time) override; + + // + // Error estimation for regridding. + // + virtual void errorEst(amrex::TagBoxArray& tb, int clearval, int tagval, amrex::Real time, + int n_error_buf=0, int ngrow=0) override; + + // + // Error estimation for regridding. + // + virtual int WorkEstType () override { return 0; } + + // + // Called in grid_places after other tagging routines to modify + // the list of tagged points + // + virtual void manual_tags_placement (amrex::TagBoxArray& tags, + const amrex::Vector& bf_lev) override; + + // Returns a amrex::MultiFab containing the derived data for this level. The user + // is responsible for deleting this pointer when done with it. If + // `ngrow` > 0 the amrex::MultiFab is built on the appropriately grown amrex::BoxArray. + std::unique_ptr derive(const std::string& name, amrex::Real time, int ngrow) override; + + // This version of `derive()` fills the dcomp'th component of mf with the + // derived quantity. + void derive(const std::string& name, amrex::Real time, amrex::MultiFab& mf, int dcomp) override; + + static int Do_Hydro(); + +#ifndef CONST_SPECIES + void enforce_nonnegative_species(amrex::MultiFab& S_new); +#endif + void enforce_consistent_e(amrex::MultiFab& S); + + // Synchronize (rho e) and (rho E) so they are consistent with each other + void reset_internal_energy(amrex::MultiFab& State, amrex::MultiFab& DiagEOS, amrex::MultiFab& reset_e_src); + + // Synchronize (rho e) and (rho E) so they are consistent with each other without storing the change + void reset_internal_energy_nostore(amrex::MultiFab& State, amrex::MultiFab& DiagEOS); + + void reset_internal_energy_interp(amrex::MultiFab& State, amrex::MultiFab& DiagEOS, amrex::MultiFab& reset_e_src); + + // Note: this no longer includes the call to reset_internal_energy + void compute_new_temp(amrex::MultiFab& S_new, amrex::MultiFab& D_new); + + void compute_rho_temp(amrex::Real& rho_T_avg, amrex::Real& T_avg, amrex::Real& Tinv_avg, amrex::Real& T_meanrho); + void compute_gas_fractions(amrex::Real T_cut, amrex::Real rho_cut, + amrex::Real& whim_mass_frac, amrex::Real& whim_vol_frac, + amrex::Real& hh_mass_frac, amrex::Real& hh_vol_frac, + amrex::Real& igm_mass_frac, amrex::Real& igm_vol_frac); + + void get_old_source(amrex::Real old_time, amrex::Real dt, amrex::MultiFab& Rhs); + void get_new_source(amrex::Real old_time, amrex::Real new_time, amrex::Real dt, amrex::MultiFab& Rhs); + + amrex::Real vol_weight_sum (const std::string& name, amrex::Real time, bool masked); + amrex::Real vol_weight_sum (amrex::MultiFab& mf, bool masked); + amrex::Real vol_weight_squared_sum (const std::string& name, amrex::Real time); + amrex::Real vol_weight_squared_sum_level(const std::string& name, amrex::Real time); + +#ifdef AUX_UPDATE + void advance_aux(amrex::Real time, amrex::Real dt); +#endif +#ifdef AMREX_PARTICLES + static amrex::Vector& theActiveParticles(); + static amrex::Vector& theVirtualParticles(); + static amrex::Vector& theGhostParticles(); + + static DarkMatterParticleContainer* theDMPC(); + static DarkMatterParticleContainer* theVirtPC(); + static DarkMatterParticleContainer* theGhostPC(); + + static StellarParticleContainer* theSPC(); + static StellarParticleContainer* theVirtSPC(); + static StellarParticleContainer* theGhostSPC(); + +#ifdef AGN + static AGNParticleContainer* theAPC(); + static AGNParticleContainer* theVirtAPC(); + static AGNParticleContainer* theGhostAPC(); +#endif + +#ifdef NEUTRINO_PARTICLES + #ifdef NEUTRINO_DARK_PARTICLES + static DarkMatterParticleContainer* theNPC(); + static DarkMatterParticleContainer* theVirtNPC(); + static DarkMatterParticleContainer* theGhostNPC(); + #else + static NeutrinoParticleContainer* theNPC(); + static NeutrinoParticleContainer* theVirtNPC(); + static NeutrinoParticleContainer* theGhostNPC(); + #endif +#endif + static DarkMatterParticleContainer* theShellPC(); +#endif + + static int NUM_STATE; + + static int Zhi_comp; + + static int NumSpec; + static amrex::Real gamma; + static amrex::Real h_species; + static amrex::Real he_species; + + static int init_with_sph_particles; + + static std::string particle_plotfile_format; + + // Average value of gas, dark matter, neutrino and total density, computed every coarse timestep + static amrex::Real average_gas_density; + static amrex::Real average_dm_density; + static amrex::Real average_neutr_density; + static amrex::Real average_total_density; + + static amrex::Real tagging_base; + + // specifies the memory priority + static int reuse_mlpoisson; + + // + // This amrex::MultiFab is used for the level coarser than this level to mask out + // this level. We only build this when it is needed. + // This amrex::MultiFab has to live on this level even though it is at the resolution + // of the next coarser level because it must be updated whenever this level changes. + // + amrex::MultiFab* fine_mask; + amrex::MultiFab* build_fine_mask(); + + static void InitErrorList(); + static void InitDeriveList(); + + //! Get the level directory names + void LevelDirectoryNames (const std::string &dir, + const std::string &secondDir, // ---- probably DM or AGN + std::string &LevelDir, + std::string &FullPath); + //! Create the Level_ and other directories in checkpoint and plot files + virtual void CreateLevelDirectory (const std::string &dir) override; + +protected: + + static void read_params(); + + Nyx& get_level(int lev); + + std::string retrieveDM(); +#ifdef AGN + std::string retrieveAGN(); +#endif +#ifdef NEUTRINO_DARK_PARTICLES + std::string retrieveNPC(); +#endif + +#ifndef NO_HYDRO + amrex::FluxRegister& get_flux_reg(); + amrex::FluxRegister& get_flux_reg(int lev); + + void reflux(); + +#endif + + void average_down(); + void average_down(int state_indx); + + void build_metrics(); + +#ifndef NO_HYDRO + virtual void sum_integrated_quantities(); + + void compute_average_density(); + void compute_average_temperature(amrex::Real& average_temperature); + void compute_average_species(amrex::Vector& average_species); + + void set_small_values(); + +#endif + + void set_small_values_given_average (amrex::Real average_dens, amrex::Real average_temp, amrex::Real a, + amrex::Real & small_dens_inout, amrex::Real & small_temp_inout, + amrex::Real &small_pres_inout, amrex::Real gamma_minus_1, + amrex::Real h_species); + + void write_info(); + +#ifndef NO_HYDRO + amrex::FluxRegister* flux_reg; +#endif + + // + // Static data members. + // + static bool dump_old; + static int verbose; + static amrex::Real cfl; + static amrex::Real init_shrink; + static amrex::Real change_max; + static int do_reflux; + static amrex::ErrorList err_list; + static amrex::Vector errtags; + static amrex::BCRec phys_bc; + + static int nsteps_from_plotfile; + + static int nghost_state; + static int ppm_type; + static int strang_grown_box; + + static std::string enforce_min_density_type; + + static int num_particle_ghosts; + + static amrex::Real small_dens; + static amrex::Real small_pres; + static amrex::Real small_temp; + static amrex::Real small; + + static amrex::Real large_temp; + + static bool do_dm_particles; + + // How do we want to initialize the particles? + // Must be "Random", "Cosmological" or "AsciiFile" + static std::string particle_init_type; + + // These control random initialization + static bool particle_initrandom_serialize; + static long particle_initrandom_count; + static long particle_initrandom_count_per_box; + static amrex::Real particle_initrandom_mass; + static amrex::Real particle_initrandom_mass_total; + static int particle_initrandom_iseed; + static int particle_skip_factor; + static amrex::Real particle_inituniform_mass; + static amrex::Real particle_inituniform_vx; + static amrex::Real particle_inituniform_vy; + static amrex::Real particle_inituniform_vz; + + // This controls the LaunchSafeGuard for particle initialization + static int particle_launch_ics; + + static amrex::Vector plot_z_values; // These are the value of "z" at which to dump plotfiles. + static amrex::Vector analysis_z_values; // These are the value of "z" at which to perform analysis + static int insitu_int; + static int insitu_start; + + static int load_balance_int; + static amrex::Real load_balance_start_z; + static int load_balance_wgt_strategy; + static int load_balance_wgt_nmax; + static int load_balance_strategy; + + bool FillPatchedOldState_ok; + + // permits hydro to be turned on and off for running pure rad problems: + static int do_hydro; + + // permits gravity calculation to be turned on and off + static int do_grav; + + // if true, define an additional source term + static int add_ext_src; + + // specifies the heating/cooling source term + static int heat_cool_type; + + // specifies the reduction type for snudials vectors + static int sundials_atomic_reductions; + + // specifies the allocation type for sundials vector + static int sundials_alloc_type; + + // specifies the allocation type for sundials vector + static int use_typical_steps; + + // if true, use non-negative constraint on energy + static int use_sundials_constraint; + + // if true, use non-negative constraint on energy + static int use_sundials_fused; + + // specifies whether we allow tiling when calling the SUNDIALS integrator + static bool sundials_use_tiling; + + // specifies tolerances when calling the SUNDIALS integrator + static amrex::Real sundials_reltol; + static amrex::Real sundials_abstol; + + // specifies tile_size of advance_hydro for unsplit Godonuv hydro advance + static amrex::IntVect hydro_tile_size; + + // specifies tile_size of integrate_state for heating-cooling reactions + static amrex::IntVect sundials_tile_size; + + // specifies the memory priority + static int minimize_memory; + + // specifies the ShrinkToFit priority + static int shrink_to_fit; + + // specifies inhomogeneous reionization type + static int inhomo_reion; + static std::string inhomo_zhi_file; + static int inhomo_grid; + + // permits forcing to be switched on and off + static int do_forcing; + + // if true , incorporate the source term through Strang-splitting + static int strang_split; + +#ifdef SDC + // if true , use SDC to couple hydro and reactions + static int sdc_split; + static int strang_restart_from_sdc; +#endif + + // There can be only one Gravity object, it covers all levels: + static class Gravity *gravity; + + // There can be only one forcing object (Fourier space): + static class StochasticForcing *forcing; + + static amrex::Real lightcone_start_z; + static amrex::Real lightcone_end_z; +#ifdef REEBER + // + // Threshold for halo to create SMBH + // + static amrex::Real mass_halo_min; + + // + // Seed mass of SMBH + // + static amrex::Real mass_seed; + + static int min_halo_n_cells; + static amrex::Real halo_component_threshold; + static amrex::Real halo_extrema_threshold; + +#endif + + // Previous maximum number of steps for sundials + static long int old_max_sundials_steps; + static long int new_max_sundials_steps; + + // for keeping track of the amount of CPU time used -- this will persist + // after restarts + static amrex::Real previousCPUTimeUsed; + static amrex::Real startCPUTime; + + static amrex::Real getCPUTime(); + +}; + +#ifndef NO_HYDRO +#include +#endif + +// time step interval for finding halos +extern int reeber_int; +// time step interval for doing Gimlet post-processing +extern int gimlet_int; + +// +// Inlines. +// +inline +int +Nyx::Do_Hydro() +{ + return do_hydro; +} + +inline +Nyx& +Nyx::get_level(int my_level) +{ + return *(Nyx *) &parent->getLevel(my_level); +} + +#ifndef NO_HYDRO +inline +amrex::FluxRegister& +Nyx::get_flux_reg() +{ + BL_ASSERT(flux_reg); + return *flux_reg; +} + +inline +amrex::FluxRegister& +Nyx::get_flux_reg(int my_level) +{ + return get_level(my_level).get_flux_reg(); +} +#endif // NO_HYDRO + +#endif /*_Nyx_H_*/ diff --git a/Exec/HaloFinder_GPU/Nyx.cpp b/Exec/HaloFinder_GPU/Nyx.cpp new file mode 100644 index 000000000..f3b9613b4 --- /dev/null +++ b/Exec/HaloFinder_GPU/Nyx.cpp @@ -0,0 +1,2924 @@ +#include +#include +#include +#include +#include + +using std::cout; +using std::cerr; +using std::endl; +using std::istream; +using std::ostream; +using std::pair; +using std::string; + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef BL_USE_MPI +#include +#endif + +#ifndef NO_HYDRO +#include +#include +#endif + +#include + +#ifdef GIMLET +#include +#include +#endif + +#ifdef AGN +#include +#endif + +using namespace amrex; + +extern "C" { + int get_comp_urho(); + int get_comp_temp(); + int get_comp_e_int(); +} + +const int NyxHaloFinderSignal = 42; +const int GimletSignal = 55; + +static int sum_interval = -1; +static int max_temp_dt = -1; + +static Real fixed_dt = -1.0; +static Real initial_dt = -1.0; +static Real dt_cutoff = 0; + +Real Nyx::old_a = -1.0; +Real Nyx::new_a = -1.0; +Real Nyx::old_a_time = -1.0; +Real Nyx::new_a_time = -1.0; + +Vector Nyx::plot_z_values; +Vector Nyx::analysis_z_values; +int Nyx::insitu_start = 0; +int Nyx::insitu_int = 0; + +int Nyx::load_balance_int = -1; +amrex::Real Nyx::load_balance_start_z = 15; +int Nyx::load_balance_wgt_strategy = 0; +int Nyx::load_balance_wgt_nmax = -1; +int Nyx::load_balance_strategy = DistributionMapping::SFC; + +bool Nyx::dump_old = false; +int Nyx::verbose = 0; + +Real Nyx::cfl = 0.8; +Real Nyx::init_shrink = 1.0; +Real Nyx::change_max = 1.1; + +BCRec Nyx::phys_bc; +int Nyx::do_reflux = 1; +int Nyx::NUM_STATE = -1; + +int Nyx::nsteps_from_plotfile = -1; + +ErrorList Nyx::err_list; +Vector Nyx::errtags; + +int Nyx:: Zhi_comp = -1; + +int Nyx::NumSpec = 0; + +Real Nyx::small_dens = -1.e200; +Real Nyx::small_temp = -1.e200; +Real Nyx::small_pres = -1.e200; +Real Nyx::large_temp = 1.e9; +Real Nyx::gamma = 5.0/3.0; + +// This is no longer an optional input; +// we use the value hard-wired here +Real Nyx::small = 1.e-6; + +Real Nyx::comoving_OmB; +Real Nyx::comoving_OmM; +Real Nyx::comoving_OmR = 0.0; +Real Nyx::comoving_h; +int Nyx::comoving_type = 1; + +int Nyx::do_hydro = 0; +int Nyx::add_ext_src = 0; +int Nyx::heat_cool_type = 0; +int Nyx::use_sundials_constraint = 0; +int Nyx::use_sundials_fused = 0; +int Nyx::use_typical_steps = 0; +#ifndef AMREX_USE_GPU +int Nyx::sundials_alloc_type = 0; +int Nyx::sundials_atomic_reductions = -1; // CUDA and HIP only +#else +#ifdef AMREX_USE_CUDA +int Nyx::sundials_atomic_reductions = 1; +#ifndef _OPENMP +int Nyx::sundials_alloc_type = 0; //consider changing to 5 +#else +int Nyx::sundials_alloc_type = 5; +#endif +#endif +#ifdef AMREX_USE_HIP +int Nyx::sundials_atomic_reductions = 0; +int Nyx::sundials_alloc_type = 5; +#endif +#ifdef AMREX_USE_DPCPP +int Nyx::sundials_atomic_reductions = -1; // CUDA and HIP only +int Nyx::sundials_alloc_type = 5; +#endif +#endif + +Real Nyx::sundials_reltol = 1e-4; +Real Nyx::sundials_abstol = 1e-4; + +int Nyx::minimize_memory = 0; +int Nyx::shrink_to_fit = 0; + +bool Nyx::sundials_use_tiling = true; + +#ifndef AMREX_USE_GPU +IntVect Nyx::hydro_tile_size(1024,16,16); +#else +IntVect Nyx::hydro_tile_size(1048576,1048576,1048576); +#endif + +#ifndef AMREX_USE_GPU +IntVect Nyx::sundials_tile_size(1024,16,16); +#else +#ifdef AMREX_USE_OMP +IntVect Nyx::sundials_tile_size(1024,16,16); +#else +IntVect Nyx::sundials_tile_size(1048576,1048576,1048576); +#endif +#endif + +int Nyx::strang_split = 1; +int Nyx::strang_grown_box = 1; +#ifdef SDC +int Nyx::sdc_split = 0; +int Nyx::strang_restart_from_sdc = 0; +#endif + +Real Nyx::average_gas_density = 0.; +Real Nyx::average_dm_density = 0.; +Real Nyx::average_neutr_density = 0.; +Real Nyx::average_total_density = 0.; + +long int Nyx::old_max_sundials_steps = 3; +long int Nyx::new_max_sundials_steps = 3; + +int Nyx::inhomo_reion = 0; +std::string Nyx::inhomo_zhi_file = ""; +int Nyx::inhomo_grid = -1; + +Gravity* Nyx::gravity = 0; +int Nyx::do_grav = -1; + +#ifndef NO_HYDRO +StochasticForcing* Nyx::forcing = 0; +#endif +int Nyx::do_forcing = 0; + +int Nyx::nghost_state = 1; +Real Nyx::tagging_base = 8.0; +int Nyx::reuse_mlpoisson = 0; +int Nyx::ppm_type = 1; + +// Options are "floor" or "conservative" +std::string Nyx::enforce_min_density_type = "floor"; + +Real Nyx:: h_species = 0.76; +Real Nyx::he_species = 0.24; + +amrex::Real Nyx::lightcone_start_z = 7; +amrex::Real Nyx::lightcone_end_z = 0; +#ifdef REEBER +Real Nyx::mass_halo_min = 1.e10; +Real Nyx::mass_seed = 1.e5; +Real Nyx::halo_component_threshold = 81.66; +Real Nyx::halo_extrema_threshold = 160.0; +int Nyx::min_halo_n_cells = 10; +#endif + +#ifdef _OPENMP +#include +#endif + +// The default for how many digits to use for each column in the runlog +// We have a second parameter (rlp_terse) for regression-testing those quantities +// which show more variation than others +int Nyx::runlog_precision = 6; +int Nyx::runlog_precision_terse = 6; + +int Nyx::write_parameters_in_plotfile = true; +int Nyx::write_skip_prepost = 0; +int Nyx::write_hdf5 = 0; + +// Do we use separate SPH particles to initialize +// the density and momentum on the grid? +int Nyx::init_with_sph_particles = 0; + +// Do we write the particles in single (IEEE32) +// or doublue (NATIVE) precision? +#ifdef BL_SINGLE_PRECISION_PARTICLES +std::string Nyx::particle_plotfile_format = "IEEE32"; +#else +std::string Nyx::particle_plotfile_format = "NATIVE"; +#endif + +#ifndef NO_HYDRO +#ifndef HEATCOOL +AtomicRates* atomic_rates_glob; +#endif +#endif + +// this will be reset upon restart +Real Nyx::previousCPUTimeUsed = 0.0; + +Real Nyx::startCPUTime = 0.0; + +int reeber_int(0); +int gimlet_int(0); + +// Note: Nyx::variableSetUp is in Nyx_setup.cpp +void +Nyx::variable_cleanup () +{ +if (do_grav) +{ + if (gravity != 0) + { + if (verbose > 1 && ParallelDescriptor::IOProcessor()) + std::cout << "Deleting gravity in variable_cleanup...\n"; + delete gravity; + gravity = 0; + } +} + +#ifndef NO_HYDRO + if (forcing != 0) + { + if (verbose > 1 && ParallelDescriptor::IOProcessor()) + std::cout << "Deleting forcing in variable_cleanup...\n"; + delete forcing; + forcing = 0; + } +#endif + + desc_lst.clear(); +} + +void +Nyx::read_params () +{ + BL_PROFILE("Nyx::read_params()"); + + ParmParse pp_nyx("nyx"); + + pp_nyx.query("v", verbose); + pp_nyx.get("init_shrink", init_shrink); + pp_nyx.get("cfl", cfl); + pp_nyx.query("change_max", change_max); + pp_nyx.query("fixed_dt", fixed_dt); + pp_nyx.query("initial_dt", initial_dt); + pp_nyx.query("max_temp_dt", max_temp_dt); + pp_nyx.query("sum_interval", sum_interval); + pp_nyx.get("dt_cutoff", dt_cutoff); + + // Get boundary conditions + Vector lo_bc(AMREX_SPACEDIM), hi_bc(AMREX_SPACEDIM); + pp_nyx.getarr("lo_bc", lo_bc, 0, AMREX_SPACEDIM); + pp_nyx.getarr("hi_bc", hi_bc, 0, AMREX_SPACEDIM); + for (int i = 0; i < AMREX_SPACEDIM; i++) + { + phys_bc.setLo(i, lo_bc[i]); + phys_bc.setHi(i, hi_bc[i]); + } + + // + // Check phys_bc against possible periodic geometry + // if periodic, must have internal BC marked. + // + if (DefaultGeometry().isAnyPeriodic() || (!do_dm_particles && !do_hydro)) + { + // + // Do idiot check. Periodic means interior in those directions. + // + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + if (DefaultGeometry().isPeriodic(dir)) + { + if (lo_bc[dir] != amrex::PhysBCType::interior) + { + std::cerr << "Nyx::read_params:periodic in direction " + << dir + << " but low BC is not Interior" << std::endl; + amrex::Error(); + } + if (hi_bc[dir] != amrex::PhysBCType::interior) + { + std::cerr << "Nyx::read_params:periodic in direction " + << dir + << " but high BC is not Interior" << std::endl; + amrex::Error(); + } + } + } + } + else + { + // + // Do idiot check. If not periodic, should be no interior. + // + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + if (lo_bc[dir] == amrex::PhysBCType::interior) + { + std::cerr << "Nyx::read_params:interior bc in direction " + << dir + << " but not periodic" << std::endl; + amrex::Error(); + } + if (hi_bc[dir] == amrex::PhysBCType::interior) + { + std::cerr << "Nyx::read_params:interior bc in direction " + << dir + << " but not periodic" << std::endl; + amrex::Error(); + } + } + } + + read_comoving_params(); + +#ifdef AMREX_PARTICLES + read_particle_params(); +#endif + + if (do_grav) + pp_nyx.get("do_grav", do_grav); + +#ifndef NO_HYDRO + read_hydro_params(); +#else + if (!do_grav) + amrex::Error("Dont know what to do with both hydro and gravity off"); +#endif + if (do_dm_particles || do_hydro) + read_init_params(); + + pp_nyx.query("runlog_precision",runlog_precision); + pp_nyx.query("runlog_precision_terse",runlog_precision_terse); + + pp_nyx.query("write_parameter_file",write_parameters_in_plotfile); + if(pp_nyx.query("write_hdf5",write_hdf5)) + write_skip_prepost = write_hdf5; + else + pp_nyx.query("write_skip_prepost",write_skip_prepost); + +#ifndef AMREX_USE_HDF5 + if(write_hdf5 == 1) + amrex::Error("Must compile with USE_HDF5 = TRUE for write_hdf5 = 1"); +#endif + +#ifdef AMREX_USE_HDF5_ASYNC + // Complete all previous async writes on amrex::Finalize() + amrex::ExecOnFinalize(H5VLasync_waitall); +#endif + + if (pp_nyx.contains("plot_z_values")) + { + int num_z_values = pp_nyx.countval("plot_z_values"); + plot_z_values.resize(num_z_values); + pp_nyx.queryarr("plot_z_values",plot_z_values,0,num_z_values); + } + + if (pp_nyx.contains("analysis_z_values")) + { + int num_z_values = pp_nyx.countval("analysis_z_values"); + analysis_z_values.resize(num_z_values); + pp_nyx.queryarr("analysis_z_values",analysis_z_values,0,num_z_values); + } + + ParmParse pp_insitu("insitu"); + pp_insitu.query("int", insitu_int); + pp_insitu.query("start", insitu_start); + pp_insitu.query("reeber_int", reeber_int); + + pp_nyx.query("load_balance_int", load_balance_int); + pp_nyx.query("load_balance_start_z", load_balance_start_z); + pp_nyx.query("load_balance_wgt_strategy", load_balance_wgt_strategy); + load_balance_wgt_nmax = amrex::ParallelDescriptor::NProcs(); + pp_nyx.query("load_balance_wgt_nmax", load_balance_wgt_nmax); + + std::string theStrategy; + + if (pp_nyx.query("load_balance_strategy", theStrategy)) + { + if (theStrategy == "SFC") + { + load_balance_strategy=DistributionMapping::Strategy::SFC; + } + else if (theStrategy == "KNAPSACK") + { + load_balance_strategy=DistributionMapping::Strategy::KNAPSACK; + } + else if (theStrategy == "ROUNDROBIN") + { + load_balance_strategy=DistributionMapping::Strategy::ROUNDROBIN; + } + else + { + std::string msg("Unknown strategy: "); + msg += theStrategy; + amrex::Warning(msg.c_str()); + } + } + + pp_nyx.query("gimlet_int", gimlet_int); + pp_nyx.query("lightcone_start_z", lightcone_start_z); + pp_nyx.query("lightcone_end_z", lightcone_end_z); +#ifdef REEBER + pp_nyx.query("mass_halo_min", mass_halo_min); + pp_nyx.query("mass_seed", mass_seed); + ParmParse pp_reeber("reeber"); + pp_reeber.query("min_halo_n_cells", min_halo_n_cells); + pp_reeber.query("halo_component_threshold", halo_component_threshold); + pp_reeber.query("halo_extrema_threshold", halo_extrema_threshold); +#endif +} + +#ifndef NO_HYDRO +void +Nyx::read_hydro_params () +{ + ParmParse pp_nyx("nyx"); + + pp_nyx.get("do_hydro", do_hydro); + + pp_nyx.query("small_dens", small_dens); + pp_nyx.query("small_temp", small_temp); + pp_nyx.query("small_pres", small_pres); + pp_nyx.query("large_temp", large_temp); + pp_nyx.query("gamma", gamma); + +#ifndef NO_HYDRO +#ifdef HEATCOOL + atomic_rates_glob = (AtomicRates*)The_Arena()->alloc(sizeof(AtomicRates)); +#else + atomic_rates_glob = NULL; +#endif +#endif + + pp_nyx.query("add_ext_src", add_ext_src); + pp_nyx.query("strang_split", strang_split); + pp_nyx.query("strang_grown_box", strang_grown_box); + +#ifdef HEATCOOL +#ifdef SDC + pp_nyx.query("sdc_split", sdc_split); + pp_nyx.query("strang_restart_from_sdc", strang_restart_from_sdc); + if (sdc_split == 1 && strang_split == 1) + amrex::Error("Cant have strang_split == 1 and sdc_split == 1"); + if (sdc_split == 0 && strang_split == 0) + amrex::Error("Cant have strang_split == 0 and sdc_split == 0"); + if (sdc_split != 1 && strang_split != 1) + amrex::Error("Cant have strang_split != 1 and sdc_split != 1"); +#else + if (strang_split != 1) + amrex::Error("Cant have strang_split != 1 with USE_SDC != TRUE"); +#endif +#endif + + pp_nyx.query("do_forcing", do_forcing); + if (do_forcing == 1 && add_ext_src == 0) + amrex::Error("Nyx::must set add_ext_src to 1 if do_forcing = 1 "); + + pp_nyx.query("heat_cool_type", heat_cool_type); + pp_nyx.query("inhomo_reion", inhomo_reion); + + if (inhomo_reion) { + pp_nyx.get("inhomo_zhi_file", inhomo_zhi_file); + pp_nyx.get("inhomo_grid", inhomo_grid); + } + +#ifdef HEATCOOL + if (ParallelDescriptor::IOProcessor()) { + std::cout << "Integrating heating/cooling method with the following method: "; + switch (heat_cool_type) + { + case 11: + std::cout << "Vectorized CVODE"; + break; + } + std::cout << std::endl; + } + +#else + if (inhomo_reion > 0) + amrex::Error("Nyx::you set inhomo_reion > 0 but forgot to set USE_HEATCOOL = TRUE"); +#endif // HEATCOOL + + pp_nyx.query("use_sundials_constraint", use_sundials_constraint); + pp_nyx.query("use_sundials_fused", use_sundials_fused); + pp_nyx.query("nghost_state", nghost_state); + pp_nyx.query("sundials_atomic_reductions", sundials_atomic_reductions); + pp_nyx.query("sundials_alloc_type", sundials_alloc_type); + pp_nyx.query("sundials_reltol", sundials_reltol); + pp_nyx.query("sundials_abstol", sundials_abstol); + pp_nyx.query("minimize_memory", minimize_memory); + pp_nyx.query("shrink_to_fit", shrink_to_fit); + pp_nyx.query("use_typical_steps", use_typical_steps); + pp_nyx.query("tagging_base", tagging_base); + pp_nyx.query("reuse_mlpoisson", reuse_mlpoisson); + pp_nyx.query("ppm_type", ppm_type); + pp_nyx.query("enforce_min_density_type", enforce_min_density_type); + + pp_nyx.query("sundials_use_tiling", sundials_use_tiling); + + Vector tilesize(AMREX_SPACEDIM); + if (pp_nyx.queryarr("hydro_tile_size", tilesize, 0, AMREX_SPACEDIM)) + { + for (int i=0; i 0 && do_reflux) + flux_reg = new FluxRegister(grids, dmap, crse_ratio, level, NUM_STATE); + } +#endif + + // Initialize to zero here in case we run with do_grav = false. + MultiFab& new_grav_mf = get_new_data(Gravity_Type); + new_grav_mf.setVal(0); + + MultiFab& new_phi_grav = get_new_data(PhiGrav_Type); + new_phi_grav.setVal(0); + + if (do_grav) + { + // gravity is a static object, only alloc if not already there + if (gravity == 0) { + gravity = new Gravity(parent, parent->finestLevel(), &phys_bc, 0); + } + + gravity->install_level(level, this); + } + +#ifndef NO_HYDRO + if (do_forcing) + { + // forcing is a static object, only alloc if not already there + if (forcing == 0) + forcing = new StochasticForcing(); + + const Real* prob_lo = geom.ProbLo(); + const Real* prob_hi = geom.ProbHi(); + + forcing->init(AMREX_SPACEDIM, prob_lo, prob_hi); + } +#endif + + // Initialize the "a" variable + if (level == 0 && time == 0.0 && old_a_time < 0.) + { + old_a_time = 0.0; + new_a_time = 0.0; + + old_a = 1.0 / (1.0 + initial_z); + new_a = old_a; + } +} + +Nyx::~Nyx () +{ +#ifndef NO_HYDRO + if (do_hydro == 1) + delete flux_reg; +#endif + delete fine_mask; +} + +void +Nyx::restart (Amr& papa, + istream& is, + bool b_read_special) +{ + BL_PROFILE("Nyx::restart()"); + AmrLevel::restart(papa, is, b_read_special); + + build_metrics(); + + + // get the elapsed CPU time to now; + if (level == 0 && ParallelDescriptor::IOProcessor()) + { + // get elapsed CPU time + std::ifstream CPUFile; + std::string FullPathCPUFile = parent->theRestartFile(); + FullPathCPUFile += "/CPUtime"; + CPUFile.open(FullPathCPUFile.c_str(), std::ios::in); + + CPUFile >> previousCPUTimeUsed; + CPUFile.close(); + + std::cout << "read CPU time: " << previousCPUTimeUsed << "\n"; + } + +#ifndef NO_HYDRO + if (do_hydro == 1) + { + BL_ASSERT(flux_reg == 0); + if (level > 0 && do_reflux) + flux_reg = new FluxRegister(grids, dmap, crse_ratio, level, NUM_STATE); + } +#endif + + if (do_grav && level == 0) + { + BL_ASSERT(gravity == 0); + gravity = new Gravity(parent, parent->finestLevel(), &phys_bc, 0); + } + +#ifndef NO_HYDRO + if (do_forcing) + { + // forcing is a static object, only alloc if not already there + if (forcing == 0) + forcing = new StochasticForcing(); + + const Real* prob_lo = geom.ProbLo(); + const Real* prob_hi = geom.ProbHi(); + + forcing->init(AMREX_SPACEDIM, prob_lo, prob_hi); + } +#endif + +} + +void +Nyx::build_metrics () +{ +} + +void +Nyx::setTimeLevel (Real time, + Real dt_old, + Real dt_new) +{ + if (verbose && ParallelDescriptor::IOProcessor()) { + std::cout << "Setting the current time in the state data to " + << parent->cumTime() << std::endl; + } + AmrLevel::setTimeLevel(time, dt_old, dt_new); +} + +void +Nyx::init (AmrLevel& old) +{ + BL_PROFILE("Nyx::init(old)"); + + MultiFab::RegionTag amrInit_tag("Init_" + std::to_string(level)); + Nyx* old_level = (Nyx*) &old; + // + // Create new grid data by fillpatching from old. + // + Real dt_new = parent->dtLevel(level); + +#ifndef NO_HYDRO + Real cur_time = old_level->state[State_Type].curTime(); + Real prev_time = old_level->state[State_Type].prevTime(); +#else + Real cur_time = old_level->state[PhiGrav_Type].curTime(); + Real prev_time = old_level->state[PhiGrav_Type].prevTime(); +#endif + + Real dt_old = cur_time - prev_time; + setTimeLevel(cur_time, dt_old, dt_new); + +#ifndef NO_HYDRO + if (do_hydro == 1) + { + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + FillPatch(old, S_new, 0, cur_time, State_Type, 0, NUM_STATE); + FillPatch(old, D_new, 0, cur_time, DiagEOS_Type, 0, D_new.nComp()); + + MultiFab reset_e_src(S_new.boxArray(), S_new.DistributionMap(), 1, NUM_GROW); + reset_e_src.setVal(0.0); + reset_internal_energy_interp(S_new,D_new,reset_e_src); + + } +#endif + + MultiFab& Phi_new = get_new_data(PhiGrav_Type); + if (do_grav) + { + FillPatch(old, Phi_new, 0, cur_time, PhiGrav_Type, 0, 1); + } else { + // We need to initialize it otherwise we might write out NaNs in the checkpoint + Phi_new.setVal(0.); + } + +#ifdef SDC + MultiFab& IR_new = get_new_data(SDC_IR_Type); + FillPatch(old, IR_new, 0, cur_time, SDC_IR_Type, 0, 1); +#endif + + amrex::Gpu::Device::streamSynchronize(); + +} + +// +// This version inits the data on a new level that did not +// exist before regridding. +// +void +Nyx::init () +{ + BL_PROFILE("Nyx::init()"); + Real dt = parent->dtLevel(level); + +#ifndef NO_HYDRO + Real cur_time = get_level(level-1).state[State_Type].curTime(); + Real prev_time = get_level(level-1).state[State_Type].prevTime(); +#else + Real cur_time = get_level(level-1).state[PhiGrav_Type].curTime(); + Real prev_time = get_level(level-1).state[PhiGrav_Type].prevTime(); +#endif + + Real dt_old = (cur_time - prev_time) / (Real)parent->MaxRefRatio(level-1); + + setTimeLevel(cur_time, dt_old, dt); + +#ifndef NO_HYDRO + + if(do_hydro) + { + + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + FillCoarsePatch(S_new, 0, cur_time, State_Type, 0, S_new.nComp()); + FillCoarsePatch(D_new, 0, cur_time, DiagEOS_Type, 0, D_new.nComp()); + + Real rho_E = 0; + Real rho_e = 0; + + for (int lev = 0; lev <= 0; lev++) + // for (int lev = 0; lev <= finest_level; lev++) + { + Nyx& nyx_lev = get_level(lev); + + rho_E += nyx_lev.vol_weight_sum("rho_E", prev_time, true); + rho_e += nyx_lev.vol_weight_sum("rho_e", prev_time, true); + } + amrex::Print()<<"total RHO*E "<setDtLevel(1.e100, level); +} + +Real +Nyx::initial_time_step () +{ + BL_PROFILE("Nyx::initial_time_step()"); + Real dummy_dt = 0; + Real init_dt = 0; + + if (initial_dt > 0) + { + init_dt = initial_dt; + } + else + { + init_dt = init_shrink * est_time_step(dummy_dt); + } + + bool dt_changed = false; + if (level == 0 && plot_z_values.size() > 0) + plot_z_est_time_step(init_dt,dt_changed); + + if (level == 0 && analysis_z_values.size() > 0) + analysis_z_est_time_step(init_dt,dt_changed); + + return init_dt; +} + +Real +Nyx::est_time_step (Real /*dt_old*/) +{ + BL_PROFILE("Nyx::est_time_step()"); + if (fixed_dt > 0) + return fixed_dt; + + // This is just a dummy value to start with + Real est_dt = 1.0e+200; + +#ifndef NO_HYDRO + const MultiFab& stateMF = get_new_data(State_Type); +#endif + +#ifndef NO_HYDRO + Real cur_time = state[State_Type].curTime(); +#else + Real cur_time = state[PhiGrav_Type].curTime(); +#endif + +#ifndef NO_HYDRO + if (do_hydro) + { + Real a = get_comoving_a(cur_time); + const auto dx = geom.CellSizeArray(); + + { + Real dt = 1.e200; + Real sound_speed_factor=sqrt(gamma*(gamma-1)); + + Real local_small_dens = small_dens; + int local_max_temp_dt = max_temp_dt; + + dt = amrex::ReduceMin(stateMF, 0, + [=] AMREX_GPU_HOST_DEVICE (Box const& bx, Array4 const& u) -> Real + { + const auto lo = amrex::lbound(bx); + const auto hi = amrex::ubound(bx); +#if !defined(__CUDACC__) || (__CUDACC_VER_MAJOR__ != 9) || (__CUDACC_VER_MINOR__ != 2) + amrex::Real dt_gpu = std::numeric_limits::max(); +#else + amrex::Real dt_gpu = 1.e37; +#endif + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + if(u(i,j,k,Density_comp)<=1.1*local_small_dens && local_max_temp_dt==1) + continue; + + Real rhoInv = 1.0 / u(i,j,k,Density_comp); + Real ux = u(i,j,k,Xmom_comp)*rhoInv; + Real uy = u(i,j,k,Ymom_comp)*rhoInv; + Real uz = u(i,j,k,Zmom_comp)*rhoInv; + + // Use internal energy for calculating dt + Real e = u(i,j,k,Eint_comp)*rhoInv; + + Real c; + // Protect against negative e +#ifdef HEATCOOL + if (e > 0.0) + c=sound_speed_factor*std::sqrt(e); +#else + if (e > 0.0) + c=sound_speed_factor*std::sqrt(u(i,j,k,Density_comp)*e/u(i,j,k,Density_comp)); +#endif + else + c = 0.0; + + Real dt1 = dx[0]/(c + amrex::Math::abs(ux)); + Real dt2 = dx[1]/(c + amrex::Math::abs(uy)); + Real dt3 = dx[2]/(c + amrex::Math::abs(uz)); + dt_gpu = amrex::min(dt_gpu,amrex::min(dt1,amrex::min(dt2,dt3))); + } + } + } + return dt_gpu; + }); + + est_dt = std::min(est_dt, dt); + + } + + // If in comoving coordinates, then scale dt (based on u and c) by a + est_dt *= a; + + ParallelDescriptor::ReduceRealMin(est_dt); + est_dt *= cfl; + if (verbose && ParallelDescriptor::IOProcessor()) + std::cout << "...estdt from hydro at level " + << level << ": " + << est_dt << '\n'; + } +#endif + + if (do_grav) + particle_est_time_step(est_dt); + + if (level==0) + comoving_est_time_step(cur_time,est_dt); + + if (verbose && ParallelDescriptor::IOProcessor()) + std::cout << "Nyx::est_time_step at level " + << level + << ": estdt = " + << est_dt << '\n'; + + return est_dt; +} + +void +Nyx::computeNewDt (int finest_level, + int /*sub_cycle*/, + Vector& n_cycle, + const Vector& /*ref_ratio*/, + Vector& dt_min, + Vector& dt_level, + Real stop_time, + int post_regrid_flag) +{ + BL_PROFILE("Nyx::computeNewDt()"); + // + // We are at the start of a coarse grid timecycle. + // Compute the timesteps for the next iteration. + // + if (level > 0) + return; + + int i; + + Real dt_0 = 1.0e+100; + int n_factor = 1; + for (i = 0; i <= finest_level; i++) + { + Nyx& adv_level = get_level(i); + dt_min[i] = adv_level.est_time_step(dt_level[i]); + } + + if (fixed_dt <= 0.0) + { + if (post_regrid_flag == 1) + { + // + // Limit dt's by pre-regrid dt + // + for (i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i], dt_level[i]); + } + // + // Find the minimum over all levels + // + for (i = 0; i <= finest_level; i++) + { + n_factor *= n_cycle[i]; + dt_0 = std::min(dt_0, n_factor * dt_min[i]); + } + } + else + { + bool sub_unchanged=true; + if ((parent->maxLevel() > 0) && (level == 0) && + (parent->subcyclingMode() == "Optimal") && + (parent->okToRegrid(level) || parent->levelSteps(0) == 0) ) + { + Vector new_cycle(finest_level+1); + for (i = 0; i <= finest_level; i++) + new_cycle[i] = n_cycle[i]; + // The max allowable dt + Vector dt_max(finest_level+1); + for (i = 0; i <= finest_level; i++) + { + dt_max[i] = dt_min[i]; + } + // find the maximum number of cycles allowed. + Vector cycle_max(finest_level+1); + cycle_max[0] = 1; + for (i = 1; i <= finest_level; i++) + { + cycle_max[i] = parent->MaxRefRatio(i-1); + } + // estimate the amout of work to advance each level. + Vector est_work(finest_level+1); + for (i = 0; i <= finest_level; i++) + { + est_work[i] = parent->getLevel(i).estimateWork(); + } + + // This value will be used only if the subcycling pattern is changed. + + dt_0 = parent->computeOptimalSubcycling(finest_level+1, new_cycle.dataPtr(), dt_max.dataPtr(), + est_work.dataPtr(), cycle_max.dataPtr()); + + for (i = 0; i <= finest_level; i++) + { + if (n_cycle[i] != new_cycle[i]) + { + sub_unchanged = false; + n_cycle[i] = new_cycle[i]; + } + } + + } + + if (sub_unchanged) + // + // Limit dt's by change_max * old dt + // + { + for (i = 0; i <= finest_level; i++) + { + if (verbose && ParallelDescriptor::IOProcessor()) + { + if (dt_min[i] > change_max*dt_level[i]) + { + std::cout << "Nyx::compute_new_dt : limiting dt at level " + << i << '\n'; + std::cout << " ... new dt computed: " << dt_min[i] + << '\n'; + std::cout << " ... but limiting to: " + << change_max * dt_level[i] << " = " << change_max + << " * " << dt_level[i] << '\n'; + } + } + + dt_min[i] = std::min(dt_min[i], change_max * dt_level[i]); + } + // + // Find the minimum over all levels + // + for (i = 0; i <= finest_level; i++) + { + n_factor *= n_cycle[i]; + dt_0 = std::min(dt_0, n_factor * dt_min[i]); + } + } + else + { + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "Nyx: Changing subcycling pattern. New pattern:\n"; + for (i = 1; i <= finest_level; i++) + std::cout << " Lev / n_cycle: " << i << " " << n_cycle[i] << '\n'; + } + } + } + } + else + { + dt_0 = fixed_dt; + } + + // + // Limit dt's by the value of stop_time. + // + const Real eps = 0.001 * dt_0; +#ifndef NO_HYDRO + Real cur_time = state[State_Type].curTime(); +#else + Real cur_time = state[PhiGrav_Type].curTime(); +#endif + if (stop_time >= 0.0) + { + if ((cur_time + dt_0) > (stop_time - eps)) + dt_0 = stop_time - cur_time; + } + + // Shrink the time step if necessary in order to hit the next plot_z_value + if (level == 0 && ( plot_z_values.size() > 0 || analysis_z_values.size() > 0 ) ) + { + bool dt_changed_plot = false; + bool dt_changed_analysis = false; + + if (plot_z_values.size() > 0) + plot_z_est_time_step(dt_0,dt_changed_plot); + + if (analysis_z_values.size() > 0) + analysis_z_est_time_step(dt_0,dt_changed_analysis); + + // Update the value of a if we didn't change dt in the call to plot_z_est_time_step or analysis_z_est_time_step. + // If we didn't change dt there, then we have already done the integration. + // If we did change dt there, then we need to re-integrate here. + if (dt_changed_plot || dt_changed_analysis) + integrate_comoving_a(cur_time,dt_0); + } + else + { + integrate_comoving_a(cur_time,dt_0); + } + + n_factor = 1; + for (i = 0; i <= finest_level; i++) + { + n_factor *= n_cycle[i]; + dt_level[i] = dt_0 / n_factor; + } +} + +void +Nyx::computeInitialDt (int finest_level, + int /*sub_cycle*/, + Vector& n_cycle, + const Vector& /*ref_ratio*/, + Vector& dt_level, + Real stop_time) +{ + BL_PROFILE("Nyx::computeInitialDt()"); + // + // Grids have been constructed, compute dt for all levels. + // + if (level > 0) + return; + + int i; + Real dt_0 = 1.0e+100; + int n_factor = 1; + if (parent->subcyclingMode() == "Optimal") + { + Vector new_cycle(finest_level+1); + for (i = 0; i <= finest_level; i++) + new_cycle[i] = n_cycle[i]; + Vector dt_max(finest_level+1); + for (i = 0; i <= finest_level; i++) + { + dt_max[i] = get_level(i).initial_time_step(); + } + // Find the maximum number of cycles allowed + Vector cycle_max(finest_level+1); + cycle_max[0] = 1; + for (i = 1; i <= finest_level; i++) + { + cycle_max[i] = parent->MaxRefRatio(i-1); + } + // estimate the amout of work to advance each level. + Vector est_work(finest_level+1); + for (i = 0; i <= finest_level; i++) + { + est_work[i] = parent->getLevel(i).estimateWork(); + } + + dt_0 = parent->computeOptimalSubcycling(finest_level+1, new_cycle.dataPtr(), dt_max.dataPtr(), + est_work.dataPtr(), cycle_max.dataPtr()); + + for (i = 0; i <= finest_level; i++) + { + n_cycle[i] = new_cycle[i]; + } + if (verbose && ParallelDescriptor::IOProcessor() && finest_level > 0) + { + std::cout << "Nyx: Initial subcycling pattern:\n"; + for (i = 0; i <= finest_level; i++) + std::cout << "Level " << i << ": " << n_cycle[i] << '\n'; + } + } + else + { + for (i = 0; i <= finest_level; i++) + { + dt_level[i] = get_level(i).initial_time_step(); + n_factor *= n_cycle[i]; + dt_0 = std::min(dt_0, n_factor * dt_level[i]); + } + } + // + // Limit dt's by the value of stop_time. + // + const Real eps = 0.001 * dt_0; +#ifndef NO_HYDRO + Real cur_time = state[State_Type].curTime(); +#else + Real cur_time = state[PhiGrav_Type].curTime(); +#endif + if (stop_time >= 0) + { + if ((cur_time + dt_0) > (stop_time - eps)) + dt_0 = stop_time - cur_time; + } + + n_factor = 1; + for (i = 0; i <= finest_level; i++) + { + n_factor *= n_cycle[i]; + dt_level[i] = dt_0 / n_factor; + } + + integrate_comoving_a(cur_time,dt_0); +} + +bool +Nyx::writePlotNow () +{ + BL_PROFILE("Nyx::writePlotNow()"); + if (level > 0) + amrex::Error("Should only call writePlotNow at level 0!"); + + bool found_one = false; + + if (plot_z_values.size() > 0) + { +#ifndef NO_HYDRO + Real prev_time = state[State_Type].prevTime(); + Real cur_time = state[State_Type].curTime(); +#else + Real prev_time = state[PhiGrav_Type].prevTime(); + Real cur_time = state[PhiGrav_Type].curTime(); +#endif + + Real a_old = get_comoving_a(prev_time); + Real z_old = (1. / a_old) - 1.; + + Real a_new = get_comoving_a( cur_time); + Real z_new = (1. / a_new) - 1.; + + for (int i = 0; i < plot_z_values.size(); i++) + { + if (std::abs(z_new - plot_z_values[i]) < (0.01 * (z_old - z_new)) ) + found_one = true; + } + } + + if (found_one) { + return true; + } else { + return false; + } +} + +bool +Nyx::doAnalysisNow () +{ + BL_PROFILE("Nyx::doAnalysisNow()"); + if (level > 0) + amrex::Error("Should only call doAnalysisNow at level 0!"); + + bool found_one = false; + + if (analysis_z_values.size() > 0) + { + +#ifndef NO_HYDRO + Real prev_time = state[State_Type].prevTime(); + Real cur_time = state[State_Type].curTime(); +#else + Real prev_time = state[PhiGrav_Type].prevTime(); + Real cur_time = state[PhiGrav_Type].curTime(); +#endif + + Real a_old = get_comoving_a(prev_time); + Real z_old = (1. / a_old) - 1.; + + Real a_new = get_comoving_a( cur_time); + Real z_new = (1. / a_new) - 1.; + + for (int i = 0; i < analysis_z_values.size(); i++) + { + if (std::abs(z_new - analysis_z_values[i]) < (0.01 * (z_old - z_new)) ) + found_one = true; + } + } + + if (found_one) { + return true; + } else { + return false; + } +} + +void +Nyx::do_energy_diagnostics () +{ + // nothing to see here, folks +} + +void +Nyx::post_timestep (int iteration) +{ + BL_PROFILE("Nyx::post_timestep()"); + + MultiFab::RegionTag amrPost_tag("Post_" + std::to_string(level)); + + // + // Integration cycle on fine level grids is complete + // do post_timestep stuff here. + // + int finest_level = parent->finestLevel(); + const int ncycle = parent->nCycle(level); +#ifdef AMREX_PARTICLES + BL_PROFILE_VAR("Nyx::post_timestep()::remove_virt_ghost",rm); + // + // Remove virtual particles at this level if we have any. + // + remove_virtual_particles(); + + // + // Remove Ghost particles on the final iteration + // + if (iteration == ncycle) + remove_ghost_particles(); + amrex::Gpu::streamSynchronize(); + BL_PROFILE_VAR_STOP(rm); + BL_PROFILE_VAR("Nyx::post_timestep()::redist",redist); + + if(load_balance_int < 0 || !(nStep() % load_balance_int == 0 && level == 0 && (new_a >= 1.0/(load_balance_start_z + 1.0)))) + { + + // + // Sync up if we're level 0 or if we have particles that may have moved + // off the next finest level and need to be added to our own level. + // + if ((iteration < ncycle && level < finest_level) || level == 0) + { + for (int i = 0; i < theActiveParticles().size(); i++) + { + if(finest_level == 0) + theActiveParticles()[i]->RedistributeLocal(level, + theActiveParticles()[i]->finestLevel(), + iteration); + else + theActiveParticles()[i]->Redistribute(level, + theActiveParticles()[i]->finestLevel(), + iteration); + + if(shrink_to_fit) + theActiveParticles()[i]->ShrinkToFit(); + } + } + + } + amrex::Gpu::streamSynchronize(); + BL_PROFILE_VAR_STOP(redist); +#endif + BL_PROFILE_VAR("Nyx::post_timestep()::do_reflux",do_reflux); + +#ifndef NO_HYDRO + if (do_reflux && level < finest_level) + { + MultiFab& S_new_crse = get_new_data(State_Type); + MultiFab drho_and_drhoU; + if (do_grav) + { + // Define the update to rho and rhoU due to refluxing. + drho_and_drhoU.define(grids, dmap, AMREX_SPACEDIM + 1, 0); + MultiFab::Copy(drho_and_drhoU, S_new_crse, Density_comp, 0, + AMREX_SPACEDIM + 1, 0); + drho_and_drhoU.mult(-1.0); + } + + //We must reflux if the next finer level is subcycled relative to this level; + // otherwise the reflux was done as part of the multilevel advance + if (parent->nCycle(level+1) != 1) + reflux(); + + // We need to do this before anything else because refluxing changes the + // values of coarse cells underneath fine grids with the assumption + // they'll be over-written by averaging down + if (level < finest_level) + average_down(); + + // This needs to be done after any changes to the state from refluxing. +#ifndef CONST_SPECIES + enforce_nonnegative_species(S_new_crse); +#endif + + if (do_grav && gravity->get_no_sync() == 0) + { + MultiFab::Add(drho_and_drhoU, S_new_crse, Density_comp, 0, AMREX_SPACEDIM+1, 0); + + MultiFab dphi(grids, dmap, 1, 0); + dphi.setVal(0); + + gravity->reflux_phi(level, dphi); + + // Compute (cross-level) gravity sync based on drho, dphi + Vector > grad_delta_phi_cc(finest_level - level + 1); + for (int lev = level; lev <= finest_level; lev++) + { + grad_delta_phi_cc[lev-level].reset( + new MultiFab(get_level(lev).boxArray(), + get_level(lev).DistributionMap(), + AMREX_SPACEDIM, 0)); + grad_delta_phi_cc[lev-level]->setVal(0); + } + + gravity->gravity_sync(level,finest_level,iteration,ncycle,drho_and_drhoU,dphi, + amrex::GetVecOfPtrs(grad_delta_phi_cc)); + dphi.clear(); + + for (int lev = level; lev <= finest_level; lev++) + { + Real dt_lev = parent->dtLevel(lev); + MultiFab& S_new_lev = get_level(lev).get_new_data(State_Type); + Real cur_time = state[State_Type].curTime(); + Real a_new = get_comoving_a(cur_time); + + const auto& ba = get_level(lev).boxArray(); + const auto& dm = get_level(lev).DistributionMap(); + MultiFab grad_phi_cc(ba, dm, AMREX_SPACEDIM, 0); + gravity->get_new_grav_vector(lev, grad_phi_cc, cur_time); + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + { + FArrayBox dstate; + + for (MFIter mfi(S_new_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + + dstate.resize(bx, AMREX_SPACEDIM + 1); + Array4 d_fab = dstate.array(); + + if (lev == level) + { + dstate.copy(drho_and_drhoU[mfi]); + } + else + { + dstate.setVal(0); + } + + Array4 const& gphi = grad_phi_cc.array(mfi); + Array4 const& gdphi = grad_delta_phi_cc[lev-level]->array(mfi); + Array4 const& s_fab = S_new_lev.array(mfi); + + int iden = Density_comp; + int ieden = Eden_comp; + + amrex::ParallelFor(bx, [s_fab,d_fab,gphi,gdphi,a_new,dt_lev,iden,ieden] + AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + Real rho_pre = s_fab(i,j,k,iden ) - d_fab(i,j,k,0); + Real rhoU_pre = s_fab(i,j,k,iden+1) - d_fab(i,j,k,1); + Real rhoV_pre = s_fab(i,j,k,iden+2) - d_fab(i,j,k,2); + Real rhoW_pre = s_fab(i,j,k,iden+3) - d_fab(i,j,k,3); + + Real SrU = d_fab(i,j,k,1)*gphi(i,j,k,0) + rho_pre*gdphi(i,j,k,0); + Real SrV = d_fab(i,j,k,1)*gphi(i,j,k,1) + rho_pre*gdphi(i,j,k,1); + Real SrW = d_fab(i,j,k,1)*gphi(i,j,k,2) + rho_pre*gdphi(i,j,k,2); + + Real SrE = ( SrU * (rhoU_pre + (0.5*dt_lev)*SrU) + + SrV * (rhoV_pre + (0.5*dt_lev)*SrV) + + SrW * (rhoW_pre + (0.5*dt_lev)*SrW) ) / rho_pre; + + Real a_new_inv = 1. / a_new; + + s_fab(i,j,k,iden+1) += SrU * a_new_inv * 0.5 * dt_lev; + s_fab(i,j,k,iden+2) += SrV * a_new_inv * 0.5 * dt_lev; + s_fab(i,j,k,iden+3) += SrW * a_new_inv * 0.5 * dt_lev; + s_fab(i,j,k,ieden ) += SrE * a_new_inv * 0.5 * dt_lev; + }); + + } + } + } + } + } +#endif // end ifndef NO_HYDRO + + if (level < finest_level) + average_down(); + + amrex::Gpu::streamSynchronize(); + BL_PROFILE_VAR_STOP(do_reflux); + BL_PROFILE_VAR("Nyx::post_timestep()::sum_write",sum_write); + + if (level == 0) + { + int nstep = parent->levelSteps(0); +#ifndef NO_HYDRO + if ( (do_hydro == 1) && (sum_interval > 0) && (nstep % sum_interval == 0)) + { + sum_integrated_quantities(); + } +#endif + bool do_insitu = ((nstep+1) >= insitu_start) && + (insitu_int > 0) && ((nstep+1) % insitu_int == 0); + + if(do_insitu || doAnalysisNow()) + updateInSitu(); + + write_info(); + +#ifdef BL_USE_MPI + // Memory monitoring: + MemInfo* mInfo = MemInfo::GetInstance(); + char info[32]; + snprintf(info, sizeof(info), "Step %4d", nstep); + mInfo->LogSummary(info); +#endif + } + + amrex::Gpu::streamSynchronize(); + BL_PROFILE_VAR_STOP(sum_write); + BL_PROFILE_VAR("Nyx::post_timestep()::compute_temp",compute_temp); + +#ifndef NO_HYDRO + if (do_hydro) + { + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + // First reset internal energy before call to compute_temp + reset_internal_energy_nostore(S_new,D_new); + + // Re-compute temperature after all the other updates. + compute_new_temp(S_new,D_new); + } +#endif + + amrex::Gpu::Device::streamSynchronize(); + BL_PROFILE_VAR_STOP(compute_temp); + +} + +void +Nyx::typical_values_post_restart (const std::string& restart_file) +{ + if (level > 0) + return; + + if (use_typical_steps) + { + if (ParallelDescriptor::IOProcessor()) + { + std::string FileName = restart_file + "/first_max_steps"; + std::ifstream File; + File.open(FileName.c_str(),std::ios::in); + if (!File.good()) + amrex::FileOpenFailed(FileName); + File >> old_max_sundials_steps; + } + ParallelDescriptor::Bcast(&old_max_sundials_steps, 1, ParallelDescriptor::IOProcessorNumber()); + + if (ParallelDescriptor::IOProcessor()) + { + std::string FileName = restart_file + "/second_max_steps"; + std::ifstream File; + File.open(FileName.c_str(),std::ios::in); + if (!File.good()) + amrex::FileOpenFailed(FileName); + File >> new_max_sundials_steps; + } + ParallelDescriptor::Bcast(&new_max_sundials_steps, 1, ParallelDescriptor::IOProcessorNumber()); + } +} + +void +Nyx::post_restart () +{ + BL_PROFILE("Nyx::post_restart()"); +#ifdef AMREX_PARTICLES + if (level == 0) + particle_post_restart(parent->theRestartFile()); +#endif + if (level == 0) + comoving_a_post_restart(parent->theRestartFile()); + + if (level == 0) + typical_values_post_restart(parent->theRestartFile()); + + if (inhomo_reion) init_zhi(); + +#ifndef NO_HYDRO + Real cur_time = state[State_Type].curTime(); +#else + Real cur_time = state[PhiGrav_Type].curTime(); +#endif + + // Update the value of a only if restarting from chk00000 + // (special case for which computeNewDt is *not* called from Amr::coarseTimeStep) + if (level == 0 && cur_time == 0.0) + integrate_comoving_a(cur_time,parent->dtLevel(0)); + +#ifdef TISF + int blub = parent->finestLevel(); + fort_set_finest_level(&blub); +#endif + + if (do_grav) + { + if (level == 0) + { + for (int lev = 0; lev <= parent->finestLevel(); lev++) + { + AmrLevel& this_level = get_level(lev); + gravity->install_level(lev, &this_level); + } + + gravity->set_mass_offset(cur_time); + + // Do multilevel solve here. We now store phi in the checkpoint file so we can use it + // at restart. + int ngrow_for_solve = 1; + int use_previous_phi_as_guess = 1; + gravity->multilevel_solve_for_new_phi(0,parent->finestLevel(),ngrow_for_solve,use_previous_phi_as_guess); + +#ifndef AGN + if (do_dm_particles) +#endif + { + for (int k = 0; k <= parent->finestLevel(); k++) + { + const auto& ba = get_level(k).boxArray(); + const auto& dm = get_level(k).DistributionMap(); + MultiFab grav_vec_new(ba, dm, AMREX_SPACEDIM, 0); + gravity->get_new_grav_vector(k, grav_vec_new, cur_time); + } + } + } + } + +#ifndef NO_HYDRO + if (do_forcing) + { + if (level == 0) + forcing_post_restart(parent->theRestartFile()); + } + + if (level == 0) + { + // Need to compute this *before* regridding in case this is needed + compute_average_density(); + set_small_values(); + } +#endif +} + +#ifndef NO_HYDRO +void +Nyx::set_small_values () +{ + if (do_hydro == 0) { + return; + } + + const Real cur_time = state[State_Type].curTime(); + Real a = get_comoving_a(cur_time); + + Real average_temperature; + compute_average_temperature(average_temperature); + // + // Get the number of species from the network model. + // +#ifndef CONST_SPECIES + NumSpec = 2; +#endif + Real gamma_minus_1 = gamma - 1.0; + set_small_values_given_average + (average_gas_density, average_temperature, + a, small_dens, small_temp, small_pres, gamma_minus_1, h_species); + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "... setting small_dens to " << small_dens << '\n'; + std::cout << "... setting small_temp to " << small_temp << '\n'; + std::cout << "... setting small_pres to " << small_pres << '\n'; + } +} +#endif + +void +Nyx::postCoarseTimeStep (Real cumtime) +{ + BL_PROFILE("Nyx::postCoarseTimeStep()"); + MultiFab::RegionTag amrPost_tag("Post_" + std::to_string(level)); + +#ifdef AMREX_PARTICLES + if(load_balance_int >= 0 && nStep() % load_balance_int == 0 && (new_a >= 1.0/(load_balance_start_z + 1.0))) + { + if(verbose>0) + amrex::Print()<<"Load balancing since "<finestLevel(); lev++) + { + + Nyx* cs = dynamic_cast(&parent->getLevel(lev)); + Vector wgts(parent->boxArray(lev).size()); + DistributionMapping dm; + + //Should these weights be constructed based on level=0, or lev from for? + if(load_balance_wgt_strategy == 0) + { + for (unsigned int i = 0; i < wgts.size(); i++) + { + wgts[i] = parent->boxArray(lev)[i].numPts(); + } + if(load_balance_strategy==DistributionMapping::Strategy::KNAPSACK) + dm.KnapSackProcessorMap(wgts, load_balance_wgt_nmax); + else if(load_balance_strategy==DistributionMapping::Strategy::SFC) + dm.SFCProcessorMap(parent->boxArray(lev), wgts, load_balance_wgt_nmax); + else if(load_balance_strategy==DistributionMapping::Strategy::ROUNDROBIN) + dm.RoundRobinProcessorMap(wgts, load_balance_wgt_nmax); + } + else if(load_balance_wgt_strategy == 1) + { + wgts = cs->theDMPC()->NumberOfParticlesInGrid(lev,false,false); + if(load_balance_strategy==DistributionMapping::Strategy::KNAPSACK) + dm.KnapSackProcessorMap(wgts, load_balance_wgt_nmax); + else if(load_balance_strategy==DistributionMapping::Strategy::SFC) + dm.SFCProcessorMap(parent->boxArray(lev), wgts, load_balance_wgt_nmax); + else if(load_balance_strategy==DistributionMapping::Strategy::ROUNDROBIN) + dm.RoundRobinProcessorMap(wgts, load_balance_wgt_nmax); + } + else if(load_balance_wgt_strategy == 2) + { + MultiFab particle_mf(parent->boxArray(lev),theDMPC()->ParticleDistributionMap(lev),1,1); + cs->theDMPC()->Increment(particle_mf, lev); + if(load_balance_strategy==DistributionMapping::Strategy::KNAPSACK) + dm = DistributionMapping::makeKnapSack(particle_mf, load_balance_wgt_nmax); + else if(load_balance_strategy==DistributionMapping::Strategy::SFC) + dm = DistributionMapping::makeSFC(particle_mf, load_balance_wgt_nmax); + else if(load_balance_strategy==DistributionMapping::Strategy::ROUNDROBIN) + dm = DistributionMapping::makeRoundRobin(particle_mf); + } + else + { + amrex::Abort("Selected load balancing strategy not implemented"); + } + + amrex::Gpu::Device::streamSynchronize(); + const DistributionMapping& newdmap = dm; + + if(verbose > 2) + amrex::Print()<<"Using ba: "<boxArray(lev)<<"\nUsing dm: "< 0) + amrex::Abort("Particle load balancing needs multilevel testing"); + /* + cs->theActiveParticles()[i]->Redistribute(lev, + theActiveParticles()[i]->finestLevel(), + 1); + */ + cs->theActiveParticles()[i]->Regrid(newdmap, parent->boxArray(lev), lev); + + if(shrink_to_fit) + cs->theActiveParticles()[i]->ShrinkToFit(); + } + + if(cs->Nyx::theVirtPC() != 0) + { + cs->Nyx::theVirtPC()->Regrid(newdmap, parent->boxArray(lev), lev); + } + + if(cs->Nyx::theGhostPC() != 0) + { + cs->Nyx::theGhostPC()->Regrid(newdmap, parent->boxArray(lev), lev); + } + + amrex::Gpu::streamSynchronize(); + } + + } +#endif + AmrLevel::postCoarseTimeStep(cumtime); + +#ifdef AGN + if (level == 0) + { + agn_halo_find(parent->dtLevel(level)); + } +#endif + +#ifdef GIMLET + LyA_statistics(); +#endif + + int nstep = parent->levelSteps(0); + + if (verbose>1) + { + amrex::Print() << "End of postCoarseTimeStep, printing:" <level_being_advanced(); + bool do_grav_solve_here; + + if (which_level_being_advanced >= 0) + { + do_grav_solve_here = (level == which_level_being_advanced) && (lbase == which_level_being_advanced); + } else { + do_grav_solve_here = (level == lbase); + } + + // if(parent->maxLevel() == 0) + if(reuse_mlpoisson != 0) + gravity->setup_Poisson(level,new_finest); + + // Only do solve here if we will be using it in the timestep right after without re-solving, + // or if this is called from somewhere other than Amr::timeStep +#ifndef NO_HYDRO + const Real cur_time = state[State_Type].curTime(); + +#else + const Real cur_time = state[PhiGrav_Type].curTime(); +#endif + if ((cur_time > 0) && do_grav_solve_here) + { + // We have done a particle redistribute above so we shouldn't need to grow by more than 1 + // to capture the effect of all particles on this level + int ngrow_for_solve = 1; + int use_previous_phi_as_guess = 1; + gravity->multilevel_solve_for_new_phi(level, new_finest, ngrow_for_solve, use_previous_phi_as_guess); + +#ifndef AGN + if (do_dm_particles) +#endif + { + for (int k = 0; k <= parent->finestLevel(); k++) + { + const auto& ba = get_level(k).boxArray(); + const auto& dm = get_level(k).DistributionMap(); + MultiFab grav_vec_new(ba, dm, AMREX_SPACEDIM, 0); + gravity->get_new_grav_vector(k, grav_vec_new, cur_time); + } + } + } + } + delete fine_mask; + fine_mask = 0; +} + +void +Nyx::post_init (Real /*stop_time*/) +{ + BL_PROFILE("Nyx::post_init()"); + if (level > 0) { + return; + } + + // If we restarted from a plotfile, we need to reset the level_steps counter + if ( ! parent->theRestartPlotFile().empty()) { + parent->setLevelSteps(0,nsteps_from_plotfile); + } + + // + // Average data down from finer levels + // so that conserved data is consistent between levels. + // + int finest_level = parent->finestLevel(); + for (int k = finest_level - 1; k >= 0; --k) { + get_level(k).average_down(); + } + + if (do_grav) + { +#ifndef NO_HYDRO + const Real cur_time = state[State_Type].curTime(); +#else + const Real cur_time = state[PhiGrav_Type].curTime(); +#endif + + // + // Calculate offset before first multilevel solve. + // + gravity->set_mass_offset(cur_time); + + // + // Solve on full multilevel hierarchy + // + int ngrow_for_solve = 1; + gravity->multilevel_solve_for_new_phi(0, finest_level, ngrow_for_solve); + + // Make this call just to fill the initial state data. + for (int k = 0; k <= finest_level; k++) + { + const auto& ba = get_level(k).boxArray(); + const auto& dm = get_level(k).DistributionMap(); + MultiFab grav_vec_new(ba, dm, AMREX_SPACEDIM, 0); + gravity->get_new_grav_vector(k, grav_vec_new, cur_time); + } + } + +#ifndef NO_HYDRO + if ( (do_hydro == 1) && (sum_interval > 0) && (parent->levelSteps(0) % sum_interval == 0) ) + { + sum_integrated_quantities(); + } + else + { + // Even if we don't call `sum_integrated_quantities` we need to compute + // average_density before regridding + compute_average_density(); + } + + if (do_hydro == 1) + { + set_small_values(); + } +#endif + + write_info(); + +} + +int +Nyx::okToContinue () +{ + if (level > 0) { + return 1; + } + + int test = 1; + if (parent->dtLevel(0) < dt_cutoff) { + test = 0; + } + + if ((test == 1) && (final_a > 0)) + { +#ifndef NO_HYDRO + const Real cur_time = state[State_Type].curTime(); +#else + const Real cur_time = state[PhiGrav_Type].curTime(); +#endif + Real a = get_comoving_a(cur_time); + if (a >= final_a) test = 0; + if (verbose && ParallelDescriptor::IOProcessor()) + { + if (test == 0) { + std::cout << "...a " << a + << " is greater than or equal to final_a " << final_a + << '\n'; + } + } + } + return test; +} + +#ifdef AUX_UPDATE +void +Nyx::advance_aux (Real time, + Real dt) +{ + BL_PROFILE("Nyx::advance_aux()"); + if (verbose && ParallelDescriptor::IOProcessor()) + std::cout << "... special update for auxiliary variables \n"; + + MultiFab& S_old = get_old_data(State_Type); + MultiFab& S_new = get_new_data(State_Type); + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_old,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& box = mfi.tilebox(); + FArrayBox& old_fab = S_old[mfi]; + FArrayBox& new_fab = S_new[mfi]; + fort_auxupdate + (BL_TO_FORTRAN(old_fab), BL_TO_FORTRAN(new_fab), box.loVect(), + box.hiVect(), &dt); + } +} +#endif + +#ifndef NO_HYDRO +void +Nyx::reflux () +{ + BL_PROFILE("Nyx::reflux()"); + + BL_ASSERT(levelfinestLevel()); + + get_flux_reg(level+1).Reflux(get_new_data(State_Type), 1.0, 0, 0, NUM_STATE, + geom); +} +#endif // NO_HYDRO + +void +Nyx::average_down () +{ + BL_PROFILE("Nyx::average_down()"); + if (level == parent->finestLevel()) return; + +#ifndef NO_HYDRO + // With State_Type we do DiagEOS_Type + average_down(State_Type); +#endif + + if (do_grav) + { + average_down(PhiGrav_Type); + average_down(Gravity_Type); + } +} + +#ifndef NO_HYDRO +#ifndef CONST_SPECIES +void +Nyx::enforce_nonnegative_species (MultiFab& S_new) +{ + BL_PROFILE("Nyx::enforce_nonnegative_species()"); + Real eps = -1.0e-16; + int NumSpec = S_new.nComp()-FirstSpec_comp; + + if (NumSpec > 0) + { +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + const auto uout = S_new.array(mfi); + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + bool any_negative=false; + Real x; + Real dom_spec; + int int_dom_spec; + for (int n = FirstSpec_comp; n < FirstSpec_comp + NumSpec; n++) + { + if (uout(i,j,k,n) < 0.0) + { + x = uout(i,j,k,n)/uout(i,j,k,Density_comp); + if (x > eps) + uout(i,j,k,n) = 0.0; + else + any_negative = true; + } + } + + // + // We know there are one or more undershoots needing correction. + // + if (any_negative) + { + // + // Find the dominant species. + // + int_dom_spec = FirstSpec_comp; + dom_spec = uout(i,j,k,int_dom_spec); + + for (int n = 0; n < FirstSpec_comp + NumSpec; n++) + { + if (uout(i,j,k,n) > dom_spec) + { + dom_spec = uout(i,j,k,n); + int_dom_spec = n; + } + } + // + // Now take care of undershoots greater in magnitude than 1e-16. + // + for (int n = 0; n < FirstSpec_comp + NumSpec; n++) + { + if (uout(i,j,k,n) < 0.0) + { + x = uout(i,j,k,n)/uout(i,j,k,Density_comp); + // + // Take enough from the dominant species to fill the negative one. + // + uout(i,j,k,int_dom_spec) = uout(i,j,k,int_dom_spec) + uout(i,j,k,n); + // + // Test that we didn't make the dominant species negative. + // + // + // Now set the negative species to zero. + // + uout(i,j,k,n) = 0.00; + } + } + } + }); + } + } +} +#endif + +void +Nyx::enforce_consistent_e (MultiFab& S) +{ + BL_PROFILE("Nyx::enforce_consistent_e()"); + + // Set (rho E) = (rho e) + 1/2 rho (u^2 +_ v^2 + w^2) + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + auto const s_arr = S.array(mfi); + + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + s_arr(i,j,k,Eden_comp) = s_arr(i,j,k,Eint_comp) + 0.5 * ( + s_arr(i,j,k,Xmom_comp)*s_arr(i,j,k,Xmom_comp) + + s_arr(i,j,k,Ymom_comp)*s_arr(i,j,k,Ymom_comp) + + s_arr(i,j,k,Zmom_comp)*s_arr(i,j,k,Zmom_comp) ) / s_arr(i,j,k,Density_comp); + }); + } +} +#endif + +void +Nyx::average_down (int state_index) +{ + BL_PROFILE("Nyx::average_down(si)"); +#ifndef NO_HYDRO + // We average DiagEOS_Type when average_down is called with State_Type + if (state_index == DiagEOS_Type) return; +#endif + + if (level == parent->finestLevel()) return; + + Nyx& fine_lev = get_level(level+1); + + const Geometry& fgeom = fine_lev.geom; + const Geometry& cgeom = geom; + +#ifndef NO_HYDRO + if (state_index == State_Type) + { + MultiFab& S_crse = get_new_data(State_Type); + MultiFab& S_fine = fine_lev.get_new_data(State_Type); + + amrex::average_down(S_fine, S_crse, + fgeom, cgeom, + 0, S_fine.nComp(), fine_ratio); + + MultiFab& D_crse = get_new_data(DiagEOS_Type); + MultiFab& D_fine = fine_lev.get_new_data(DiagEOS_Type); + + amrex::average_down(D_fine, D_crse, + fgeom, cgeom, + 0, D_fine.nComp(), fine_ratio); + } + else +#endif + { + MultiFab& S_crse = get_new_data(state_index); + MultiFab& S_fine = fine_lev.get_new_data(state_index); + + const int num_comps = S_fine.nComp(); + + amrex::average_down(S_fine,S_crse,fgeom,cgeom,0,num_comps,fine_ratio); + } +} + +void +Nyx::errorEst (TagBoxArray& tags, + int clearval, + int tagval, + Real time, + int /*n_error_buf*/, + int /*ngrow*/) +{ + BL_PROFILE("Nyx::errorEst()"); + + for (int j=0; j mf; + if (errtags[0].Field() != std::string()) { + mf = std::unique_ptr(derive(errtags[j].Field(), time, errtags[j].NGrow())); + } + errtags[j](tags,mf.get(),clearval,tagval,time,level,geom); + } +} + +std::unique_ptr +Nyx::derive (const std::string& name, + Real time, + int ngrow) +{ + BL_PROFILE("Nyx::derive()"); + + + if (name == "Rank") + { + std::unique_ptr derive_dat (new MultiFab(grids, dmap, 1, 0)); +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*derive_dat,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const auto fab_derive_dat=derive_dat->array(mfi); + const Box& bx = mfi.tilebox(); + Real my_proc = ParallelDescriptor::MyProc(); + AMREX_HOST_DEVICE_FOR_3D ( bx, i, j, k, + { + fab_derive_dat(i,j,k,0)=my_proc; + }); + } + return derive_dat; + } else { + return particle_derive(name, time, ngrow); + } +} + +void +Nyx::derive (const std::string& name, + Real time, + MultiFab& mf, + int dcomp) +{ + BL_PROFILE("Nyx::derive(mf)"); + if (name == "Rank") + { +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(mf,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const auto fab_derive_dat=mf.array(mfi); + const Box& bx = mfi.tilebox(); + Real my_proc = ParallelDescriptor::MyProc(); + AMREX_HOST_DEVICE_FOR_3D ( bx, i, j, k, + { + fab_derive_dat(i,j,k,dcomp)=my_proc; + }); + } + } else { + const auto& derive_dat = particle_derive(name, time, mf.nGrow()); + MultiFab::Copy(mf, *derive_dat, 0, dcomp, 1, mf.nGrow()); + } +} + +#ifndef NO_HYDRO +void +Nyx::reset_internal_energy (MultiFab& S_new, MultiFab& D_new, MultiFab& reset_e_src) +{ + BL_PROFILE("Nyx::reset_internal_energy()"); + // Synchronize (rho e) and (rho E) so they are consistent with each other + + const Real cur_time = state[State_Type].curTime(); + Real a = get_comoving_a(cur_time); + int interp=false; + Real gamma_minus_1 = gamma - 1.0; + auto atomic_rates = atomic_rates_glob; + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + + const auto fab = S_new.array(mfi); + const auto fab_diag = D_new.array(mfi); + const auto fab_reset = reset_e_src.array(mfi); + Real h_species_in=h_species; + Real small_temp_in=small_temp; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + reset_internal_e + (i,j,k,fab,fab_diag,fab_reset, + atomic_rates, a, gamma_minus_1,h_species_in, small_temp_in, interp); + }); + } +} + +void +Nyx::reset_internal_energy_interp (MultiFab& S_new, MultiFab& D_new, MultiFab& reset_e_src) +{ + BL_PROFILE("Nyx::reset_internal_energy()"); + // Synchronize (rho e) and (rho E) so they are consistent with each other + + const Real cur_time = state[State_Type].curTime(); + Real a = get_comoving_a(cur_time); + int interp=true; + Real gamma_minus_1 = gamma - 1.0; + auto atomic_rates = atomic_rates_glob; + + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + const auto fab = S_new.array(mfi); + const auto fab_diag = D_new.array(mfi); + const auto fab_reset = reset_e_src.array(mfi); + Real h_species_in=h_species; + Real small_temp_in=small_temp; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + reset_internal_e + (i,j,k,fab,fab_diag,fab_reset, + atomic_rates, a, gamma_minus_1,h_species_in, small_temp_in, interp); + }); + } + +} + +void +Nyx::reset_internal_energy_nostore (MultiFab& S_new, MultiFab& D_new) +{ + BL_PROFILE("Nyx::reset_internal_energy_nostore()"); + // Synchronize (rho e) and (rho E) so they are consistent with each other + + MultiFab reset_e_src(S_new.boxArray(), S_new.DistributionMap(), 1, NUM_GROW); + reset_e_src.setVal(0.0); + + reset_internal_energy (S_new, D_new, reset_e_src); +} +#endif + +#ifndef NO_HYDRO +void +Nyx::compute_new_temp (MultiFab& S_new, MultiFab& D_new) +{ + BL_PROFILE("Nyx::compute_new_temp()"); + + Real cur_time = state[State_Type].curTime(); + Real a = get_comoving_a(cur_time); + + amrex::Gpu::synchronize(); + FArrayBox test, test_d; + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + + /* + test.resize(bx,S_new.nComp()); + test_d.resize(bx,S_new.nComp()); + test.copy(S_new[mfi],bx); + test_d.copy(D_new[mfi],bx); + // test.copy(S_new[mfi],0,0,S_new.nComp()); + //test_d.copy(D_new[mfi],0,0,D_new.nComp());*/ + + const auto state_fab = S_new.array(mfi); + const auto diag_eos_fab = D_new.array(mfi); + + // AMREX_LAUNCH_DEVICE_LAMBDA + + + Real local_small_temp = small_temp; + Real local_large_temp = large_temp; + int local_max_temp_dt = max_temp_dt; + + Real h_species_in=h_species; + Real gamma_minus_1_in=gamma - 1.0; + auto atomic_rates = atomic_rates_glob; + AMREX_PARALLEL_FOR_3D(bx, i, j ,k, + { + Real rhoInv = 1.0 / state_fab(i,j,k,Density_comp); + Real eint = state_fab(i,j,k,Eint_comp) * rhoInv; + + if (state_fab(i,j,k,Eint_comp) > 0.0) + { + eint = state_fab(i,j,k,Eint_comp) * rhoInv; + + int JH = 1; + int JHe = 1; + + nyx_eos_T_given_Re_device(atomic_rates, gamma_minus_1_in, h_species_in, 1, 1, + &diag_eos_fab(i,j,k,Temp_comp), &diag_eos_fab(i,j,k,Ne_comp), + state_fab(i,j,k,Density_comp), state_fab(i,j,k,Eint_comp) * (1.0 / state_fab(i,j,k,Density_comp)), a); + + if(diag_eos_fab(i,j,k,Temp_comp)>=local_large_temp && local_max_temp_dt == 1) + { + diag_eos_fab(i,j,k,Temp_comp) = local_large_temp; + + Real dummy_pres=0.0; + // Set temp to small_temp and compute corresponding internal energy + nyx_eos_given_RT(atomic_rates, gamma_minus_1_in, h_species_in, &eint, &dummy_pres, + state_fab(i,j,k,Density_comp), diag_eos_fab(i,j,k,Temp_comp), + diag_eos_fab(i,j,k,Ne_comp), a); + + Real ke = 0.5e0 * (state_fab(i,j,k,Xmom_comp) * state_fab(i,j,k,Xmom_comp) + + state_fab(i,j,k,Ymom_comp) * state_fab(i,j,k,Ymom_comp) + + state_fab(i,j,k,Zmom_comp) * state_fab(i,j,k,Zmom_comp)) * rhoInv; + + state_fab(i,j,k,Eint_comp) = state_fab(i,j,k,Density_comp) * eint; + state_fab(i,j,k,Eden_comp) = state_fab(i,j,k,Eint_comp) + ke; + + } + } + else + { + Real dummy_pres=0.0; + // Set temp to small_temp and compute corresponding internal energy + nyx_eos_given_RT(atomic_rates, gamma_minus_1_in, h_species_in, &eint, &dummy_pres, + state_fab(i,j,k,Density_comp), local_small_temp, + diag_eos_fab(i,j,k,Ne_comp), a); + + Real ke = 0.5e0 * (state_fab(i,j,k,Xmom_comp) * state_fab(i,j,k,Xmom_comp) + + state_fab(i,j,k,Ymom_comp) * state_fab(i,j,k,Ymom_comp) + + state_fab(i,j,k,Zmom_comp) * state_fab(i,j,k,Zmom_comp)) * rhoInv; + + diag_eos_fab(i,j,k,Temp_comp) = local_small_temp; + state_fab(i,j,k,Eint_comp) = state_fab(i,j,k,Density_comp) * eint; + state_fab(i,j,k,Eden_comp) = state_fab(i,j,k,Eint_comp) + ke; + } + + }); + amrex::Gpu::synchronize(); + } + + // Find the cell which has the maximum temp -- but only if not the first + // time step because in the first time step too many points have the same + // value. + Real prev_time = state[State_Type].prevTime(); + if (prev_time > 0.0 && verbose > 1) + { + BL_PROFILE("Nyx::compute_new_temp()::max_temp"); + // Compute the maximum temperature + Real max_temp = D_new.norm0(Temp_comp); + IntVect max_temp_loc = D_new.maxIndex(Temp_comp); + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + if (bx.contains(max_temp_loc)) + { + const auto fab_state = S_new.array(mfi); + Real den_maxt=fab_state(max_temp_loc,Density_comp); + std::cout << "Maximum temp. at level " << level << " is " << max_temp + << " at density " << den_maxt + << " at (i,j,k) " << max_temp_loc << std::endl; + } + } + } + amrex::Gpu::synchronize(); +} +#endif + +#ifndef NO_HYDRO +void +Nyx::compute_rho_temp (Real& rho_T_avg, Real& T_avg, Real& Tinv_avg, Real& T_meanrho) +{ + BL_PROFILE("Nyx::compute_rho_temp()"); + + { + + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + Real rho_T_sum=0.0, T_sum=0.0, Tinv_sum=0.0, T_meanrho_sum=0.0; + Real rho_sum=0.0, vol_sum=0.0, vol_mn_sum=0.0; + + Real rho_hi = 1.1*average_gas_density; + Real rho_lo = 0.9*average_gas_density; + const auto dx= geom.CellSizeArray(); + +#ifdef AMREX_USE_GPU + if (Gpu::inLaunchRegion()) + { + BL_PROFILE("Nyx::compute_rho_temp()::ReduceOpsOnDevice"); + ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + for (MFIter mfi(S_new); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + const auto state = S_new.const_array(mfi); + const auto diag_eos = D_new.const_array(mfi); + Real vol = dx[0]*dx[1]*dx[2]; + reduce_op.eval(bx, reduce_data, + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple + { + Real T_tmp = diag_eos(i,j,k,Temp_comp); + Real rho_tmp = state(i,j,k,Density_comp); + Real l_rho_T_sum = 0._rt; + Real l_rho_sum = 0._rt; + Real l_T_sum = 0._rt; + Real l_Tinv_sum = 0._rt; + Real l_T_meanrho_sum = 0._rt; + Real l_vol_sum = 0._rt; + Real l_vol_mn_sum = 0._rt; + l_T_sum = vol*T_tmp; + l_Tinv_sum = rho_tmp/T_tmp; + l_rho_T_sum = rho_tmp*T_tmp; + l_rho_sum = rho_tmp; + if ( (rho_tmp < rho_hi) && (rho_tmp > rho_lo) && (T_tmp <= 1.0e5) ) { + l_T_meanrho_sum = vol*log10(T_tmp); + l_vol_mn_sum = vol; + } + l_vol_sum = vol; + return {l_rho_T_sum, l_rho_sum, l_T_sum, l_Tinv_sum, l_T_meanrho_sum, l_vol_sum, l_vol_mn_sum}; + }); + } + + ReduceTuple hv = reduce_data.value(); + rho_T_sum = amrex::get<0>(hv); + rho_sum = amrex::get<1>(hv); + T_sum = amrex::get<2>(hv); + Tinv_sum = amrex::get<3>(hv); + T_meanrho_sum = amrex::get<4>(hv); + vol_sum = amrex::get<5>(hv); + vol_mn_sum = amrex::get<6>(hv); + } + else +#endif + { +#ifdef _OPENMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion() && !system::regtest_reduction) \ + reduction(+:rho_T_sum, rho_sum, T_sum, Tinv_sum, T_meanrho_sum, vol_sum, vol_mn_sum) +#endif + { + BL_PROFILE("Nyx::compute_rho_temp()::OnHost"); + for (MFIter mfi(S_new,true); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + const auto s_arr = S_new.const_array(mfi); + const auto d_arr = D_new.const_array(mfi); + Real vol = dx[0]*dx[1]*dx[2]; + AMREX_LOOP_3D(bx, i, j, k, + { + Real T_tmp = d_arr(i,j,k,Temp_comp); + Real rho_tmp = s_arr(i,j,k,Density_comp); + + T_sum += vol*T_tmp; + Tinv_sum += rho_tmp/T_tmp; + rho_T_sum += rho_tmp*T_tmp; + rho_sum += rho_tmp; + + if ( (rho_tmp < rho_hi) && (rho_tmp > rho_lo) && (T_tmp <= 1.0e5) ) { + T_meanrho_sum += vol*log10(T_tmp); + vol_mn_sum += vol; + } + vol_sum += vol; + }); + } + } + } + + Real sums[7] = {rho_T_sum, rho_sum, T_sum, Tinv_sum, T_meanrho_sum, vol_sum, vol_mn_sum}; + + ParallelDescriptor::ReduceRealSum(sums,7); + + rho_T_avg = sums[0] / sums[1]; // density weighted T + T_avg = sums[2] / sums[5]; // volume weighted T + Tinv_avg = sums[3] / sums[1]; // 21cm T + + if (sums[6] > 0) { + T_meanrho = sums[4] / sums[6]; // T at mean density + T_meanrho = pow(10.0, T_meanrho); + } + } +} +#endif + +#ifndef NO_HYDRO +void +Nyx::compute_gas_fractions (Real T_cut, Real rho_cut, + Real& whim_mass_frac, Real& whim_vol_frac, + Real& hh_mass_frac, Real& hh_vol_frac, + Real& igm_mass_frac, Real& igm_vol_frac) +{ + BL_PROFILE("Nyx::compute_gas_fractions()"); + + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + Real whim_mass=0.0, whim_vol=0.0, hh_mass=0.0, hh_vol=0.0; + Real igm_mass=0.0, igm_vol=0.0, mass_sum=0.0, vol_sum=0.0; + + const auto dx= geom.CellSizeArray(); + +#ifdef AMREX_USE_GPU + Real local_average_gas_density = average_gas_density; + if (Gpu::inLaunchRegion()) + { + BL_PROFILE("Nyx::compute_gas_fractions()::ReduceOpsOnDevice"); + ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + + for (MFIter mfi(S_new); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + const auto state = S_new.const_array(mfi); + const auto diag_eos = D_new.const_array(mfi); + Real vol = dx[0]*dx[1]*dx[2]; + reduce_op.eval(bx, reduce_data, + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple + { + Real T = diag_eos(i,j,k,Temp_comp); + Real R = state(i,j,k,Density_comp) / local_average_gas_density; + Real rho_vol = state(i,j,k,Density_comp)*vol; + Real l_whim_mass = 0._rt; + Real l_whim_vol = 0._rt; + Real l_hh_mass = 0._rt; + Real l_hh_vol = 0._rt; + Real l_igm_mass = 0._rt; + Real l_igm_vol = 0._rt; + if ( (T > T_cut) && (R <= rho_cut) ) { + l_whim_mass = rho_vol; + l_whim_vol = vol; + } + else if ( (T > T_cut) && (R > rho_cut) ) { + l_hh_mass = rho_vol; + l_hh_vol = vol; + } + else if ( (T <= T_cut) && (R <= rho_cut) ) { + l_igm_mass = rho_vol; + l_igm_vol = vol; + } + Real l_mass_sum = rho_vol; + Real l_vol_sum = vol; + return {l_whim_mass, l_whim_vol, l_hh_mass, l_hh_vol, + l_igm_mass, l_igm_vol, l_mass_sum, l_vol_sum}; + }); + } + + ReduceTuple hv = reduce_data.value(); + whim_mass = amrex::get<0>(hv); + whim_vol = amrex::get<1>(hv); + hh_mass = amrex::get<2>(hv); + hh_vol = amrex::get<3>(hv); + igm_mass = amrex::get<4>(hv); + igm_vol = amrex::get<5>(hv); + mass_sum = amrex::get<6>(hv); + vol_sum = amrex::get<7>(hv); + } + else +#endif + { + BL_PROFILE("Nyx::compute_gas_fractions()::OnHost"); +#ifdef _OPENMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) \ + reduction(+:whim_mass, whim_vol, hh_mass, hh_vol, igm_mass, igm_vol, mass_sum, vol_sum) +#endif + for (MFIter mfi(S_new,true); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); + const auto s_arr = S_new.const_array(mfi); + const auto diag_eos = D_new.const_array(mfi); + Real vol = dx[0]*dx[1]*dx[2]; + AMREX_LOOP_3D(bx, i, j, k, + { + Real T = diag_eos(i,j,k,Temp_comp); + Real R = s_arr(i,j,k,Density_comp) / average_gas_density; + Real rho_vol = s_arr(i,j,k,Density_comp)*vol; + if ( (T > T_cut) && (R <= rho_cut) ) { + whim_mass += rho_vol; + whim_vol += vol; + } + else if ( (T > T_cut) && (R > rho_cut) ) { + hh_mass += rho_vol; + hh_vol += vol; + } + else if ( (T <= T_cut) && (R <= rho_cut) ) { + igm_mass += rho_vol; + igm_vol += vol; + } + mass_sum += rho_vol; + vol_sum += vol; + }); + } + } + + Real sums[8] = {whim_mass, whim_vol, hh_mass, hh_vol, igm_mass, igm_vol, mass_sum, vol_sum}; + + ParallelDescriptor::ReduceRealSum(sums,8); + + whim_mass_frac = sums[0] / sums[6]; + whim_vol_frac = sums[1] / sums[7]; + hh_mass_frac = sums[2] / sums[6]; + hh_vol_frac = sums[3] / sums[7]; + igm_mass_frac = sums[4] / sums[6]; + igm_vol_frac = sums[5] / sums[7]; +} +#endif + +Real +Nyx::getCPUTime() +{ + + int numCores = ParallelDescriptor::NProcs(); +#ifdef _OPENMP + numCores = numCores*omp_get_max_threads(); +#endif + + Real T = numCores*(ParallelDescriptor::second() - startCPUTime) + + previousCPUTimeUsed; + + return T; +} + +void +Nyx::InitErrorList() { + //err_list.clear(true); + //err_list.add("FULLSTATE",1,ErrorRec::Special,FORT_DENERROR); +} + + +//static Box the_same_box (const Box& b) { return b; } + +void +Nyx::InitDeriveList() { +} + + +void +Nyx::LevelDirectoryNames(const std::string &dir, + const std::string &secondDir, + std::string &LevelDir, + std::string &FullPath) +{ + LevelDir = amrex::Concatenate("Level_", level, 1); + // + // Now for the full pathname of that directory. + // + FullPath = dir; + if( ! FullPath.empty() && FullPath.back() != '/') { + FullPath += '/'; + } + FullPath += secondDir; + FullPath += "/"; + FullPath += LevelDir; +} + + +void +Nyx::CreateLevelDirectory (const std::string &dir) +{ + AmrLevel::CreateLevelDirectory(dir); // ---- this sets levelDirectoryCreated = true +#ifdef AMREX_PARTICLES + std::string dm(dir + "/" + Nyx::retrieveDM()); + if(ParallelDescriptor::IOProcessor()) { + if( ! amrex::UtilCreateDirectory(dm, 0755)) { + amrex::CreateDirectoryFailed(dm); + } + } + + std::string LevelDir, FullPath; + LevelDirectoryNames(dir, Nyx::retrieveDM(), LevelDir, FullPath); + if(ParallelDescriptor::IOProcessor()) { + if( ! amrex::UtilCreateDirectory(FullPath, 0755)) { + amrex::CreateDirectoryFailed(FullPath); + } + } + +#ifdef AGN + std::string agn(dir + "/" + Nyx::retrieveAGN()); + if(ParallelDescriptor::IOProcessor()) { + if( ! amrex::UtilCreateDirectory(agn, 0755)) { + amrex::CreateDirectoryFailed(agn); + } + } + + LevelDirectoryNames(dir, Nyx::retrieveAGN(), LevelDir, FullPath); + if(ParallelDescriptor::IOProcessor()) { + if( ! amrex::UtilCreateDirectory(FullPath, 0755)) { + amrex::CreateDirectoryFailed(FullPath); + } + } +#endif + +#ifdef NEUTRINO_DARK_MATTER + std::string npc(dir + "/" + Nyx::retrieveNPC()); + if(ParallelDescriptor::IOProcessor()) { + if( ! amrex::UtilCreateDirectory(npc, 0755)) { + amrex::CreateDirectoryFailed(npc); + } + } + + LevelDirectoryNames(dir, Nyx::retrieveNPC(), LevelDir, FullPath); + if(ParallelDescriptor::IOProcessor()) { + if( ! amrex::UtilCreateDirectory(FullPath, 0755)) { + amrex::CreateDirectoryFailed(FullPath); + } + } +#endif + + if(parent->UsingPrecreateDirectories()) { + if(Nyx::theDMPC()) { + Nyx::theDMPC()->SetLevelDirectoriesCreated(true); + } +#ifdef AGN + if(Nyx::theAPC()) { + Nyx::theAPC()->SetLevelDirectoriesCreated(true); + } +#endif + } +#endif +} diff --git a/Exec/HaloFinder_GPU/NyxParticleContainer.H b/Exec/HaloFinder_GPU/NyxParticleContainer.H new file mode 100644 index 000000000..12df6519a --- /dev/null +++ b/Exec/HaloFinder_GPU/NyxParticleContainer.H @@ -0,0 +1,524 @@ +#ifndef _NyxParticleContainer_H_ +#define _NyxParticleContainer_H_ + +#include +#include +#include +#include +#include + +class NyxParticleContainerBase +{ +public: + + virtual ~NyxParticleContainerBase() {} + + virtual void moveKickDrift (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_old = 1.0, amrex::Real a_half = 1.0, int where_width = 0, + amrex::Real radius_inner = -1.e34, amrex::Real radius_outer = -1.e34, + amrex::Vector* shell_particles=nullptr) = 0; + virtual void moveKick (amrex::MultiFab& acceleration, int level, amrex::Real time, + amrex::Real timestep, + amrex::Real a_new = 1.0, amrex::Real a_half = 1.0) = 0; + virtual void Redistribute (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual void RedistributeLocal (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual void RedistributeOK (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual void RedistributeGPU (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) = 0; + + virtual int finestLevel() const = 0; + virtual void ShrinkToFit() = 0; + virtual void RemoveParticlesAtLevel (int level) = 0; + virtual amrex::Real sumParticleMass (int level) const = 0; + virtual void Regrid (const amrex::DistributionMapping& dmap, const amrex::BoxArray& ba, const int lev) = 0; + // void RegridHat (const amrex::DistributionMapping& dmap, const amrex::BoxArray& ba, const int lev); + virtual amrex::Vector NumberOfParticlesInGrid (int lev, bool only_valid, bool only_local) = 0; + + virtual void AssignDensitySingleLevel (amrex::MultiFab& mf, int level, int ncomp=1, + int particle_lvl_offset = 0) const = 0; + virtual void AssignDensity (amrex::Vector >& mf, int lev_min = 0, int ncomp = 1, + int finest_level = -1, int ngrow = 1) const = 0; +}; + +template +class NyxParticleContainer + : public amrex::NeighborParticleContainer, + public NyxParticleContainerBase +{ +public: + + using ParticleTileType = amrex::ParticleTile,NAR,NAI>; + using MyParIter = amrex::ParIter; + using MyConstParIter = amrex::ParConstIter; + + NyxParticleContainer (amrex::Amr* amr, int nghost=0) + : amrex::NeighborParticleContainer((amrex::ParGDBBase*) amr->GetParGDB(), nghost), + sub_cycle(amr->subCycle()) + {} + + virtual ~NyxParticleContainer () {} + + void GetParticleVelocities (amrex::Vector& part_vels); + void SetParticleVelocities (amrex::Vector& part_data); + + virtual amrex::Real sumParticleMass (int level) const override { + using PType = typename amrex::ParticleContainer::SuperParticleType; + auto msum = amrex::ReduceSum(*this, level, + [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> amrex::Real + { + return (p.id() > 0) ? p.rdata(0) : 0.0; + }); + amrex::ParallelAllReduce::Sum(msum, amrex::ParallelContext::CommunicatorSub()); + return msum; + } + + virtual void Regrid (const amrex::DistributionMapping& dmap, const amrex::BoxArray& ba, const int lev) override { + amrex::NeighborParticleContainer::Regrid(dmap, ba, lev); + } + + virtual amrex::Vector NumberOfParticlesInGrid (int lev, bool only_valid, bool only_local) override { + return amrex::NeighborParticleContainer::NumberOfParticlesInGrid(lev,only_valid,only_local); + } + void sumParticleMomentum (int lev, amrex::Real* mom) const; + + virtual void AssignDensitySingleLevel (amrex::MultiFab& mf, int level, int ncomp=1, int particle_lvl_offset = 0) const override + { + amrex::NeighborParticleContainer::AssignCellDensitySingleLevel(0, mf, level, ncomp, particle_lvl_offset); + } + virtual void AssignDensity (amrex::Vector >& mf, int lev_min = 0, int ncomp = 1, int finest_level = -1, int ngrow = 1) const override + { + amrex::NeighborParticleContainer::AssignDensity(0, mf, lev_min, ncomp, finest_level, ngrow); + } + + void MultiplyParticleMass (int lev, amrex::Real mult); + + amrex::Real estTimestep (amrex::MultiFab& acceleration, int level, amrex::Real cfl) const; + amrex::Real estTimestep (amrex::MultiFab& acceleration, amrex::Real a, int level, amrex::Real cfl) const; + + virtual int finestLevel() const override + { + return amrex::NeighborParticleContainer::finestLevel(); + } + + virtual void ShrinkToFit() override + { + amrex::NeighborParticleContainer::ShrinkToFit(); + } + + virtual void Redistribute (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) override + { + // amrex::Gpu::synchronize(); + amrex::NeighborParticleContainer::Redistribute(lev_min, lev_max, nGrow); + // amrex::Gpu::synchronize(); + } + + virtual void RedistributeLocal (int /* lev_min = 0*/, + int /* lev_max =-1*/, + int /* nGrow = 0*/) override + { + AMREX_ASSERT(this->finestLevel() == 0); + + const int local = true; + const int lev_minal = 0; + const int lev_maxal = 0; + const int nGrowal = 0; + + amrex::NeighborParticleContainer + ::Redistribute(lev_minal, lev_maxal, nGrowal, local); + } + + virtual void RedistributeOK (int lev_min = 0, + int lev_max =-1, + int nGrow = 0) override + { + + const int local = amrex::NeighborParticleContainer + ::OK(lev_min, lev_max); + // amrex::Print()<<"local flag is: "< + ::Redistribute(lev_min, lev_max, nGrow, local); + + } + + virtual void RedistributeGPU (int /*lev_min = 0*/, + int /*lev_max =-1*/, + int /*nGrow = 0*/) override + { + AMREX_ASSERT(this->finestLevel() == 0); + + int local = false; + const int lev_minal = 0; + const int lev_maxal = 0; + const int nGrowal = 0; + + amrex::Gpu::synchronize(); + local = amrex::NeighborParticleContainer + ::OK(lev_minal, lev_maxal); + // amrex::Print()<<"local flag is: "< + ::RedistributeGPU(lev_minal, lev_maxal, nGrowal, local); + else + amrex::NeighborParticleContainer + ::RedistributeCPU(lev_minal, lev_maxal, nGrowal, local); + } + + virtual void RemoveParticlesAtLevel (int level) override + { + amrex::NeighborParticleContainer::RemoveParticlesAtLevel(level); + } + + virtual void WriteNyxPlotFile (const std::string& dir, + const std::string& name) const; + + virtual void NyxCheckpoint (const std::string& dir, + const std::string& name) const; + + typedef amrex::Particle ParticleType; + using AoS = typename amrex::ParticleContainer::AoS; + using ParticleLevel = typename amrex::ParticleContainer::ParticleLevel; + +protected: + bool sub_cycle; + amrex::Vector real_comp_names; +}; + +template +void +NyxParticleContainer::GetParticleVelocities (amrex::Vector& part_data) +{ + BL_PROFILE("NyxParticleContainer::GetParticleVelocities()"); + // This assumes that the mass/charge is stored in the first position + // in the particle data, followed by the velocity components + int start_comp = 1; + int num_comp = AMREX_SPACEDIM; + this->GetParticleData(part_data,1,AMREX_SPACEDIM); +} + +template +void +NyxParticleContainer::SetParticleVelocities (amrex::Vector& part_data) +{ + BL_PROFILE("NyxParticleContainer::SetParticleVelocities()"); + // This gives us the starting point into the part_data array + // If only one processor (or no MPI), then that's all we need + int cnt = 0; + +#ifdef BL_USE_MPI + amrex::Vector cnts(amrex::ParallelDescriptor::NProcs()); + + // This returns the number of particles on this processor + long lcnt = this->TotalNumberOfParticles(true,true); + + // This accumulates the "lcnt" values into "cnts" + MPI_Gather(&lcnt,1, + amrex::ParallelDescriptor::Mpi_typemap::type(), + cnts.dataPtr(), + 1, + amrex::ParallelDescriptor::Mpi_typemap::type(), + amrex::ParallelDescriptor::IOProcessorNumber(), + amrex::ParallelDescriptor::Communicator()); + + amrex::ParallelDescriptor::Bcast(cnts.dataPtr(), cnts.size(), amrex::ParallelDescriptor::IOProcessorNumber()); + + for (int iproc = 0; iproc < amrex::ParallelDescriptor::MyProc(); iproc++) + cnt += cnts[iproc]; + + // Each particle takes up (AMREX_SPACEDIM) Reals + cnt*= (AMREX_SPACEDIM); +#endif + + // This is the total number of particles on *all* processors + long npart = this->TotalNumberOfParticles(true,false); + + // Velocities + if (part_data.size() != npart*(AMREX_SPACEDIM)) + amrex::Abort("Sending in wrong size part_data to SetParticleVelocities"); + + for (int lev = 0; lev <= this->m_gdb->finestLevel(); lev++) + { + ParticleLevel& pmap = this->GetParticles(lev); + + for (typename ParticleLevel::iterator pmap_it = pmap.begin(), pmapEnd = pmap.end(); pmap_it != pmapEnd; ++pmap_it) + { + AoS& pbx = pmap_it->second.GetArrayOfStructs(); + const int n = pbx.size(); + + for (int i = 0; i < n; i++) + { + ParticleType& p = pbx[i]; + if (p.id() > 0) + { + // Load velocities + for (int d=0; d < AMREX_SPACEDIM; d++) + p.rdata(d+1) = part_data[cnt+d]; + + // Update counter + cnt += AMREX_SPACEDIM; + } + } + } + } +} + +// +// Assumes mass is in rdata(0), vx in rdata(1), ...! +// dim defines the cartesian direction in which the momentum is summed, x is 0, y is 1, ... +// + +template +void +NyxParticleContainer::sumParticleMomentum (int lev, + amrex::Real* mom) const +{ + BL_PROFILE("NyxParticleContainer::sumParticleMomentum()"); + BL_ASSERT(NSR >= AMREX_SPACEDIM+1); + BL_ASSERT(lev >= 0 && lev < this->GetParticles().size()); + + const ParticleLevel& pmap = this->GetParticles(lev); + + AMREX_D_TERM(mom[0] = 0;, mom[1] = 0;, mom[2] = 0;); + + for (typename ParticleLevel::const_iterator pmap_it = pmap.begin(), pmapEnd = pmap.end(); pmap_it != pmapEnd; ++pmap_it) + { + const AoS& pbox = pmap_it->second.GetArrayOfStructs(); + const int n = pbox.size(); + + amrex::Real mom_0 = 0, mom_1 = 0, mom_2 = 0; + +#ifdef _OPENMP +#pragma omp parallel for reduction(+:mom_0,mom_1,mom_2) +#endif + for (int i = 0; i < n; i++) + { + const ParticleType& p = pbox[i]; + + if (p.id() > 0) + { + AMREX_D_TERM(mom_0 += p.rdata(0) * p.rdata(1);, + mom_1 += p.rdata(0) * p.rdata(2);, + mom_2 += p.rdata(0) * p.rdata(3);); + } + } + + AMREX_D_TERM(mom[0] += mom_0;, mom[1] += mom_1;, mom[2] += mom_2;); + } + + amrex::ParallelDescriptor::ReduceRealSum(mom,AMREX_SPACEDIM); +} + +template +amrex::Real +NyxParticleContainer::estTimestep (amrex::MultiFab& acceleration, + int lev, + amrex::Real cfl) const +{ + return estTimestep(acceleration,1.0,lev,cfl); +} + +template +amrex::Real +NyxParticleContainer::estTimestep (amrex::MultiFab& acceleration, + amrex::Real a, + int lev, + amrex::Real cfl) const +{ + BL_PROFILE("NyxParticleContainer::estTimestep(lev)"); + amrex::Real dt = 1e50; + BL_ASSERT(NSR >= AMREX_SPACEDIM+1); + BL_ASSERT(lev >= 0); + + if (this->GetParticles().size() == 0) + return dt; + + const amrex::Real strttime = amrex::ParallelDescriptor::second(); + + const amrex::Geometry& geom = this->m_gdb->Geom(lev); + const amrex::IntVect base = geom.Domain().smallEnd(); + const auto dxi = geom.InvCellSizeArray(); + amrex::GpuArray adxi{dxi[0]/a,dxi[1]/a,dxi[2]/a}; + const auto plo = geom.ProbLoArray(); + + long num_particles_at_level = 0; + + std::unique_ptr ac_pointer; + if (!this->OnSameGrids(lev, acceleration)) + { + ac_pointer.reset(new amrex::MultiFab(this->m_gdb->ParticleBoxArray(lev), + this->m_gdb->ParticleDistributionMap(lev), + acceleration.nComp(), acceleration.nGrow())); + if(acceleration.boxArray() == ac_pointer->boxArray())//this->finestLevel() == 0) + { + ac_pointer->Redistribute(acceleration,0,0,AMREX_SPACEDIM,acceleration.nGrowVect()); + ac_pointer->FillBoundary(geom.periodicity()); // DO WE NEED GHOST CELLS FILLED ??? + } + else + { + ac_pointer->ParallelCopy(acceleration,0,0,AMREX_SPACEDIM,acceleration.nGrowVect(),acceleration.nGrowVect()); + ac_pointer->FillBoundary(geom.periodicity()); // DO WE NEED GHOST CELLS FILLED ??? + } + } + +#ifdef _OPENMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) reduction(min:dt) +#endif + { + amrex::ReduceOps reduce_op; + amrex::ReduceData reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + + for (MyConstParIter pti(*this, lev); pti.isValid(); ++pti) { + const int grid = pti.index(); + const AoS& pbox = pti.GetArrayOfStructs(); + const ParticleType* pstruct = pbox().data(); + const int n = pbox.size(); + + const auto gfab = (ac_pointer) ? (*ac_pointer).array(grid) : acceleration.array(grid); + + num_particles_at_level += n; + + reduce_op.eval(n, reduce_data, + [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple { + const ParticleType& p = pstruct[i]; + + if (p.id() > 0) + { + + amrex::IntVect cell; + cell[0]= static_cast(amrex::Math::floor((p.pos(0)-plo[0])*dxi[0])); + cell[1]= static_cast(amrex::Math::floor((p.pos(1)-plo[1])*dxi[1])); + cell[2]= static_cast(amrex::Math::floor((p.pos(2)-plo[2])*dxi[2])); + cell += base; + + const amrex::Real mag_vel_over_dx = amrex::Math::abs(p.rdata(1))*adxi[0]; + const amrex::Real mag_vel_over_dy = amrex::Math::abs(p.rdata(2))*adxi[1]; + const amrex::Real mag_vel_over_dz = amrex::Math::abs(p.rdata(3))*adxi[2]; + + amrex::Real max_mag_vel_over_dx = amrex::max(mag_vel_over_dx,mag_vel_over_dy,mag_vel_over_dz); + amrex::Real dt_part = (max_mag_vel_over_dx > 0) ? (cfl / max_mag_vel_over_dx) : 1e50; + + const amrex::Real aval_x = gfab(cell[0],cell[1],cell[2],0); + const amrex::Real aval_y = gfab(cell[0],cell[1],cell[2],1); + const amrex::Real aval_z = gfab(cell[0],cell[1],cell[2],2); + const amrex::Real mag_accel = std::sqrt(aval_x*aval_x+aval_y*aval_y+aval_z*aval_z); + + if (mag_accel > 0) { + dt_part = amrex::min( dt_part, 1/std::sqrt(mag_accel*dxi[0]) ); + } + + return dt_part; + } + + return 1e50; + }); + } + + ReduceTuple hv = reduce_data.value(); + amrex::Real ldt_cpu = amrex::get<0>(hv); + + dt = std::min(dt,ldt_cpu); + } + + amrex::ParallelDescriptor::ReduceRealMin(dt); + + if (this->m_verbose > 1) + { + long min_local_num_particles_at_level = num_particles_at_level; + long max_local_num_particles_at_level = num_particles_at_level; + + amrex::ParallelDescriptor::ReduceLongMin(min_local_num_particles_at_level); + amrex::ParallelDescriptor::ReduceLongMax(max_local_num_particles_at_level); + + amrex::Print()<<"Min particles : "<m_verbose > 1) + { + amrex::Real stoptime = amrex::ParallelDescriptor::second() - strttime; + + amrex::ParallelDescriptor::ReduceRealMax(stoptime,amrex::ParallelDescriptor::IOProcessorNumber()); + + if (amrex::ParallelDescriptor::IOProcessor()) + { + std::cout << "NyxParticleContainer::estTimestep() time: " << stoptime << '\n'; + } + } + + return dt; +} + +template +void +NyxParticleContainer::MultiplyParticleMass (int lev, amrex::Real mult) +{ + BL_PROFILE("NyxParticleContainer::MultiplyParticleMass()"); + BL_ASSERT(lev == 0); + + ParticleLevel& pmap = this->GetParticles(lev); + + for (typename ParticleLevel::iterator pmap_it = pmap.begin(), pmapEnd = pmap.end(); pmap_it != pmapEnd; ++pmap_it) + { + AoS& pbx = pmap_it->second.GetArrayOfStructs(); + ParticleType* pstruct = pbx().data(); + const long np = pbx.size(); + + amrex::ParallelFor(np, + [=] AMREX_GPU_HOST_DEVICE ( long i) + { + if (pstruct[i].id() > 0) + { + // + // Note: rdata(0) is mass, ... + // + pstruct[i].rdata(0) *= mult; + } + }); + } + amrex::Gpu::streamSynchronize(); +} + +template +void +NyxParticleContainer::WriteNyxPlotFile (const std::string& dir, + const std::string& name) const +{ + BL_PROFILE("NyxParticleContainer::WriteNyxPlotFile()"); + + amrex::NeighborParticleContainer::WritePlotFile(dir, name, real_comp_names); + +} + +template +void +NyxParticleContainer::NyxCheckpoint (const std::string& dir, + const std::string& name) const +{ + BL_PROFILE("NyxParticleContainer::NyxCheckpoint()"); + + bool is_checkpoint = true; + amrex::NeighborParticleContainer::Checkpoint(dir, name, is_checkpoint, real_comp_names); +} + +#endif /*_NyxParticleContainer_H_*/ diff --git a/Exec/HaloFinder_GPU/NyxParticles.cpp b/Exec/HaloFinder_GPU/NyxParticles.cpp new file mode 100644 index 000000000..aabe92ccc --- /dev/null +++ b/Exec/HaloFinder_GPU/NyxParticles.cpp @@ -0,0 +1,1273 @@ +#include +#include +#include + +#include + +using namespace amrex; + +namespace +{ + bool virtual_particles_set = false; + + std::string ascii_particle_file; + std::string binary_particle_file; + std::string sph_particle_file; + std::string restart_particle_file; + +#ifdef AGN + std::string agn_particle_file; +#endif + +#ifdef NEUTRINO_PARTICLES + std::string neutrino_particle_file; +#endif + + // const std::string chk_particle_file("DM"); + const std::string dm_chk_particle_file("DM"); + const std::string agn_chk_particle_file("AGN"); + const std::string npc_chk_particle_file("NPC"); + + // + // We want to call this routine when on exit to clean up particles. + // + + // + // Array of containers for all active particles + // + Vector ActiveParticles; + // + // Array of containers for all virtual particles + // + Vector VirtualParticles; + // + // Array of containers for all ghost particles + // + Vector GhostParticles; + + // + // Containers for the real "active" Particles + // + DarkMatterParticleContainer* DMPC = 0; + StellarParticleContainer* SPC = 0; +#ifdef AGN + AGNParticleContainer* APC = 0; +#endif +#ifdef NEUTRINO_PARTICLES + +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer* NPC = 0; +#else + NeutrinoParticleContainer* NPC = 0; +#endif + +#endif + + // + // This is only used as a temporary container for + // reading in SPH particles and using them to + // initialize the density and velocity field on the + // grids. + // + DarkMatterParticleContainer* SPHPC = 0; + // + // Container for temporary, virtual Particles + // + DarkMatterParticleContainer* VirtPC = 0; + StellarParticleContainer* VirtSPC = 0; +#ifdef AGN + AGNParticleContainer* VirtAPC = 0; +#endif +#ifdef NEUTRINO_PARTICLES + +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer* VirtNPC = 0; +#else + NeutrinoParticleContainer* VirtNPC = 0; + +#endif +#endif + // + // Container for temporary, ghost Particles + // + DarkMatterParticleContainer* GhostPC = 0; + StellarParticleContainer* GhostSPC = 0; +#ifdef AGN + AGNParticleContainer* GhostAPC = 0; +#endif +#ifdef NEUTRINO_PARTICLES +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer* GhostNPC = 0; +#else + NeutrinoParticleContainer* GhostNPC = 0; +#endif +#endif + DarkMatterParticleContainer* ShellPC = 0; + + void RemoveParticlesOnExit () + { + for (int i = 0; i < ActiveParticles.size(); i++) + { + delete ActiveParticles[i]; + ActiveParticles[i] = 0; + } + for (int i = 0; i < GhostParticles.size(); i++) + { + delete GhostParticles[i]; + GhostParticles[i] = 0; + } + for (int i = 0; i < VirtualParticles.size(); i++) + { + delete VirtualParticles[i]; + VirtualParticles[i] = 0; + } + } +} + +bool Nyx::do_dm_particles = false; +int Nyx::num_particle_ghosts = 1; +int Nyx::particle_skip_factor = 1; + +std::string Nyx::particle_init_type = ""; + +// Allows us to output particles in the plotfile +// in either single (IEEE32) or double (NATIVE) precision. +// Particles are always written in double precision +// in the checkpoint files. + +bool Nyx::particle_initrandom_serialize = false; +Real Nyx::particle_initrandom_mass = -1; +Real Nyx::particle_initrandom_mass_total = 869658119634944.0; +long Nyx::particle_initrandom_count; +long Nyx::particle_initrandom_count_per_box; +int Nyx::particle_initrandom_iseed; +Real Nyx::particle_inituniform_mass; +Real Nyx::particle_inituniform_vx; +Real Nyx::particle_inituniform_vy; +Real Nyx::particle_inituniform_vz; + +int Nyx::particle_launch_ics = 1; + +int Nyx::particle_verbose = 1; +int Nyx::write_particle_density_at_init = 0; +Real Nyx::particle_cfl = 0.5; +#ifdef NEUTRINO_PARTICLES +Real Nyx::neutrino_cfl = 0.5; +#endif + +Vector& +Nyx::theActiveParticles () +{ + return ActiveParticles; +} + +Vector& +Nyx::theGhostParticles () +{ + return GhostParticles; +} + +Vector& +Nyx::theVirtualParticles () +{ + return VirtualParticles; +} + +DarkMatterParticleContainer* +Nyx::theDMPC () +{ + return DMPC; +} + +DarkMatterParticleContainer* +Nyx::theVirtPC () +{ + return VirtPC; +} +DarkMatterParticleContainer* +Nyx::theGhostPC () +{ + return GhostPC; +} + +StellarParticleContainer* +Nyx::theSPC () +{ + return SPC; +} +StellarParticleContainer* +Nyx::theVirtSPC () +{ + return VirtSPC; +} +StellarParticleContainer* +Nyx::theGhostSPC () +{ + return GhostSPC; +} + +#ifdef AGN +AGNParticleContainer* +Nyx::theAPC () +{ + return APC; +} +AGNParticleContainer* +Nyx::theVirtAPC () +{ + return VirtAPC; +} +AGNParticleContainer* +Nyx::theGhostAPC () +{ + return GhostAPC; +} +#endif + +#ifdef NEUTRINO_PARTICLES +#ifdef NEUTRINO_DARK_PARTICLES +DarkMatterParticleContainer* +Nyx::theNPC () +{ + return NPC; +} +DarkMatterParticleContainer* +Nyx::theVirtNPC () +{ + return VirtNPC; +} +DarkMatterParticleContainer* +Nyx::theGhostNPC () +{ + return GhostNPC; +} +#else +NeutrinoParticleContainer* +Nyx::theNPC () +{ + return NPC; +} +NeutrinoParticleContainer* +Nyx::theVirtNPC () +{ + return VirtNPC; +} +NeutrinoParticleContainer* +Nyx::theGhostNPC () +{ + return GhostNPC; +} +#endif +#endif + +DarkMatterParticleContainer* +Nyx::theShellPC () +{ + return ShellPC; +} + +void +Nyx::read_particle_params () +{ + ParmParse pp("nyx"); + pp.query("do_dm_particles", do_dm_particles); + if(!do_dm_particles) + return; +#ifdef AGN + pp.get("particle_init_type", particle_init_type); +#else + if (do_dm_particles) + { + pp.get("particle_init_type", particle_init_type); + pp.query("init_with_sph_particles", init_with_sph_particles); + } +#endif + + pp.query("particle_initrandom_serialize", particle_initrandom_serialize); + pp.query("particle_initrandom_count", particle_initrandom_count); + pp.query("particle_initrandom_count_per_box", particle_initrandom_count_per_box); + pp.query("particle_initrandom_mass", particle_initrandom_mass); + pp.query("particle_initrandom_mass_total", particle_initrandom_mass_total); + pp.query("particle_initrandom_iseed", particle_initrandom_iseed); + pp.query("particle_skip_factor", particle_skip_factor); + pp.query("ascii_particle_file", ascii_particle_file); + pp.query("particle_inituniform_mass", particle_inituniform_mass); + pp.query("particle_inituniform_vx", particle_inituniform_vx); + pp.query("particle_inituniform_vy", particle_inituniform_vy); + pp.query("particle_inituniform_vz", particle_inituniform_vz); + + pp.query("restart_particle_file", restart_particle_file); + + // Input error check + if (do_dm_particles && !ascii_particle_file.empty() && particle_init_type != "AsciiFile") + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not AsciiFile but you specified ascii_particle_file" << std::endl;; + amrex::Error(); + } + + pp.query("sph_particle_file", sph_particle_file); + + // Input error check + if (init_with_sph_particles != 1 && !sph_particle_file.empty()) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::init_with_sph_particles is not 1 but you specified sph_particle_file" << std::endl;; + amrex::Error(); + } + + // Input error check + if (init_with_sph_particles == 1 && sph_particle_file.empty()) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::init_with_sph_particles is 1 but you did not specify sph_particle_file" << std::endl;; + amrex::Error(); + } + + pp.query("binary_particle_file", binary_particle_file); + + // Input error check + if (!binary_particle_file.empty() && (particle_init_type != "BinaryFile" && + particle_init_type != "BinaryMetaFile" && + particle_init_type != "BinaryMortonFile")) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not BinaryFile, BinaryMetaFile, or BinaryMortonFile but you specified binary_particle_file" << std::endl; + amrex::Error(); + } + +#ifdef AGN + pp.query("agn_particle_file", agn_particle_file); + if (!agn_particle_file.empty() && particle_init_type != "AsciiFile") + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not AsciiFile but you specified agn_particle_file" << std::endl;; + amrex::Error(); + } +#endif + +#ifdef NEUTRINO_PARTICLES + pp.query("neutrino_particle_file", neutrino_particle_file); + if (!neutrino_particle_file.empty() && (particle_init_type != "AsciiFile"&& + particle_init_type != "BinaryMetaFile" && + particle_init_type != "BinaryFile" )) + { + if (ParallelDescriptor::IOProcessor()) + std::cerr << "ERROR::particle_init_type is not AsciiFile or BinaryFile but you specified neutrino_particle_file" << std::endl;; + amrex::Error(); + } +#endif + + pp.query("write_particle_density_at_init", write_particle_density_at_init); + + if((particle_initrandom_mass <= 0 && particle_initrandom_mass_total <= 0) + && (particle_init_type == "Random" || + particle_init_type == "RandomPerBox" || + particle_init_type == "RandomPerCell" )) + amrex::Abort("Starting random intialization with particles of non-positive mass"); + + // + // Control the verbosity of the Particle class + // + ParmParse ppp("particles"); + ppp.query("v", particle_verbose); + + // + // Set the cfl for particle motion (fraction of cell that a particle can + // move in a timestep). + // + ppp.query("cfl", particle_cfl); +#ifdef NEUTRINO_PARTICLES + ppp.query("neutrino_cfl", neutrino_cfl); +#endif +} + +void +Nyx::init_particles () +{ + BL_PROFILE("Nyx::init_particles()"); + + if (level > 0) + return; + + // + // Need to initialize particles before defining gravity. + // + if (do_dm_particles) + { + BL_ASSERT (DMPC == 0); + DMPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(DMPC); + if (false)//do_lightcone + ShellPC = new DarkMatterParticleContainer(parent); + + if (init_with_sph_particles == 1) + SPHPC = new DarkMatterParticleContainer(parent); + + if (parent->subCycle()) + { + VirtPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtPC); + + GhostPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostPC); } + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + DMPC->SetVerbose(particle_verbose); + + // Check particle mass for random setups +#ifdef AMREX_USE_SINGLE_PRECISION_PARTICLES + Real tol = 1e-6; +#else + Real tol = 1e-12; +#endif + + Real number_of_cells = Geom().ProbLength(0)/Geom().CellSize(0) * + Geom().ProbLength(1)/Geom().CellSize(1) * + Geom().ProbLength(2)/Geom().CellSize(2); + if(particle_initrandom_mass <= 0) { + particle_initrandom_mass = particle_initrandom_mass_total / number_of_cells; + if(verbose) + amrex::Print()<<"... setting particle_initrandom_mass to "< tol + && (particle_init_type == "Random" || + particle_init_type == "RandomPerBox" || + particle_init_type == "RandomPerCell" )) + amrex::Abort("Starting random intialization with particle_initrandom_mass_total / (n_cell)^3 != particle_initrandom_mass"); + + + DarkMatterParticleContainer::ParticleInitData pdata = {{particle_initrandom_mass}, {}, {}, {}}; + + if (particle_init_type == "Random") + { + if (particle_initrandom_count <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_count must be > 0"); + } + if (particle_initrandom_iseed <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_iseed must be > 0"); + } + + if (verbose) + { + amrex::Print() << "\nInitializing DM with cloud of " + << particle_initrandom_count + << " random particles with initial seed: " + << particle_initrandom_iseed << "\n\n"; + } + { + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitRandom(particle_initrandom_count, + particle_initrandom_iseed, pdata, + particle_initrandom_serialize); + } + + } + else if (particle_init_type == "RandomPerBox") + { + if (particle_initrandom_count_per_box <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_count_per_box must be > 0"); + } + if (particle_initrandom_iseed <= 0) + { + amrex::Abort("Nyx::init_particles(): particle_initrandom_iseed must be > 0"); + } + + if (verbose) + amrex::Print() << "\nInitializing DM with of " << particle_initrandom_count_per_box + << " random particles per box with initial seed: " + << particle_initrandom_iseed << "\n\n"; + + DMPC->InitRandomPerBox(particle_initrandom_count_per_box, + particle_initrandom_iseed, pdata); + + } + else if (particle_init_type == "RandomPerCell") + { + if (verbose) + amrex::Print() << "\nInitializing DM with 1 random particle per cell " << "\n"; + + int n_per_cell = 1; + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitNRandomPerCell(n_per_cell, pdata); + amrex::Gpu::Device::synchronize(); + + } + else if (particle_init_type == "OnePerCell") + { + if (verbose) + amrex::Print() << "\nInitializing DM with 1 uniform particle per cell " << "\n"; + + // int n_per_cell = 1; + DarkMatterParticleContainer::ParticleInitData pdata_vel = {{particle_inituniform_mass, particle_inituniform_vx, particle_inituniform_vy, particle_inituniform_vz},{},{},{}}; + DMPC->InitOnePerCell(0.5, 0.5, 0.5, pdata_vel); + } + else if (particle_init_type == "AsciiFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from \"" + << ascii_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Print() << "\nInitializing SPH particles from ascii \"" + << sph_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass and velocity. + // + DMPC->InitFromAsciiFile(ascii_particle_file, AMREX_SPACEDIM + 1); + if (init_with_sph_particles == 1) + SPHPC->InitFromAsciiFile(sph_particle_file, AMREX_SPACEDIM + 1); + } + else if (particle_init_type == "BinaryFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from \"" + << binary_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Print() << "\nInitializing SPH particles from binary \"" + << sph_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass and velocity. + // + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitFromBinaryFile(binary_particle_file, AMREX_SPACEDIM + 1); + if (init_with_sph_particles == 1) + SPHPC->InitFromBinaryFile(sph_particle_file, AMREX_SPACEDIM + 1); + + } + else if (particle_init_type == "BinaryMetaFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from meta file\"" + << binary_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Print() << "\nInitializing SPH particles from meta file\"" + << sph_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]` in each of the binary particle files. + // Here we're reading in the particle mass and velocity. + // + amrex::Gpu::LaunchSafeGuard lsg(particle_launch_ics); + DMPC->InitFromBinaryMetaFile(binary_particle_file, AMREX_SPACEDIM + 1); + if (init_with_sph_particles == 1) + SPHPC->InitFromBinaryMetaFile(sph_particle_file, AMREX_SPACEDIM + 1); + } + else if (particle_init_type == "BinaryMortonFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing DM particles from morton-ordered binary file\"" + << binary_particle_file << "\" ...\n\n"; + if (init_with_sph_particles == 1) + amrex::Error("Morton-ordered input is not supported for sph particles."); + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]` in each of the binary particle files. + // Here we're reading in the particle mass and velocity. + // + DMPC->InitFromBinaryMortonFile(binary_particle_file, + AMREX_SPACEDIM + 1, + particle_skip_factor); + } + else if (particle_init_type == "Restart") + { + DMPC->Restart(restart_particle_file, dm_chk_particle_file); + } + else + { + amrex::Error("not a valid input for nyx.particle_init_type"); + } + } +#ifdef AGN + { + // Note that we don't initialize any actual AGN particles here, we just create the container. + BL_ASSERT (APC == 0); + APC = new AGNParticleContainer(parent, num_particle_ghosts); + ActiveParticles.push_back(APC); + + if (parent->subCycle()) + { + VirtAPC = new AGNParticleContainer(parent, num_particle_ghosts); + VirtualParticles.push_back(VirtAPC); + + GhostAPC = new AGNParticleContainer(parent, num_particle_ghosts); + GhostParticles.push_back(GhostAPC); + } + // + // 2 gives more stuff than 1. + // + APC->SetVerbose(particle_verbose); + } +#endif +#ifdef NEUTRINO_PARTICLES + { + BL_ASSERT (NPC == 0); +#ifdef NEUTRINO_DARK_PARTICLES + NPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(NPC); +#else + NPC = new NeutrinoParticleContainer(parent); + ActiveParticles.push_back(NPC); + + // Set the relativistic flag to 1 so that the density will be gamma-weighted + // when returned by AssignDensity calls. + NPC->SetRelativistic(1); + + // We must set the value for csquared which is used in computing gamma = 1 / sqrt(1-vsq/csq) + // Obviously this value is just a place-holder for now. + NPC->SetCSquared(1.); + +#endif + + if (parent->subCycle()) + { +#ifdef NEUTRINO_DARK_PARTICLES + VirtNPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtNPC); +#else + VirtNPC = new NeutrinoParticleContainer(parent); + VirtualParticles.push_back(VirtNPC); + + // Set the relativistic flag to 1 so that the density will be gamma-weighted + // when returned by AssignDensity calls. + VirtNPC->SetRelativistic(1); + +#endif + +#ifdef NEUTRINO_DARK_PARTICLES + GhostNPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostNPC); + +#else + GhostNPC = new NeutrinoParticleContainer(parent); + GhostParticles.push_back(GhostNPC); + + // Set the relativistic flag to 1 so that the density will be gamma-weighted + // when returned by AssignDensity calls. + GhostNPC->SetRelativistic(1); +#endif + + } + // + // Make sure to call RemoveParticlesOnExit() on exit. + // (if do_dm_particles then we have already called ExecOnFinalize) + // + if (!do_dm_particles) + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + NPC->SetVerbose(particle_verbose); + if (particle_init_type == "AsciiFile") + { + if (verbose) + amrex::Print() << "\nInitializing Neutrino particles from \"" + << neutrino_particle_file << "\" ...\n\n"; + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass, velocity and angles. + // +#ifdef NEUTRINO_DARK_PARTICLES + NPC->InitFromAsciiFile(neutrino_particle_file, AMREX_SPACEDIM + 1); +#else + NPC->InitFromAsciiFile(neutrino_particle_file, 2*AMREX_SPACEDIM + 1); +#endif + } + else if (particle_init_type == "BinaryFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing Neutrino particles from \"" + << neutrino_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]`. Here we're reading in the particle + // mass and velocity. + // + NPC->InitFromBinaryFile(neutrino_particle_file, AMREX_SPACEDIM + 1); + } + else if (particle_init_type == "BinaryMetaFile") + { + if (verbose) + { + amrex::Print() << "\nInitializing NPC particles from meta file\"" + << neutrino_particle_file << "\" ...\n\n"; + } + // + // The second argument is how many Reals we read into `m_data[]` + // after reading in `m_pos[]` in each of the binary particle files. + // Here we're reading in the particle mass and velocity. + // + NPC->InitFromBinaryMetaFile(neutrino_particle_file, AMREX_SPACEDIM + 1); + } + else + { + amrex::Error("for right now we only init Neutrino particles with ascii or binary"); + } + } + + if (write_particle_density_at_init == 1) + { + + Vector > particle_mf;//(new MultiFab(grids,dmap,1,1)); + + DMPC->AssignDensity(particle_mf,0,1,0,0); + + writeMultiFabAsPlotFile("ParticleDensity", *particle_mf[0], "density"); + +#ifdef NEUTRINO_PARTICLES + Vector > particle_npc_mf;//(new MultiFab(grids,dmap,1,1)); + // DMPC->AssignDensitySingleLevel(particle_mf,0,1,0,0); + + NPC->AssignDensity(particle_npc_mf,0,1,0,0); + + writeMultiFabAsPlotFile("ParticleNPCDensity", *particle_npc_mf[0], "density"); +#endif + exit(0); + } + +#endif +} + +#ifndef NO_HYDRO +void +Nyx::init_santa_barbara (int init_sb_vels) +{ + BL_PROFILE("Nyx::init_santa_barbara()"); + Real cur_time = state[State_Type].curTime(); + Real a = old_a; + + BL_PROFILE_VAR("Nyx::init_santa_barbara()::part", CA_part); + amrex::Print() << "... time and comoving a when data is initialized at level " + << level << " " << cur_time << " " << a << '\n'; + + if (level == 0) + { + Real frac_for_hydro = comoving_OmB / comoving_OmM; + Real omfrac = 1.0 - frac_for_hydro; + + if ( (init_with_sph_particles == 0) && (frac_for_hydro != 1.0) ) { + DMPC->MultiplyParticleMass(level, omfrac); + } + + Vector > particle_mf(1); + if (init_sb_vels == 1) + { + if (init_with_sph_particles == 1) { + SPHPC->AssignDensityAndVels(particle_mf); + } else { + DMPC->AssignDensityAndVels(particle_mf); + } + + } else { + if (init_with_sph_particles == 1) { + SPHPC->AssignDensity(particle_mf); + } else { + DMPC->AssignDensity(particle_mf); + } + } + + // As soon as we have used the SPH particles to define the density + // and velocity on the grid, we can go ahead and destroy them. + if (init_with_sph_particles == 1) { + delete SPHPC; + } + + BL_PROFILE_VAR_STOP(CA_part); + BL_PROFILE_VAR("Nyx::init_santa_barbara()::avg", CA_avg); + + for (int lev = parent->finestLevel()-1; lev >= 0; lev--) + { + amrex::average_down(*particle_mf[lev+1], *particle_mf[lev], + parent->Geom(lev+1), parent->Geom(lev), 0, 1, + parent->refRatio(lev)); + } + BL_PROFILE_VAR_STOP(CA_avg); + BL_PROFILE_VAR("Nyx::init_santa_barbara()::partmf", CA_partmf); + // Only multiply the density, not the velocities + if (init_with_sph_particles == 0) + { + if (frac_for_hydro == 1.0) + { + particle_mf[level]->mult(0,0,1); + } + else + { + particle_mf[level]->mult(frac_for_hydro / omfrac,0,1); + } + } + BL_PROFILE_VAR_STOP(CA_partmf); + + BL_PROFILE_VAR("Nyx::init_santa_barbara()::init", CA_init); + const auto geomdata = geom.data(); + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + // Temp unused for GammaLaw, set it here so that pltfiles have + // defined numbers + D_new.setVal(0, Temp_comp); + D_new.setVal(0, Ne_comp); + +#ifdef _OPENMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + const auto fab_S_new=S_new.array(mfi); + const auto fab_D_new=D_new.array(mfi); + + GpuArray prob_param; + prob_param_fill(prob_param); + prob_param_special_fill(prob_param); + comoving_type=int(std::round(prob_param[comoving_type_comp])); + + prob_initdata_on_box(bx, fab_S_new, fab_D_new, geomdata, prob_param); + +// amrex::ParallelFor( +// bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { +// prob_initdata +// (i, j ,k, fab_S_new, fab_D_new, geomdata,prob_param); +// }); + } + + amrex::Gpu::Device::streamSynchronize(); + + if (inhomo_reion) init_zhi(); + + BL_PROFILE_VAR_STOP(CA_init); + + // Add the particle density to the gas density + MultiFab::Add(S_new, *particle_mf[level], 0, Density_comp, 1, 1); + + if (init_sb_vels == 1) + { + // Convert velocity to momentum + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + MultiFab::Multiply(*particle_mf[level], *particle_mf[level], 0, 1+i, 1, 0); + } + + // Add the particle momenta to the gas momenta (initially zero) + MultiFab::Add(S_new, *particle_mf[level], 1, Xmom_comp, AMREX_SPACEDIM, S_new.nGrow()); + } + + enforce_minimum_density_floor(S_new, new_a); + } else { + + MultiFab& S_new = get_new_data(State_Type); + FillCoarsePatch(S_new, 0, cur_time, State_Type, 0, S_new.nComp()); + + MultiFab& D_new = get_new_data(DiagEOS_Type); + FillCoarsePatch(D_new, 0, cur_time, DiagEOS_Type, 0, D_new.nComp()); + + MultiFab& Phi_new = get_new_data(PhiGrav_Type); + FillCoarsePatch(Phi_new, 0, cur_time, PhiGrav_Type, 0, Phi_new.nComp()); + +#ifndef CONST_SPECIES + // Convert (rho X)_i to X_i before calling init_e_from_T + for (int i = 0; i < NumSpec; ++i) + MultiFab::Divide(S_new, S_new, Density_comp, FirstSpec_comp+i, 1, 0); +#endif + } + + init_e_from_T(a); + +#ifndef CONST_SPECIES + // Convert X_i to (rho X)_i + MultiFab& S_new = get_new_data(State_Type); + for (int i = 0; i < NumSpec; ++i) + MultiFab::Multiply(S_new, S_new, Density_comp, FirstSpec_comp+i, 1, 0); +#endif +} +#endif + +void +Nyx::particle_post_restart (const std::string& restart_file, bool is_checkpoint) +{ + BL_PROFILE("Nyx::particle_post_restart()"); + + if (level > 0) + return; + + if (do_dm_particles) + { + BL_ASSERT(DMPC == 0); + DMPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(DMPC); + + if (parent->subCycle()) + { + VirtPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtPC); + + GhostPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostPC); + } + + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + + // + // 2 gives more stuff than 1. + // + DMPC->SetVerbose(particle_verbose); + + { + DMPC->Restart(restart_file, dm_chk_particle_file, is_checkpoint); + amrex::Gpu::Device::streamSynchronize(); + } + // + // We want the ability to write the particles out to an ascii file. + // + ParmParse pp("particles"); + + std::string dm_particle_output_file; + pp.query("dm_particle_output_file", dm_particle_output_file); + + if (!dm_particle_output_file.empty()) + { + DMPC->WriteAsciiFile(dm_particle_output_file); + } + } +#ifdef AGN + { + BL_ASSERT(APC == 0); + APC = new AGNParticleContainer(parent, num_particle_ghosts); + ActiveParticles.push_back(APC); + + if (parent->subCycle()) + { + VirtAPC = new AGNParticleContainer(parent, num_particle_ghosts); + VirtualParticles.push_back(VirtAPC); + + GhostAPC = new AGNParticleContainer(parent, num_particle_ghosts); + GhostParticles.push_back(GhostAPC); + } + + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + APC->SetVerbose(particle_verbose); + APC->Restart(restart_file, agn_chk_particle_file, is_checkpoint); + // + // We want the ability to write the particles out to an ascii file. + // + ParmParse pp("particles"); + + std::string agn_particle_output_file; + pp.query("agn_particle_output_file", agn_particle_output_file); + + if (!agn_particle_output_file.empty()) + { + APC->WriteAsciiFile(agn_particle_output_file); + } + } +#endif + +#ifdef NEUTRINO_DARK_PARTICLES + { + BL_ASSERT(NPC == 0); + NPC = new DarkMatterParticleContainer(parent); + ActiveParticles.push_back(NPC); + + if (parent->subCycle()) + { + VirtNPC = new DarkMatterParticleContainer(parent); + VirtualParticles.push_back(VirtNPC); + + GhostNPC = new DarkMatterParticleContainer(parent); + GhostParticles.push_back(GhostNPC); + } + + // + // Make sure to call RemoveParticlesOnExit() on exit. + // + amrex::ExecOnFinalize(RemoveParticlesOnExit); + // + // 2 gives more stuff than 1. + // + NPC->SetVerbose(particle_verbose); + NPC->Restart(restart_file, npc_chk_particle_file, is_checkpoint); + // + // We want the ability to write the particles out to an ascii file. + // + ParmParse pp("particles"); + + std::string npc_particle_output_file; + pp.query("npc_particle_output_file", npc_particle_output_file); + + if (!npc_particle_output_file.empty()) + { + NPC->WriteAsciiFile(npc_particle_output_file); + } + } +#endif +} + +void +Nyx::particle_est_time_step (Real& est_dt) +{ + BL_PROFILE("Nyx::particle_est_time_step()"); + const Real cur_time = state[PhiGrav_Type].curTime(); + const Real a = get_comoving_a(cur_time); + MultiFab& grav = get_new_data(Gravity_Type); + const Real est_dt_particle = DMPC->estTimestep(grav, a, level, particle_cfl); + + if (est_dt_particle > 0) { + est_dt = std::min(est_dt, est_dt_particle); + } + +#ifdef NEUTRINO_PARTICLES + const Real est_dt_neutrino = NPC->estTimestep(grav, a, level, neutrino_cfl); + if (est_dt_neutrino > 0) { + est_dt = std::min(est_dt, est_dt_neutrino); + } +#endif + + if (verbose) + { + if (est_dt_particle > 0) + { + amrex::Print() << "...estdt from particles at level " + << level << ": " << est_dt_particle << '\n'; + } + else + { + amrex::Print() << "...there are no particles at level " + << level << '\n'; + } +#ifdef NEUTRINO_PARTICLES + if (est_dt_neutrino > 0) + { + amrex::Print() << "...estdt from neutrinos at level " + << level << ": " << est_dt_neutrino << '\n'; + } +#endif + } +} + +void +Nyx::particle_redistribute (int lbase, bool my_init) +{ + BL_PROFILE("Nyx::particle_redistribute()"); + if (DMPC) + { + + + // + // If we are calling with my_init = true, then we want to force the redistribute + // without checking whether the grids have changed. + // + if (my_init) + { + DMPC->Redistribute(lbase); + return; + } + + // + // These are usually the BoxArray and DMap from the last regridding. + // + static Vector ba; + static Vector dm; + + bool changed = false; + bool dm_changed = false; + bool ba_changed = false; + bool ba_size_changed = false; + + int flev = parent->finestLevel(); + + while ( parent->getAmrLevels()[flev] == nullptr ) { + flev--; + } + + if (ba.size() != flev+1) + { + amrex::Print() << "BA SIZE " << ba.size() << std::endl; + amrex::Print() << "FLEV " << flev << std::endl; + ba.resize(flev+1); + dm.resize(flev+1); + changed = true; + ba_size_changed = true; + } + else + { + for (int i = 0; i <= flev && !changed; i++) + { + if (ba[i] != parent->boxArray(i)) + { + // + // The BoxArrays have changed in the regridding. + // + changed = true; + ba_changed = true; + } + + if ( ! changed) + { + if (dm[i] != parent->getLevel(i).get_new_data(0).DistributionMap()) + { + // + // The DistributionMaps have changed in the regridding. + // + changed = true; + dm_changed = true; + } + } + } + } + + if (changed) + { + // + // We only need to call Redistribute if the BoxArrays or DistMaps have changed. + // We also only call it for particles >= lbase. This is + // because of we called redistribute during a subcycle, there may be particles not in + // the proper position on coarser levels. + // + if (verbose) + { + if (ba_size_changed) + amrex::Print() << "Calling redistribute because the size of BoxArray changed " << '\n'; + else if (ba_changed) + amrex::Print() << "Calling redistribute because BoxArray changed " << '\n'; + else if (dm_changed) + amrex::Print() << "Calling redistribute because DistMap changed " << '\n'; + } + + int iteration = 1; + for (int i = 0; i < theActiveParticles().size(); i++) + { + theActiveParticles()[i]->Redistribute(lbase, + theActiveParticles()[i]->finestLevel(), + iteration); + + } + + // + // Use the new BoxArray and DistMap to define ba and dm for next time. + // + for (int i = 0; i <= flev; i++) + { + ba[i] = parent->boxArray(i); + dm[i] = parent->getLevel(i).get_new_data(0).DistributionMap(); + } + } + else + { + if (verbose) + amrex::Print() << "NOT calling redistribute because NOT changed " << '\n'; + } + } +} + +void +Nyx::setup_virtual_particles() +{ + BL_PROFILE("Nyx::setup_virtual_particles()"); + + if(Nyx::theDMPC() != 0 && !virtual_particles_set) + { + DarkMatterParticleContainer::AoS virts; + if (level < parent->finestLevel()) + { + get_level(level + 1).setup_virtual_particles(); + Nyx::theVirtPC()->CreateVirtualParticles(level+1, virts); + Nyx::theVirtPC()->AddParticlesAtLevel(virts, level); + Nyx::theDMPC()->CreateVirtualParticles(level+1, virts); + Nyx::theVirtPC()->AddParticlesAtLevel(virts, level); + } + virtual_particles_set = true; + } +} + +void +Nyx::remove_virtual_particles() +{ + BL_PROFILE("Nyx::remove_virtual_particles()"); + + for (int i = 0; i < VirtualParticles.size(); i++) + { + if (VirtualParticles[i] != 0) + VirtualParticles[i]->RemoveParticlesAtLevel(level); + virtual_particles_set = false; + } +} + +void +Nyx::setup_ghost_particles(int ngrow) +{ + BL_PROFILE("Nyx::setup_ghost_particles()"); + BL_ASSERT(level < parent->finestLevel()); + + if(Nyx::theDMPC() != 0) + { + DarkMatterParticleContainer::AoS ghosts; + Nyx::theDMPC()->CreateGhostParticles(level, ngrow, ghosts); + Nyx::theGhostPC()->AddParticlesAtLevel(ghosts, level+1, ngrow); + } +#ifdef AGN + if(Nyx::theAPC() != 0) + { + AGNParticleContainer::AoS ghosts; + Nyx::theAPC()->CreateGhostParticles(level, ngrow, ghosts); + Nyx::theGhostAPC()->AddParticlesAtLevel(ghosts, level+1, ngrow); + } +#endif +#ifdef NEUTRINO_PARTICLES + if(Nyx::theNPC() != 0) + { +#ifdef NEUTRINO_DARK_PARTICLES + DarkMatterParticleContainer::AoS ghosts; +#else + NeutrinoParticleContainer::AoS ghosts; +#endif + Nyx::theNPC()->CreateGhostParticles(level, ngrow, ghosts); + Nyx::theGhostNPC()->AddParticlesAtLevel(ghosts, level+1, ngrow); + } +#endif + amrex::Gpu::Device::streamSynchronize(); +} + +void +Nyx::remove_ghost_particles() +{ + BL_PROFILE("Nyx::remove_ghost_particles()"); + + for (int i = 0; i < GhostParticles.size(); i++) + { + if (GhostParticles[i] != 0) + GhostParticles[i]->RemoveParticlesAtLevel(level); + } +} + +//NyxParticleContainerBase::~NyxParticleContainerBase() {} diff --git a/Exec/HaloFinder_GPU/Nyx_advance.cpp b/Exec/HaloFinder_GPU/Nyx_advance.cpp new file mode 100644 index 000000000..e3d6fbb90 --- /dev/null +++ b/Exec/HaloFinder_GPU/Nyx_advance.cpp @@ -0,0 +1,635 @@ + +#include +#include +#include +#include +#include + +using namespace amrex; + +using std::string; + +std::vector shell_particles; + +Real +Nyx::advance (Real time, + Real dt, + int iteration, + int ncycle) + + // Arguments: + // time : the current simulation time + // dt : the timestep to advance (e.g., go from time to + // time + dt) + // iteration : where we are in the current AMR subcycle. Each + // level will take a number of steps to reach the + // final time of the coarser level below it. This + // counter starts at 1 + // ncycle : the number of subcycles at this level + +{ + + const std::string region_name = ncycle > 1 ? "R::Nyx::advance" : "R::Nyx::advance::STEP1"; + BL_PROFILE_REGION(region_name); + MultiFab::RegionTag amrlevel_tag("AmrLevel_Level_" + std::to_string(level)); + +#ifdef NO_HYDRO + + return advance_particles_only(time, dt, iteration, ncycle); + +#else + if (do_hydro) + { + if (Nyx::theActiveParticles().size() > 0) + { +#ifndef AGN + if (do_dm_particles) +#endif + { + return advance_hydro_plus_particles(time, dt, iteration, ncycle); + } + } + else + { + return advance_hydro(time, dt, iteration, ncycle); + } + } + else if(do_dm_particles) + return advance_particles_only(time, dt, iteration, ncycle); + else + return advance_heatcool(time, dt, iteration, ncycle); +#endif + return 0; +} + +// +// This will advance the Nyx AmrLevel. +// If no subcycling is used, this will be a full multilevel advance at +// at level 0 and a finer levels will be skipped over. +// If subcycling is used, this will check if finer levels are subcycled +// relative to this level. All levels that are not subcycled will be +// advanced in a multilevel advance with this level and be skipped over +// subsequently. If the next level is subcycled relative to this one, +// then this will only be a single level advance. +// +#ifndef NO_HYDRO +#ifdef AMREX_PARTICLES +Real +Nyx::advance_hydro_plus_particles (Real time, + Real dt, + int iteration, + int ncycle) +{ + + // A particle in cell (i) can affect cell values in (i-1) to (i+1) + int stencil_deposition_width = 1; + + // A particle in cell (i) may need information from cell values in (i-1) to (i+1) + // to update its position (typically via interpolation of the acceleration from the grid) + int stencil_interpolation_width = 1; + + // A particle that starts in cell (i + ncycle) can reach + // cell (i) in ncycle number of steps .. after "iteration" steps + // the particle has to be within (i + ncycle+1-iteration) to reach cell (i) + // in the remaining (ncycle-iteration) steps + + // *** ghost_width *** is used + // *) to set how many cells are used to hold ghost particles i.e copies of particles + // that live on (level-1) can affect the grid over all of the ncycle steps. + // We define ghost cells at the coarser level to cover all iterations so + // we can't reduce this number as iteration increases. + + int ghost_width = ncycle + stencil_deposition_width; + + // *** where_width *** is used + // *) to set how many cells the Where call in moveKickDrift tests = + // ghost_width + (1-iteration) - 1: + // the minus 1 arises because this occurs *after* the move + + int where_width = ghost_width + (1-iteration) - 1; + + // *** grav_n_grow *** is used + // *) to determine how many ghost cells we need to fill in the MultiFab from + // which the particle interpolates its acceleration + // *) to set how many cells the Where call in moveKickDrift tests = (grav.nGrow()-2). + // *) the (1-iteration) arises because the ghost particles are created on the coarser + // level which means in iteration 2 the ghost particles may have moved 1 additional cell along + + int grav_n_grow = ghost_width + (1-iteration) + (iteration-1) + + stencil_interpolation_width ; + + { + BL_PROFILE_REGION("R::Nyx::advance_hydro_plus_particles"); + BL_PROFILE("Nyx::advance_hydro_plus_particles()"); + // Sanity checks + if (!do_hydro) + amrex::Abort("In `advance_hydro_plus_particles` but `do_hydro` not true"); + + if (Nyx::theActiveParticles().size() <= 0) + amrex::Abort("In `advance_hydro_plus_particles` but no active particles"); + + const int finest_level = parent->finestLevel(); + int finest_level_to_advance; + bool nosub = !parent->subCycle(); + + if (nosub) + { + if (level > 0) + return dt; + + finest_level_to_advance = finest_level; + } + else + { + // This level was advanced by a previous multilevel advance. + if (level > 0 && ncycle == 1) + return dt; + + // Find the finest level to advance + int lev = level; + while(lev < finest_level && parent->nCycle(lev+1) == 1) + lev++; + finest_level_to_advance = lev; + + // We must setup virtual and Ghost Particles + // + // Setup the virtual particles that represent finer level particles + // + setup_virtual_particles(); + // + // Setup ghost particles for use in finer levels. Note that Ghost particles + // that will be used by this level have already been created, the + // particles being set here are only used by finer levels. + // + for(int lev = level; lev <= finest_level_to_advance && lev < finest_level; lev++) + { + get_level(lev).setup_ghost_particles(ghost_width); + } + } + + Real dt_lev; + + // + // Move current data to previous, clear current. + // Don't do this if a coarser level has done this already. + // + if (level == 0 || iteration > 1) + { + + for (int lev = level; lev <= finest_level; lev++) + { + dt_lev = parent->dtLevel(lev); + for (int k = 0; k < NUM_STATE_TYPE; k++) + { + get_level(lev).state[k].allocOldData(); + get_level(lev).state[k].swapTimeLevels(dt_lev); + } + } + } + + const Real prev_time = state[State_Type].prevTime(); + const Real cur_time = state[State_Type].curTime(); + const Real a_old = get_comoving_a(prev_time); + const Real a_new = get_comoving_a(cur_time); + + if (do_grav) { + // + // We now do a multilevel solve for old Gravity. This goes to the + // finest level regardless of subcycling behavior. Consequentially, + // If we are subcycling we skip this step on the first iteration of + // finer levels. + BL_PROFILE_VAR("solve_for_old_phi", solve_for_old_phi); + if (level == 0 || iteration > 1) + { + + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + + // fix fluxes on finer grids + if (do_reflux) + { + for (int lev = level; lev < finest_level; lev++) + { + gravity->zero_phi_flux_reg(lev + 1); + } + } + + // swap grav data + for (int lev = level; lev <= finest_level; lev++) + get_level(lev).gravity->swap_time_levels(lev); + + // + // Solve for phi using the previous phi as a guess. + // + int use_previous_phi_as_guess = 1; + int ngrow_for_solve = iteration + stencil_deposition_width; + gravity->multilevel_solve_for_old_phi(level, finest_level, + ngrow_for_solve, + use_previous_phi_as_guess); + } + BL_PROFILE_VAR_STOP(solve_for_old_phi); + + { + // + // Advance Particles + // + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities to the half-time and the positions to the new time + // We use the cell-centered gravity to correctly interpolate onto particle locations + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKickDrift ... updating particle positions and velocity\n"; + + MultiFab::RegionTag amrMoveKickDrift_tag("MoveKickDrift_" + std::to_string(level)); + + Vector shell_particles; + shell_particles.clear(); + shell_particles.shrink_to_fit(); + + std::string filename=Concatenate("lightcone_", int(100*(1/a_old-1)), 7); + std::string filename_vtk="./Output/LightCones/VTK/"+filename+".vtk"; + std::string filename_bin="./Output/LightCones/SimpleBinary/"+filename+".bin"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + // We need grav_n_grow grow cells to track boundary particles + const auto& ba = get_level(lev).get_new_data(State_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(State_Type).DistributionMap(); + MultiFab grav_vec_old(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_old_grav_vector(lev, grav_vec_old, time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) { + Real radius_inner = -1.e34; + Real radius_outer = -1.e34; + Real z_old = 1/a_old-1; + if(z_old=lightcone_end_z) { + integrate_distance_given_a(a_old, 1.0, radius_outer); + integrate_distance_given_a(a_new, 1.0, radius_inner); + Print()<moveKickDrift(grav_vec_old, lev, time, dt, + a_old, a_half, where_width, + radius_inner, radius_outer, + &shell_particles); + } + Real z_old = 1/a_old-1; + if(z_old=lightcone_end_z) { + //writeBinaryVTK(filename_vtk, shell_particles); + writeBinarySimple(filename_bin, shell_particles); + } + + + // Only need the coarsest virtual particles here. + if (lev == level && level < finest_level) + for (int i = 0; i < Nyx::theVirtualParticles().size(); i++) + Nyx::theVirtualParticles()[i]->moveKickDrift(grav_vec_old, lev, time, dt, a_old, a_half, where_width); + + // Miiiight need all Ghosts + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKickDrift(grav_vec_old, lev, time, dt, a_old, a_half, where_width); +#ifdef NEUTRINO_DARK_PARTICLES + for (int i = 0; i < Nyx::theActiveParticles().size() && neutrino_cfl >= 1; i++) + Nyx::theActiveParticles()[i]->Redistribute(); +#endif + } + } // if active particles + } // lsg + } // if (do_grav) + + // + // Call the hydro advance at each level to be advanced + // + BL_PROFILE_VAR("just_the_hydro", just_the_hydro); + { + + MultiFab::RegionTag amrhydro_tag("Hydro_" + std::to_string(level)); + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { +#ifdef SDC + if (sdc_split > 0 && (strang_restart_from_sdc == 0)) { + get_level(lev).sdc_hydro(time, dt, a_old, a_new); + } else { + get_level(lev).strang_hydro(time, dt, a_old, a_new); + } +#else + get_level(lev).strang_hydro(time, dt, a_old, a_new); +#endif + } + } + BL_PROFILE_VAR_STOP(just_the_hydro); + + { + // + // We must reflux before doing the next gravity solve + // + if (do_reflux) + { + for (int lev = level; lev < finest_level_to_advance; lev++) + { + get_level(lev).reflux(); + } + } + + // Always average down the new state from finer to coarser. + for (int lev = finest_level_to_advance-1; lev >= level; lev--) + { + get_level(lev).average_down( State_Type); + get_level(lev).average_down(DiagEOS_Type); + } + + // + // Here we use the "old" phi from the current time step as a guess for this + // solve + // + if (do_grav) + { + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::Copy(parent->getLevel(lev).get_new_data(PhiGrav_Type), + parent->getLevel(lev).get_old_data(PhiGrav_Type), + 0, 0, 1, 0); + } + + // Solve for new Gravity + BL_PROFILE_VAR("solve_for_new_phi", solve_for_new_phi); + int use_previous_phi_as_guess = 1; + if (finest_level_to_advance > level) + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + // The particle may be as many as "iteration" ghost cells out + int ngrow_for_solve = iteration + stencil_deposition_width; + gravity->multilevel_solve_for_new_phi(level, finest_level_to_advance, + ngrow_for_solve, + use_previous_phi_as_guess); + } + else + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + int fill_interior = 0; + gravity->solve_for_new_phi(level,get_new_data(PhiGrav_Type), + gravity->get_grad_phi_curr(level), + fill_interior, grav_n_grow); + } + BL_PROFILE_VAR_STOP(solve_for_new_phi); + + // Reflux + if (do_reflux) + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(level)); + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + gravity->add_to_fluxes(lev, iteration, ncycle); + } + } + + // + // Now do corrector part of source term update + // + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::RegionTag amrGrav_tag("Gravity_" + std::to_string(lev)); + + // Now do corrector part of source term update + correct_gsrc(lev,time,prev_time,cur_time,dt); + + MultiFab& S_new = get_level(lev).get_new_data(State_Type); + MultiFab& D_new = get_level(lev).get_new_data(DiagEOS_Type); + + // First reset internal energy before call to compute_temp + MultiFab reset_e_src(S_new.boxArray(), S_new.DistributionMap(), 1, NUM_GROW); + reset_e_src.setVal(0.0); + get_level(lev).reset_internal_energy(S_new,D_new,reset_e_src); + + get_level(lev).compute_new_temp(S_new,D_new); + } + + // Must average down again after doing the gravity correction; + // always average down from finer to coarser. + // Here we average down both the new state and phi and gravity. + for (int lev = finest_level_to_advance-1; lev >= level; lev--) + get_level(lev).average_down(); + + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities by dt/2 to the new time. We use the + // cell-centered gravity to correctly interpolate onto particle + // locations. + MultiFab::RegionTag amrMoveKickDrift_tag("MoveKick_" + std::to_string(level)); + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKick ... updating velocity only\n"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + const auto& ba = get_level(lev).get_new_data(State_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(State_Type).DistributionMap(); + MultiFab grav_vec_new(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_new_grav_vector(lev, grav_vec_new, cur_time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) + Nyx::theActiveParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + + // Virtual particles will be recreated, so we need not kick them. + + // Ghost particles need to be kicked except during the final iteration. + if (iteration != ncycle) + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + } + } + } // do_grav + + // + // Synchronize Energies + // + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::RegionTag amrReset_tag("Reset_" + std::to_string(lev)); + MultiFab& S_new = get_level(lev).get_new_data(State_Type); + MultiFab& D_new = get_level(lev).get_new_data(DiagEOS_Type); + + get_level(lev).reset_internal_energy_nostore(S_new,D_new); + + } + } + } + // BL_PROFILE_REGION_STOP("R::Nyx::advance_hydro_plus_particles"); + + // Redistribution happens in post_timestep + return dt; +} +#endif + +Real +Nyx::advance_heatcool (Real time, + Real dt, + int iteration, + int ncycle) +{ + BL_PROFILE("Nyx::advance_heatcool()"); +#ifdef HEATCOOL + amrex::Print()<<"Using advance_heatcool since hydro and dm_particles are both off"<finestLevel(); + if (do_reflux && level < finest_level) + gravity->zero_phi_flux_reg(level + 1); + gravity->swap_time_levels(level); + } + + if (do_forcing) + forcing->evolve(dt); + + // Call the hydro advance itself + BL_PROFILE_VAR("just_the_hydro", just_the_hydro); +#ifdef SDC + if (sdc_split > 0) + { + sdc_hydro(time, dt, a_old, a_new); + } else { + strang_hydro(time, dt, a_old, a_new); + } +#else + strang_hydro(time, dt, a_old, a_new); +#endif + BL_PROFILE_VAR_STOP(just_the_hydro); + + if (do_grav) + { + if (verbose && ParallelDescriptor::IOProcessor()) + std::cout << "\n... new-time level solve at level " << level << '\n'; + + // + // Solve for new phi + // Here we use the "old" phi from the current time step as a guess for this solve + // + MultiFab::Copy(get_new_data(PhiGrav_Type),get_old_data(PhiGrav_Type),0,0,1,0); + int fill_interior = 0; + gravity->solve_for_new_phi(level,get_new_data(PhiGrav_Type), + gravity->get_grad_phi_curr(level),fill_interior); + if (do_reflux) + gravity->add_to_fluxes(level,iteration,ncycle); + + // Now do corrector part of source term update + correct_gsrc(level,time,prev_time,cur_time,dt); + } + + MultiFab& S_new = get_new_data(State_Type); + MultiFab& D_new = get_new_data(DiagEOS_Type); + + MultiFab reset_e_src(S_new.boxArray(), S_new.DistributionMap(), 1, NUM_GROW); + reset_e_src.setVal(0.0); + + // First reset internal energy before call to compute_temp + reset_internal_energy(S_new,D_new,reset_e_src); + compute_new_temp(S_new,D_new); + + return dt; +} +#endif diff --git a/Exec/HaloFinder_GPU/Nyx_advance_particles.cpp b/Exec/HaloFinder_GPU/Nyx_advance_particles.cpp new file mode 100644 index 000000000..f3095ddc0 --- /dev/null +++ b/Exec/HaloFinder_GPU/Nyx_advance_particles.cpp @@ -0,0 +1,282 @@ +#include +#include +#include + +using namespace amrex; + +using std::string; + +Real +Nyx::advance_particles_only (Real time, + Real dt, + int iteration, + int ncycle) + + // Arguments: + // time : the current simulation time + // dt : the timestep to advance (e.g., go from time to + // time + dt) + // iteration : where we are in the current AMR subcycle. Each + // level will take a number of steps to reach the + // final time of the coarser level below it. This + // counter starts at 1 + // ncycle : the number of subcycles at this level + +{ + BL_PROFILE("Nyx::advance_particles_only()"); + + // A particle in cell (i) can affect cell values in (i-1) to (i+1) + int stencil_deposition_width = 1; + + // A particle in cell (i) may need information from cell values in (i-1) to (i+1) + // to update its position (typically via interpolation of the acceleration from the grid) + int stencil_interpolation_width = 1; + + // A particle that starts in cell (i + ncycle) can reach + // cell (i) in ncycle number of steps .. after "iteration" steps + // the particle has to be within (i + ncycle+1-iteration) to reach cell (i) + // in the remaining (ncycle-iteration) steps + + // *** ghost_width *** is used + // *) to set how many cells are used to hold ghost particles i.e copies of particles + // that live on (level-1) can affect the grid over all of the ncycle steps. + // We define ghost cells at the coarser level to cover all iterations so + // we can't reduce this number as iteration increases. + + int ghost_width = ncycle + stencil_deposition_width; + + // *** where_width *** is used + // *) to set how many cells the Where call in moveKickDrift tests = + // ghost_width + (1-iteration) - 1: + // the minus 1 arises because this occurs *after* the move + + int where_width = ghost_width + (1-iteration) - 1; + + // *** grav_n_grow *** is used + // *) to determine how many ghost cells we need to fill in the MultiFab from + // which the particle interpolates its acceleration + // *) to set how many cells the Where call in moveKickDrift tests = (grav.nGrow()-2). + // *) the (1-iteration) arises because the ghost particles are created on the coarser + // level which means in iteration 2 the ghost particles may have moved 1 additional cell along + + int grav_n_grow = ghost_width + (1-iteration) + (iteration-1) + + stencil_interpolation_width ; + + // Sanity checks + if (do_hydro) + amrex::Abort("In `advance_particles_only` but `do_hydro` is true"); + + if (!do_grav) + amrex::Abort("In `advance_particles_only` but `do_grav` not true"); + const int finest_level = parent->finestLevel(); + int finest_level_to_advance; + bool nosub = !parent->subCycle(); + + if (nosub) + { + if (level > 0) + return dt; + + finest_level_to_advance = finest_level; + } + else + { + // This level was advanced by a previous multilevel advance. + if (level > 0 && ncycle == 1) + return dt; + + // Find the finest level to advance + int lev = level; + while(lev < finest_level && parent->nCycle(lev+1) == 1) + lev++; + finest_level_to_advance = lev; + + // We must setup virtual and Ghost Particles + // + // Setup the virtual particles that represent finer level particles + // + setup_virtual_particles(); + // + // Setup ghost particles for use in finer levels. Note that Ghost particles + // that will be used by this level have already been created, the + // particles being set here are only used by finer levels. + // + for(int lev = level; lev <= finest_level_to_advance && lev < finest_level; lev++) + { + get_level(lev).setup_ghost_particles(ghost_width); + } + } + + Real dt_lev; + + // + // Move current data to previous, clear current. + // Don't do this if a coarser level has done this already. + // + if (level == 0 || iteration > 1) + { + for (int lev = level; lev <= finest_level; lev++) + { + dt_lev = parent->dtLevel(lev); + for (int k = 0; k < NUM_STATE_TYPE; k++) + { + get_level(lev).state[k].allocOldData(); + get_level(lev).state[k].swapTimeLevels(dt_lev); + } +#ifndef NO_HYDRO + if(do_hydro) + { + MultiFab& S_old = get_level(lev).get_old_data(State_Type); + MultiFab& S_new = get_level(lev).get_new_data(State_Type); + MultiFab::Copy(S_new, S_old, 0, 0, S_old.nComp(), 0); + } +#endif + } + } + + const Real prev_time = state[PhiGrav_Type].prevTime(); + const Real cur_time = state[PhiGrav_Type].curTime(); + + const Real a_old = get_comoving_a(prev_time); + const Real a_new = get_comoving_a(cur_time); + + // + // We now do a multilevel solve for old Gravity. This goes to the + // finest level regardless of subcycling behavior. Consequentially, + // If we are subcycling we skip this step on the first iteration of + // finer levels. + if (level == 0 || iteration > 1) + { + // fix fluxes on finer grids + if (do_reflux) + { + for (int lev = level; lev < finest_level; lev++) + { + gravity->zero_phi_flux_reg(lev + 1); + } + } + + // swap grav data + for (int lev = level; lev <= finest_level; lev++) + get_level(lev).gravity->swap_time_levels(lev); + + // + // Solve for phi + // If a single-level calculation we can still use the previous phi as a guess. + // TODO: Check this. + int use_previous_phi_as_guess = 1; + gravity->multilevel_solve_for_old_phi(level, finest_level, + use_previous_phi_as_guess); + } + + { + // + // Advance Particles + // + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities to the half-time and the positions to the new time + // We use the cell-centered gravity to correctly interpolate onto particle locations + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKickDrift ... updating particle positions and velocity\n"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + // We need grav_n_grow grow cells to track boundary particles + const auto& ba = get_level(lev).get_new_data(PhiGrav_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(PhiGrav_Type).DistributionMap(); + MultiFab grav_vec_old(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_old_grav_vector(lev, grav_vec_old, time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) { + Real radius_inner = -1.e34; + Real radius_outer = -1.e34; + Real z_old = 1/a_old-1; + if(z_old=lightcone_end_z) { + integrate_distance_given_a(a_old, 1.0, radius_outer); + integrate_distance_given_a(a_new, 1.0, radius_inner); + Print()<moveKickDrift(grav_vec_old, lev, time, dt, a_old, a_half, where_width, radius_inner, radius_outer); + } + + // Only need the coarsest virtual particles here. + if (lev == level && level < finest_level) + for (int i = 0; i < Nyx::theVirtualParticles().size(); i++) + Nyx::theVirtualParticles()[i]->moveKickDrift(grav_vec_old, level, time, dt, a_old, a_half, where_width); + + // Miiiight need all Ghosts + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKickDrift(grav_vec_old, lev, time, dt, a_new, a_half, where_width); + +#ifdef NEUTRINO_DARK_PARTICLES + for (int i = 0; i < Nyx::theActiveParticles().size() && neutrino_cfl >= 1; i++) + Nyx::theActiveParticles()[i]->Redistribute(); +#endif + } + } + + // + // Here we use the "old" phi from the current time step as a guess for this + // solve + // + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + MultiFab::Copy(parent->getLevel(lev).get_new_data(PhiGrav_Type), + parent->getLevel(lev).get_old_data(PhiGrav_Type), + 0, 0, 1, 0); + } + } + + // Solve for new Gravity + int use_previous_phi_as_guess = 1; + if (finest_level_to_advance > level) + { + gravity->multilevel_solve_for_new_phi(level, finest_level_to_advance, + use_previous_phi_as_guess); + } + else + { + int fill_interior = 0; + gravity->solve_for_new_phi(level,get_new_data(PhiGrav_Type), + gravity->get_grad_phi_curr(level), + fill_interior, grav_n_grow); + } + + { + if (Nyx::theActiveParticles().size() > 0) + { + // Advance the particle velocities by dt/2 to the new time. We use the + // cell-centered gravity to correctly interpolate onto particle + // locations. + const Real a_half = 0.5 * (a_old + a_new); + + if (particle_verbose && ParallelDescriptor::IOProcessor()) + std::cout << "moveKick ... updating velocity only\n"; + + for (int lev = level; lev <= finest_level_to_advance; lev++) + { + const auto& ba = get_level(lev).get_new_data(PhiGrav_Type).boxArray(); + const auto& dm = get_level(lev).get_new_data(PhiGrav_Type).DistributionMap(); + MultiFab grav_vec_new(ba, dm, AMREX_SPACEDIM, grav_n_grow); + get_level(lev).gravity->get_new_grav_vector(lev, grav_vec_new, cur_time); + + for (int i = 0; i < Nyx::theActiveParticles().size(); i++) + Nyx::theActiveParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + + // Virtual particles will be recreated, so we need not kick them. + + // Ghost particles need to be kicked except during the final iteration. + if (iteration != ncycle) + for (int i = 0; i < Nyx::theGhostParticles().size(); i++) + Nyx::theGhostParticles()[i]->moveKick(grav_vec_new, lev, time, dt, a_new, a_half); + } + } + } + + // Redistribution happens in post_timestep + return dt; +} diff --git a/Exec/HaloFinder_GPU/Prob.cpp b/Exec/HaloFinder_GPU/Prob.cpp new file mode 100644 index 000000000..221f7bf74 --- /dev/null +++ b/Exec/HaloFinder_GPU/Prob.cpp @@ -0,0 +1,97 @@ +#include "Nyx.H" +#include "Prob.H" + +void prob_param_special_fill(amrex::GpuArray& prob_param) +{} + +#ifndef NO_HYDRO +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void prob_initdata_state(const int i, + const int j, + const int k, + amrex::Array4 const& state, + amrex::GeometryData const& geomdata, + const amrex::GpuArray& prob_param) +{ + // This is the case where we have compiled with states defined + // but they have only one component each so we fill them this way. + if (state.nComp() == 1) + { + //Could be replaced with setVal + state(i,j,k,0) = 0.00; + } +#ifndef NO_HYDRO + // This is the regular case with NO_HYDRO = FALSE + else if (state.nComp() > 1) + { + state(i,j,k,Density_comp) = 0.00; //1.5d0 * small_dens + state(i,j,k,Xmom_comp) = 0.00; + state(i,j,k,Ymom_comp) = 0.00; + state(i,j,k,Zmom_comp) = 0.00; + + // These will both be set later in the call to init_e. + state(i,j,k,Eint_comp) = 0.0; + state(i,j,k,Eden_comp) = 0.0; + +#ifndef CONST_SPECIES + state(i,j,k,FirstSpec_comp ) = prob_param[ h_species_comp]; + state(i,j,k,FirstSpec_comp+1) = prob_param[he_species_comp]; +#endif + } +#endif +} + +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void prob_initdata(const int i, + const int j, + const int k, + amrex::Array4 const& state, + amrex::Array4 const& diag_eos, + amrex::GeometryData const& geomdata, + const amrex::GpuArray& prob_param) +{ + // This is the case where we have compiled with states defined + // but they have only one component each so we fill them this way. + if (state.nComp() == 1 && diag_eos.nComp() == 1) + { + //Could be replaced with setVal + diag_eos(i,j,k,0) = 0.00; + } +#ifndef NO_HYDRO + // This is the regular case with NO_HYDRO = FALSE + else if (state.nComp() > 1 && diag_eos.nComp() >= 2) + { + diag_eos(i,j,k,Temp_comp) = 0.0210*(1.00 + prob_param[z_in_comp])*(1.00 + prob_param[z_in_comp]); + diag_eos(i,j,k, Ne_comp) = 0.0; + + //Should be equivalent to inhomo_reion > 0 Nyx_setup.cpp + if (diag_eos.nComp() > 2) + diag_eos(i,j,k, Zhi_comp) = 7.5; + } +#endif +} + +void prob_initdata_on_box(const Box& bx, + Array4 const& state, + Array4 const& diag_eos, + GeometryData const& geomdata, + const GpuArray& prob_param) +{ + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + prob_initdata_state(i, j ,k, state, geomdata, prob_param); + prob_initdata (i, j ,k, state, diag_eos, geomdata, prob_param); + }); +} + +void prob_initdata_state_on_box(const Box& bx, + Array4 const& state, + GeometryData const& geomdata, + const GpuArray& prob_param) +{ + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + prob_initdata_state(i, j ,k, state, geomdata, prob_param); + }); +} +#endif diff --git a/Exec/HaloFinder_GPU/Prob_error.cpp b/Exec/HaloFinder_GPU/Prob_error.cpp new file mode 100644 index 000000000..e0e0503d3 --- /dev/null +++ b/Exec/HaloFinder_GPU/Prob_error.cpp @@ -0,0 +1,11 @@ +#include "Nyx.H" +#include "Prob.H" + +void prob_errtags_default(amrex::Vector& errtags) +{ + //Only include default tagging if NO_HYDRO=FALSE +#ifndef NO_HYDRO + AMRErrorTagInfo info; + errtags.push_back(AMRErrorTag(3.5e9,AMRErrorTag::GREATER,"denvol",info)); +#endif +} diff --git a/Exec/HaloFinder_GPU/ReadHalos.py b/Exec/HaloFinder_GPU/ReadHalos.py new file mode 100644 index 000000000..60c403806 --- /dev/null +++ b/Exec/HaloFinder_GPU/ReadHalos.py @@ -0,0 +1,55 @@ +import numpy as np +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +import sys +import os + +def read_binary_points(filename): + with open(filename, 'rb') as file: + data = np.fromfile(file, dtype='>f4') # big-endian float32 + + if len(data) % 5 != 0: + raise ValueError("Data size is not a multiple of 5. The file might be corrupted or improperly formatted.") + + points = data.reshape((-1, 5)) + num_points = len(points) + print(f"Total number of points: {num_points}") + + return points, num_points + +def plot_points(x, y, z): + fig = plt.figure(figsize=(10, 7)) + ax = fig.add_subplot(111, projection='3d') + ax.scatter(x, y, z, c='blue', marker='o', s=1, alpha=0.8) + ax.set_xlabel('X Coordinate') + ax.set_ylabel('Y Coordinate') + ax.set_zlabel('Z Coordinate') + ax.set_title('Lightcone Shell') + plt.show() + +def main(): + if len(sys.argv) != 2: + print(f"Usage: python {os.path.basename(sys.argv[0])} ") + sys.exit(1) + + filename = sys.argv[1] + + if not os.path.isfile(filename): + print(f"Error: File '{filename}' not found.") + sys.exit(1) + + points, num_points = read_binary_points(filename) + + # Extract coordinates + x = np.zeros(num_points) + y = np.zeros(num_points) + z = np.zeros(num_points) + + for i, point in enumerate(points): + x[i], y[i], z[i], mass, n_cells = point + print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {mass:.15g}, {n_cells:.15g}") + + plot_points(x, y, z) + +if __name__ == "__main__": + main() diff --git a/Exec/HaloFinder_GPU/ReadLightConeBinary.py b/Exec/HaloFinder_GPU/ReadLightConeBinary.py new file mode 100644 index 000000000..01b8b54a0 --- /dev/null +++ b/Exec/HaloFinder_GPU/ReadLightConeBinary.py @@ -0,0 +1,59 @@ +import numpy as np + +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D + +def read_binary_points(filename): + with open(filename, 'rb') as file: + # Read the binary data directly, assuming big-endian float32 + data = np.fromfile(file, dtype='>f4') # '>f4' indicates big-endian float32 + + # Reshape the data to Nx3 (x, y, z for each point) + if len(data) % 6 != 0: + raise ValueError("Data size is not a multiple of 6. The file might be corrupted or improperly formatted.") + + points = data.reshape((-1, 6)) + + num_points = len(data) // 6 + print(f"Total number of points: {num_points}") + + return points, num_points + +def plot_points(x, y, z): + fig = plt.figure(figsize=(10, 7)) + ax = fig.add_subplot(111, projection='3d') + + # Scatter plot for points + ax.scatter(x, y, z, c='blue', marker='o', s=1, alpha=0.8) + + # Set labels for the axes + ax.set_xlabel('X Coordinate') + ax.set_ylabel('Y Coordinate') + ax.set_zlabel('Z Coordinate') + + ax.set_title('Lightcone shell') + plt.show() + + +# Example usage +filename = "./Output/LightCones/SimpleBinary/lightcone_0004899.bin" +points, num_points = read_binary_points(filename) + +# Display the first few points +x = np.zeros(num_points); +y = np.zeros(num_points); +z = np.zeros(num_points); + +xcen = 3850.0; +ycen = 3850.0; +zcen = 3850.0; + +print("num points is ", num_points); + + +for i, point in enumerate(points[:num_points]): # Adjust the slice as needed + x[i], y[i], z[i], vx, vy, vz = point + rad = np.sqrt((x[i] - xcen) * (x[i] - xcen) + (y[i] - ycen) * (y[i] - ycen) + (z[i] - zcen) * (z[i] - zcen)); + print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {vx:.15g}, {vy:.15g}, {vz:.15g}, {rad:.15g}") + +plot_points(x, y, z) diff --git a/Exec/HaloFinder_GPU/Read_BinarySimple.py b/Exec/HaloFinder_GPU/Read_BinarySimple.py new file mode 100644 index 000000000..29ee1dc28 --- /dev/null +++ b/Exec/HaloFinder_GPU/Read_BinarySimple.py @@ -0,0 +1,57 @@ +import numpy as np + +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D + +def read_binary_points(filename): + with open(filename, 'rb') as file: + # Read the binary data directly, assuming big-endian float32 + data = np.fromfile(file, dtype='>f4') # '>f4' indicates big-endian float32 + + # Reshape the data to Nx3 (x, y, z for each point) + if len(data) % 6 != 0: + raise ValueError("Data size is not a multiple of 6. The file might be corrupted or improperly formatted.") + + points = data.reshape((-1, 6)) + + num_points = len(data) // 6 + print(f"Total number of points: {num_points}") + + return points, num_points + +def plot_points(x, y, z): + fig = plt.figure(figsize=(10, 7)) + ax = fig.add_subplot(111, projection='3d') + + # Scatter plot for points + ax.scatter(x, y, z, c='blue', marker='o', s=1, alpha=0.8) + + # Set labels for the axes + ax.set_xlabel('X Coordinate') + ax.set_ylabel('Y Coordinate') + ax.set_zlabel('Z Coordinate') + + ax.set_title('Lightcone shell') + plt.show() + + +# Example usage +filename = "lightcone_0000030.bin" +points, num_points = read_binary_points(filename) + +# Display the first few points +x = np.zeros(num_points); +y = np.zeros(num_points); +z = np.zeros(num_points); +xcen = 3850.0; +ycen = 3850.0; +zcen = 3850.0; +for i, point in enumerate(points[:num_points]): # Adjust the slice as needed + x[i], y[i], z[i], vx, vy, vz = point + rad = np.sqrt((x[i] - xcen) * (x[i] - xcen) + (y[i] - ycen) * (y[i] - ycen) + (z[i] - zcen) * (z[i] - zcen)); + print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {vx:.15g}, {vy:.15g}, {vz:.15g}, {rad:.15g}") + #print(f"{x[i]:.15g}, {y[i]:.15g}, {z[i]:.15g}, {vx:.15g}, {vy:.15g}, {vz:.15g}") + +plot_points(x, y, z) + + diff --git a/Exec/HaloFinder_GPU/TREECOOL_middle b/Exec/HaloFinder_GPU/TREECOOL_middle new file mode 100644 index 000000000..c548b2ec3 --- /dev/null +++ b/Exec/HaloFinder_GPU/TREECOOL_middle @@ -0,0 +1,301 @@ +0.000000 5.700000e-14 3.100000e-14 1.121650e-16 3.560837e-25 4.486095e-25 5.008400e-27 +0.021189 7.131077e-14 3.942314e-14 1.290508e-16 4.465957e-25 5.631802e-25 5.728569e-27 +0.041393 8.817069e-14 4.881653e-14 1.564290e-16 5.546459e-25 6.943841e-25 6.874023e-27 +0.060698 1.080520e-13 6.036742e-14 1.892055e-16 6.806021e-25 8.499327e-25 8.214962e-27 +0.079181 1.313927e-13 7.381091e-14 2.281519e-16 8.287477e-25 1.030083e-24 9.775263e-27 +0.096910 1.574751e-13 8.920400e-14 2.740300e-16 9.950685e-25 1.237482e-24 1.157747e-26 +0.113943 1.870916e-13 1.066446e-13 3.274889e-16 1.184974e-24 1.471890e-24 1.363982e-26 +0.130334 2.201403e-13 1.260925e-13 3.893120e-16 1.397618e-24 1.732952e-24 1.598488e-26 +0.146128 2.558537e-13 1.472511e-13 4.603516e-16 1.627616e-24 2.017452e-24 1.863681e-26 +0.161368 2.977649e-13 1.718511e-13 5.410824e-16 1.893292e-24 2.347358e-24 2.160698e-26 +0.176091 3.428995e-13 1.987396e-13 6.320781e-16 2.183324e-24 2.708677e-24 2.490966e-26 +0.190332 3.912293e-13 2.276695e-13 7.337759e-16 2.493622e-24 3.095318e-24 2.855453e-26 +0.204120 4.463107e-13 2.603846e-13 8.467451e-16 2.838601e-24 3.523196e-24 3.255534e-26 +0.217484 5.046292e-13 2.943905e-13 9.710171e-16 3.208211e-24 3.984673e-24 3.690837e-26 +0.230449 5.642777e-13 3.291591e-13 1.106241e-15 3.588006e-24 4.460281e-24 4.159783e-26 +0.243038 6.309768e-13 3.700881e-13 1.252896e-15 4.013032e-24 4.994599e-24 4.663438e-26 +0.255273 7.003212e-13 4.118066e-13 1.409787e-15 4.451985e-24 5.544954e-24 5.197583e-26 +0.267172 7.734476e-13 4.548221e-13 1.577197e-15 4.911441e-24 6.119276e-24 5.762626e-26 +0.278754 8.509889e-13 5.012864e-13 1.753469e-15 5.402579e-24 6.737064e-24 6.352996e-26 +0.290035 9.292096e-13 5.484886e-13 1.937662e-15 5.899552e-24 7.363681e-24 6.965391e-26 +0.301030 1.014903e-12 6.006439e-13 2.128940e-15 6.455638e-24 8.031408e-24 7.596822e-26 +0.311754 1.100000e-12 6.524997e-13 2.325220e-15 7.009520e-24 8.691806e-24 8.240606e-26 +0.322219 1.122007e-12 6.649298e-13 2.524680e-15 7.152800e-24 8.855698e-24 8.890910e-26 +0.332438 1.137809e-12 6.737611e-13 2.725227e-15 7.256136e-24 8.971874e-24 9.541154e-26 +0.342423 1.145552e-12 6.779476e-13 2.924501e-15 7.292276e-24 9.018775e-24 1.018411e-25 +0.352183 1.154360e-12 6.828133e-13 3.119812e-15 7.336837e-24 9.075818e-24 1.081175e-25 +0.361728 1.162594e-12 6.880686e-13 3.308180e-15 7.389919e-24 9.129358e-24 1.141534e-25 +0.371068 1.173550e-12 6.950557e-13 3.486392e-15 7.462784e-24 9.205408e-24 1.198566e-25 +0.380211 1.187979e-12 7.039418e-13 3.651000e-15 7.551576e-24 9.306058e-24 1.251293e-25 +0.389166 1.198634e-12 7.104930e-13 3.797474e-15 7.612901e-24 9.375911e-24 1.298449e-25 +0.397940 1.207275e-12 7.158272e-13 3.924431e-15 7.664101e-24 9.427544e-24 1.339643e-25 +0.406540 1.213105e-12 7.194667e-13 4.033722e-15 7.705948e-24 9.444720e-24 1.375349e-25 +0.414973 1.216342e-12 7.215532e-13 4.127912e-15 7.730928e-24 9.444007e-24 1.406245e-25 +0.423246 1.219054e-12 7.229159e-13 4.207614e-15 7.750458e-24 9.437258e-24 1.432369e-25 +0.431364 1.219149e-12 7.226487e-13 4.274178e-15 7.752770e-24 9.411277e-24 1.454154e-25 +0.439333 1.221622e-12 7.238018e-13 4.326827e-15 7.768263e-24 9.401333e-24 1.471344e-25 +0.447158 1.218366e-12 7.215375e-13 4.364575e-15 7.740295e-24 9.333136e-24 1.483636e-25 +0.454845 1.222231e-12 7.235103e-13 4.388926e-15 7.757998e-24 9.322051e-24 1.491523e-25 +0.462398 1.220561e-12 7.219019e-13 4.397603e-15 7.745418e-24 9.272144e-24 1.494253e-25 +0.469822 1.209901e-12 7.148017e-13 4.393478e-15 7.678796e-24 9.156752e-24 1.492791e-25 +0.477121 1.199432e-12 7.078630e-13 4.374113e-15 7.613336e-24 9.044882e-24 1.486322e-25 +0.484300 1.193909e-12 7.040435e-13 4.342413e-15 7.575579e-24 8.958947e-24 1.475807e-25 +0.491362 1.190000e-12 7.012368e-13 4.297156e-15 7.547275e-24 8.883588e-24 1.460851e-25 +0.498311 1.178629e-12 6.940542e-13 4.239794e-15 7.471795e-24 8.754437e-24 1.441924e-25 +0.505150 1.160855e-12 6.828258e-13 4.171646e-15 7.359487e-24 8.569961e-24 1.419469e-25 +0.511883 1.150522e-12 6.759325e-13 4.091439e-15 7.295230e-24 8.440136e-24 1.393048e-25 +0.518514 1.136042e-12 6.666341e-13 4.002124e-15 7.204632e-24 8.281918e-24 1.363635e-25 +0.525045 1.123528e-12 6.584897e-13 3.904306e-15 7.121947e-24 8.137490e-24 1.331418e-25 +0.531479 1.115864e-12 6.531880e-13 3.797318e-15 7.067108e-24 8.027796e-24 1.296183e-25 +0.537819 1.095126e-12 6.402568e-13 3.683704e-15 6.929647e-24 7.825678e-24 1.258756e-25 +0.544068 1.072765e-12 6.264631e-13 3.564253e-15 6.785092e-24 7.612582e-24 1.219386e-25 +0.550228 1.061742e-12 6.193867e-13 3.439655e-15 6.716654e-24 7.478913e-24 1.178297e-25 +0.556303 1.046184e-12 6.096776e-13 3.310320e-15 6.619508e-24 7.314394e-24 1.135611e-25 +0.562293 1.036077e-12 6.031568e-13 3.177940e-15 6.556823e-24 7.189026e-24 1.091889e-25 +0.568202 1.022056e-12 5.945533e-13 3.043303e-15 6.470033e-24 7.035863e-24 1.047366e-25 +0.574031 1.005956e-12 5.847470e-13 2.907188e-15 6.370046e-24 6.869274e-24 1.002312e-25 +0.579784 9.912223e-13 5.757424e-13 2.770379e-15 6.278681e-24 6.712955e-24 9.569790e-26 +0.585461 9.680384e-13 5.618682e-13 2.633502e-15 6.132892e-24 6.504294e-24 9.115552e-26 +0.591065 9.544885e-13 5.536352e-13 2.497385e-15 6.046844e-24 6.366674e-24 8.663305e-26 +0.596597 9.306970e-13 5.394692e-13 2.362745e-15 5.895920e-24 6.161646e-24 8.215325e-26 +0.602060 9.143101e-13 5.296034e-13 2.230116e-15 5.791908e-24 6.006705e-24 7.773418e-26 +0.607455 8.991008e-13 5.202923e-13 6.307590e-16 5.694157e-24 5.863423e-24 3.786485e-26 +0.612784 8.823489e-13 5.100061e-13 3.181031e-16 5.585858e-24 5.712746e-24 2.016526e-26 +0.618048 8.751899e-13 5.052654e-13 2.134829e-16 5.538291e-24 5.624235e-24 1.429888e-26 +0.623249 8.637173e-13 4.980317e-13 1.608417e-16 5.463416e-24 5.507808e-24 1.139137e-26 +0.628389 8.528019e-13 4.915248e-13 1.289660e-16 5.396334e-24 5.407943e-24 1.052143e-26 +0.633468 8.428606e-13 4.856804e-13 1.037051e-16 5.336487e-24 5.317448e-24 9.806648e-27 +0.638489 8.382619e-13 4.829132e-13 8.621439e-17 5.310497e-24 5.260379e-24 8.760179e-27 +0.643453 8.336445e-13 4.801335e-13 7.325023e-17 5.284443e-24 5.202720e-24 8.019994e-27 +0.648360 8.267178e-13 4.759967e-13 6.313902e-17 5.244351e-24 5.141382e-24 7.475048e-27 +0.653213 8.196424e-13 4.717560e-13 5.492792e-17 5.203728e-24 5.086392e-24 6.126590e-27 +0.658011 8.135775e-13 4.680948e-13 5.019306e-17 5.169572e-24 5.037542e-24 5.049079e-27 +0.662758 8.065499e-13 4.638777e-13 4.592255e-17 5.129351e-24 4.982595e-24 4.934280e-27 +0.667453 8.027916e-13 4.615450e-13 4.201102e-17 5.109119e-24 4.948612e-24 4.841405e-27 +0.672098 7.961321e-13 4.575691e-13 3.837703e-17 5.066971e-24 4.900394e-24 4.766620e-27 +0.676694 7.886928e-13 4.531439e-13 3.495599e-17 5.019863e-24 4.847315e-24 3.977646e-27 +0.681241 7.815721e-13 4.489005e-13 3.301013e-17 4.974785e-24 4.796134e-24 3.250959e-27 +0.685742 7.747680e-13 4.448374e-13 3.108054e-17 4.931723e-24 4.746826e-24 3.256814e-27 +0.690196 7.690636e-13 4.415922e-13 2.915607e-17 4.898378e-24 4.709738e-24 3.265612e-27 +0.694605 7.642091e-13 4.389615e-13 2.722633e-17 4.872297e-24 4.681562e-24 3.277108e-27 +0.698970 7.569034e-13 4.349242e-13 2.528152e-17 4.830632e-24 4.638383e-24 2.779312e-27 +0.703291 7.496390e-13 4.309116e-13 2.415028e-17 4.789259e-24 4.595466e-24 2.292495e-27 +0.707570 7.426188e-13 4.270404e-13 2.297740e-17 4.749480e-24 4.554057e-24 2.318147e-27 +0.711807 7.358775e-13 4.234541e-13 2.175971e-17 4.710133e-24 4.516890e-24 2.344472e-27 +0.716003 7.254846e-13 4.177967e-13 2.049395e-17 4.647062e-24 4.457929e-24 2.371451e-27 +0.720159 7.152903e-13 4.122517e-13 1.917677e-17 4.585242e-24 4.400158e-24 2.060401e-27 +0.724276 7.051901e-13 4.067590e-13 1.834401e-17 4.524005e-24 4.342938e-24 1.747728e-27 +0.728354 6.953706e-13 4.014265e-13 1.746429e-17 4.464550e-24 4.287421e-24 1.774039e-27 +0.732394 6.824911e-13 3.941022e-13 1.653568e-17 4.386103e-24 4.210623e-24 1.800666e-27 +0.736397 6.764605e-13 3.906762e-13 1.555619e-17 4.351803e-24 4.175464e-24 1.827609e-27 +0.740363 6.671800e-13 3.853730e-13 1.452373e-17 4.296577e-24 4.120238e-24 1.582864e-27 +0.744293 6.580179e-13 3.801378e-13 1.387060e-17 4.242072e-24 4.065723e-24 1.333318e-27 +0.748188 6.544022e-13 3.781067e-13 1.317574e-17 4.223321e-24 4.045477e-24 1.355995e-27 +0.752048 6.481107e-13 3.742073e-13 1.243778e-17 4.185266e-24 4.006934e-24 1.378892e-27 +0.755875 6.441856e-13 3.714595e-13 1.165528e-17 4.161120e-24 3.981845e-24 1.402009e-27 +0.759668 6.390398e-13 3.680078e-13 1.082679e-17 4.129089e-24 3.949210e-24 1.255493e-27 +0.763428 6.353209e-13 3.653779e-13 1.022687e-17 4.106278e-24 3.925395e-24 1.104912e-27 +0.767156 6.297119e-13 3.616611e-13 9.589602e-18 4.071249e-24 3.889899e-24 1.124369e-27 +0.770852 6.265970e-13 3.595083e-13 8.913869e-18 4.052667e-24 3.870932e-24 1.144009e-27 +0.774517 6.224705e-13 3.573013e-13 8.198515e-18 4.028825e-24 3.850199e-24 1.163833e-27 +0.778151 6.193681e-13 3.556826e-13 7.442358e-18 4.011619e-24 3.835816e-24 1.005099e-27 +0.781755 6.051062e-13 3.476532e-13 6.941549e-18 3.922097e-24 3.752257e-24 8.413854e-28 +0.785330 5.877877e-13 3.378617e-13 6.410901e-18 3.812656e-24 3.649568e-24 8.565593e-28 +0.788875 5.676714e-13 3.264544e-13 5.849590e-18 3.684930e-24 3.529277e-24 8.718795e-28 +0.792392 5.495041e-13 3.167119e-13 5.256778e-18 3.571577e-24 3.427850e-24 8.873461e-28 +0.795880 5.363643e-13 3.106727e-13 4.631610e-18 3.493560e-24 3.367904e-24 7.118943e-28 +0.799341 5.169618e-13 3.009645e-13 4.299132e-18 3.374552e-24 3.268043e-24 5.304775e-28 +0.802774 4.995822e-13 2.923781e-13 3.947483e-18 3.268479e-24 3.180164e-24 5.402246e-28 +0.806180 4.877927e-13 2.870282e-13 3.576173e-18 3.198812e-24 3.127367e-24 5.500706e-28 +0.809560 4.759080e-13 2.816036e-13 3.184706e-18 3.128429e-24 3.073691e-24 5.600157e-28 +0.812913 4.623742e-13 2.752620e-13 2.772580e-18 3.047217e-24 3.011775e-24 4.368394e-28 +0.816241 4.520541e-13 2.711678e-13 2.571902e-18 2.987705e-24 2.981882e-24 3.092253e-28 +0.819544 4.443370e-13 2.686641e-13 2.360065e-18 2.945518e-24 2.969691e-24 3.148907e-28 +0.822822 4.345795e-13 2.649609e-13 2.136800e-18 2.889947e-24 2.944484e-24 3.206155e-28 +0.826075 4.272450e-13 2.627744e-13 1.901832e-18 2.850653e-24 2.936413e-24 3.263997e-28 +0.829304 4.200690e-13 2.607429e-13 1.654885e-18 2.812654e-24 2.930479e-24 2.576582e-28 +0.832509 4.130429e-13 2.588692e-13 1.528682e-18 2.775929e-24 2.926771e-24 1.863881e-28 +0.835691 4.062667e-13 2.568159e-13 1.395797e-18 2.739122e-24 2.916111e-24 1.897383e-28 +0.838849 3.995945e-13 2.547881e-13 1.256074e-18 2.702749e-24 2.904734e-24 1.931235e-28 +0.841985 3.930652e-13 2.529187e-13 1.109356e-18 2.667606e-24 2.895517e-24 1.965440e-28 +0.845098 3.847176e-13 2.499429e-13 9.554854e-19 2.620375e-24 2.873967e-24 1.518516e-28 +0.848189 3.667928e-13 2.407400e-13 8.818350e-19 2.507890e-24 2.780786e-24 1.055345e-28 +0.851258 3.528143e-13 2.340834e-13 8.044452e-19 2.422216e-24 2.716796e-24 1.073845e-28 +0.854306 3.393126e-13 2.275320e-13 7.232325e-19 2.338935e-24 2.654234e-24 1.092536e-28 +0.857332 3.274144e-13 2.212584e-13 6.381125e-19 2.263375e-24 2.596005e-24 1.111419e-28 +0.860338 3.155981e-13 2.150280e-13 5.490001e-19 2.188337e-24 2.538177e-24 8.680585e-29 +0.863323 3.038634e-13 2.088406e-13 5.043857e-19 2.113816e-24 2.480749e-24 6.160051e-29 +0.866287 2.922076e-13 2.026948e-13 4.576116e-19 2.039797e-24 2.423706e-24 6.264938e-29 +0.869232 2.806318e-13 1.965913e-13 4.086313e-19 1.966287e-24 2.367056e-24 6.370885e-29 +0.872156 2.691342e-13 1.905289e-13 3.573978e-19 1.893271e-24 2.310788e-24 6.477896e-29 +0.875061 2.581002e-13 1.846475e-13 3.038639e-19 1.822827e-24 2.254911e-24 4.829715e-29 +0.877947 2.486829e-13 1.793655e-13 2.810482e-19 1.761161e-24 2.199478e-24 3.124579e-29 +0.880814 2.393272e-13 1.741182e-13 2.571572e-19 1.699900e-24 2.144408e-24 3.176211e-29 +0.883661 2.300330e-13 1.689053e-13 2.321684e-19 1.639041e-24 2.089699e-24 3.228351e-29 +0.886491 2.207996e-13 1.637267e-13 2.060592e-19 1.578580e-24 2.035351e-24 3.281002e-29 +0.889302 2.116261e-13 1.585815e-13 1.788069e-19 1.518513e-24 1.981352e-24 2.619701e-29 +0.892095 2.025115e-13 1.534694e-13 1.640604e-19 1.458830e-24 1.927701e-24 1.935953e-29 +0.894870 1.934552e-13 1.483901e-13 1.486533e-19 1.399530e-24 1.874395e-24 1.966965e-29 +0.897627 1.860132e-13 1.437805e-13 1.325724e-19 1.349538e-24 1.824350e-24 1.998274e-29 +0.900367 1.790063e-13 1.393093e-13 1.158044e-19 1.302091e-24 1.775350e-24 2.029880e-29 +0.903090 1.720436e-13 1.348662e-13 9.833555e-20 1.254941e-24 1.726661e-24 2.061784e-29 +0.905796 1.651244e-13 1.304507e-13 8.015240e-20 1.208085e-24 1.678274e-24 2.093989e-29 +0.908485 1.582476e-13 1.260626e-13 6.124118e-20 1.161519e-24 1.630186e-24 2.126493e-29 +0.911158 1.514133e-13 1.217015e-13 4.158803e-20 1.115240e-24 1.582395e-24 2.159299e-29 +0.913814 1.446210e-13 1.173673e-13 2.117900e-20 1.069246e-24 1.534897e-24 2.192407e-29 +0.916454 1.378694e-13 1.130588e-13 0.000000e+00 1.023525e-24 1.487683e-24 0.000000e+00 +0.919078 1.326935e-13 1.093997e-13 0.000000e+00 9.870744e-25 1.444549e-24 0.000000e+00 +0.921686 1.275485e-13 1.057624e-13 0.000000e+00 9.508408e-25 1.401672e-24 0.000000e+00 +0.924279 1.224342e-13 1.021468e-13 0.000000e+00 9.148230e-25 1.359051e-24 0.000000e+00 +0.926857 1.173502e-13 9.855261e-14 0.000000e+00 8.790185e-25 1.316683e-24 0.000000e+00 +0.929419 1.122961e-13 9.497961e-14 0.000000e+00 8.434253e-25 1.274565e-24 0.000000e+00 +0.931966 1.072719e-13 9.142766e-14 0.000000e+00 8.080418e-25 1.232694e-24 0.000000e+00 +0.934498 1.022771e-13 8.789657e-14 0.000000e+00 7.728660e-25 1.191069e-24 0.000000e+00 +0.937016 9.731088e-14 8.438562e-14 0.000000e+00 7.378909e-25 1.149682e-24 0.000000e+00 +0.939519 9.371704e-14 8.160445e-14 0.000000e+00 7.115605e-25 1.114961e-24 0.000000e+00 +0.942008 9.014364e-14 7.883910e-14 0.000000e+00 6.853799e-25 1.080437e-24 0.000000e+00 +0.944483 8.659053e-14 7.608948e-14 0.000000e+00 6.593480e-25 1.046108e-24 0.000000e+00 +0.946943 8.305762e-14 7.335546e-14 0.000000e+00 6.334640e-25 1.011975e-24 0.000000e+00 +0.949390 7.954452e-14 7.063678e-14 0.000000e+00 6.077250e-25 9.780345e-25 0.000000e+00 +0.951823 7.605111e-14 6.793334e-14 0.000000e+00 5.821304e-25 9.442834e-25 0.000000e+00 +0.954243 7.257733e-14 6.524508e-14 0.000000e+00 5.566797e-25 9.107219e-25 0.000000e+00 +0.956649 6.912279e-14 6.257172e-14 0.000000e+00 5.313699e-25 8.773464e-25 0.000000e+00 +0.959041 6.628733e-14 6.031097e-14 0.000000e+00 5.105714e-25 8.483019e-25 0.000000e+00 +0.961421 6.386646e-14 5.832717e-14 0.000000e+00 4.927940e-25 8.221747e-25 0.000000e+00 +0.963788 6.145870e-14 5.635417e-14 0.000000e+00 4.751132e-25 7.961892e-25 0.000000e+00 +0.966142 5.906407e-14 5.439189e-14 0.000000e+00 4.575287e-25 7.703451e-25 0.000000e+00 +0.968483 5.668231e-14 5.244018e-14 0.000000e+00 4.400386e-25 7.446403e-25 0.000000e+00 +0.970812 3.834510e-14 4.402700e-14 0.000000e+00 2.595247e-25 0.000000e+00 0.000000e+00 +0.973128 4.188557e-15 4.802124e-15 0.000000e+00 6.018460e-26 0.000000e+00 0.000000e+00 +0.975432 1.865695e-15 2.134670e-15 0.000000e+00 3.180310e-26 0.000000e+00 0.000000e+00 +0.977724 1.174019e-15 1.340458e-15 0.000000e+00 2.064471e-26 0.000000e+00 0.000000e+00 +0.980003 8.471102e-16 9.651459e-16 0.000000e+00 1.519086e-26 0.000000e+00 0.000000e+00 +0.982271 6.565596e-16 7.464321e-16 0.000000e+00 1.200737e-26 0.000000e+00 0.000000e+00 +0.984527 5.315749e-16 6.030196e-16 0.000000e+00 9.923108e-27 0.000000e+00 0.000000e+00 +0.986772 4.431324e-16 5.015781e-16 0.000000e+00 8.451805e-27 0.000000e+00 0.000000e+00 +0.989005 3.771366e-16 4.259206e-16 0.000000e+00 7.356993e-27 0.000000e+00 0.000000e+00 +0.991226 3.259163e-16 3.672376e-16 0.000000e+00 6.509960e-27 0.000000e+00 0.000000e+00 +0.993436 2.849390e-16 3.203243e-16 0.000000e+00 5.834653e-27 0.000000e+00 0.000000e+00 +0.995635 2.513551e-16 2.819082e-16 0.000000e+00 5.283248e-27 0.000000e+00 0.000000e+00 +0.997823 2.232830e-16 2.498288e-16 0.000000e+00 4.824163e-27 0.000000e+00 0.000000e+00 +1.000000 1.994308e-16 2.226022e-16 0.000000e+00 4.435703e-27 0.000000e+00 0.000000e+00 +1.002166 1.788819e-16 1.991761e-16 0.000000e+00 4.102474e-27 0.000000e+00 0.000000e+00 +1.004321 1.609682e-16 1.787834e-16 0.000000e+00 3.813247e-27 0.000000e+00 0.000000e+00 +1.006466 1.451916e-16 1.608518e-16 0.000000e+00 3.559640e-27 0.000000e+00 0.000000e+00 +1.008600 1.311728e-16 1.449461e-16 0.000000e+00 3.335270e-27 0.000000e+00 0.000000e+00 +1.010724 1.186187e-16 1.307295e-16 0.000000e+00 3.135191e-27 0.000000e+00 0.000000e+00 +1.012837 1.072987e-16 1.179375e-16 0.000000e+00 2.955509e-27 0.000000e+00 0.000000e+00 +1.014940 9.702979e-17 1.063595e-16 0.000000e+00 2.743582e-27 0.000000e+00 0.000000e+00 +1.017033 8.802750e-17 9.623278e-17 0.000000e+00 2.495225e-27 0.000000e+00 0.000000e+00 +1.019116 8.019337e-17 8.744034e-17 0.000000e+00 2.264206e-27 0.000000e+00 0.000000e+00 +1.021189 7.332056e-17 7.974478e-17 0.000000e+00 2.063345e-27 0.000000e+00 0.000000e+00 +1.023252 6.724845e-17 7.296182e-17 0.000000e+00 1.887445e-27 0.000000e+00 0.000000e+00 +1.025306 6.185044e-17 6.694622e-17 0.000000e+00 1.732417e-27 0.000000e+00 0.000000e+00 +1.027350 5.702533e-17 6.158191e-17 0.000000e+00 1.595002e-27 0.000000e+00 0.000000e+00 +1.029384 5.269128e-17 5.677508e-17 0.000000e+00 1.472571e-27 0.000000e+00 0.000000e+00 +1.031408 4.878134e-17 5.244901e-17 0.000000e+00 1.362982e-27 0.000000e+00 0.000000e+00 +1.033424 4.524024e-17 4.854037e-17 0.000000e+00 1.264471e-27 0.000000e+00 0.000000e+00 +1.035430 4.202191e-17 4.499644e-17 0.000000e+00 1.175576e-27 0.000000e+00 0.000000e+00 +1.037426 3.908766e-17 4.177292e-17 0.000000e+00 1.095070e-27 0.000000e+00 0.000000e+00 +1.039414 3.640472e-17 3.883237e-17 0.000000e+00 1.021924e-27 0.000000e+00 0.000000e+00 +1.041393 2.545887e-17 2.710715e-17 0.000000e+00 9.552614e-28 0.000000e+00 0.000000e+00 +1.043362 2.376379e-17 2.525780e-17 0.000000e+00 8.943373e-28 0.000000e+00 0.000000e+00 +1.045323 2.220278e-17 2.355856e-17 0.000000e+00 8.385110e-28 0.000000e+00 0.000000e+00 +1.047275 2.076240e-17 2.199408e-17 0.000000e+00 7.872305e-28 0.000000e+00 0.000000e+00 +1.049218 1.943092e-17 2.055102e-17 0.000000e+00 7.400175e-28 0.000000e+00 0.000000e+00 +1.051153 1.819806e-17 1.921769e-17 0.000000e+00 6.964559e-28 0.000000e+00 0.000000e+00 +1.053078 1.705478e-17 1.798381e-17 0.000000e+00 6.561819e-28 0.000000e+00 0.000000e+00 +1.054996 1.599309e-17 1.684033e-17 0.000000e+00 6.188765e-28 0.000000e+00 0.000000e+00 +1.056905 1.500588e-17 1.577919e-17 0.000000e+00 5.842588e-28 0.000000e+00 0.000000e+00 +1.058805 1.408684e-17 1.479325e-17 0.000000e+00 5.520808e-28 0.000000e+00 0.000000e+00 +1.060698 1.323032e-17 1.387614e-17 0.000000e+00 5.221225e-28 0.000000e+00 0.000000e+00 +1.062582 1.243127e-17 1.302215e-17 0.000000e+00 4.941885e-28 0.000000e+00 0.000000e+00 +1.064458 1.168513e-17 1.222615e-17 0.000000e+00 4.681045e-28 0.000000e+00 0.000000e+00 +1.066326 1.098779e-17 1.148352e-17 0.000000e+00 4.437148e-28 0.000000e+00 0.000000e+00 +1.068186 1.033555e-17 1.079010e-17 0.000000e+00 4.208798e-28 0.000000e+00 0.000000e+00 +1.070038 9.725022e-18 1.014211e-17 0.000000e+00 3.994742e-28 0.000000e+00 0.000000e+00 +1.071882 9.153161e-18 9.536141e-18 0.000000e+00 3.793849e-28 0.000000e+00 0.000000e+00 +1.073718 8.617175e-18 8.969074e-18 0.000000e+00 3.605101e-28 0.000000e+00 0.000000e+00 +1.075547 8.114517e-18 8.438075e-18 0.000000e+00 3.427577e-28 0.000000e+00 0.000000e+00 +1.077368 7.642860e-18 7.940556e-18 0.000000e+00 3.260441e-28 0.000000e+00 0.000000e+00 +1.079181 7.200066e-18 7.474148e-18 0.000000e+00 3.102934e-28 0.000000e+00 0.000000e+00 +1.080987 6.784177e-18 7.036681e-18 0.000000e+00 2.954367e-28 0.000000e+00 0.000000e+00 +1.082785 6.393389e-18 6.626162e-18 0.000000e+00 2.814110e-28 0.000000e+00 0.000000e+00 +1.084576 6.026038e-18 6.240760e-18 0.000000e+00 2.681590e-28 0.000000e+00 0.000000e+00 +1.086360 5.680592e-18 5.878787e-18 0.000000e+00 2.556281e-28 0.000000e+00 0.000000e+00 +1.088136 5.355632e-18 5.538686e-18 0.000000e+00 2.437701e-28 0.000000e+00 0.000000e+00 +1.089905 5.049847e-18 5.219021e-18 0.000000e+00 2.325410e-28 0.000000e+00 0.000000e+00 +1.091667 4.762020e-18 4.918463e-18 0.000000e+00 2.219002e-28 0.000000e+00 0.000000e+00 +1.093422 4.491023e-18 4.635780e-18 0.000000e+00 2.118103e-28 0.000000e+00 0.000000e+00 +1.095169 4.235808e-18 4.369832e-18 0.000000e+00 2.022369e-28 0.000000e+00 0.000000e+00 +1.096910 3.995401e-18 4.119560e-18 0.000000e+00 1.931483e-28 0.000000e+00 0.000000e+00 +1.098644 3.768894e-18 3.883981e-18 0.000000e+00 1.845150e-28 0.000000e+00 0.000000e+00 +1.100371 3.555441e-18 3.662180e-18 0.000000e+00 1.763099e-28 0.000000e+00 0.000000e+00 +1.102091 3.354254e-18 3.453305e-18 0.000000e+00 1.685079e-28 0.000000e+00 0.000000e+00 +1.103804 3.164596e-18 3.256564e-18 0.000000e+00 1.610855e-28 0.000000e+00 0.000000e+00 +1.105510 2.985780e-18 3.071217e-18 0.000000e+00 1.540212e-28 0.000000e+00 0.000000e+00 +1.107210 2.817162e-18 2.896573e-18 0.000000e+00 1.472945e-28 0.000000e+00 0.000000e+00 +1.108903 2.658141e-18 2.731988e-18 0.000000e+00 1.408869e-28 0.000000e+00 0.000000e+00 +1.110590 2.508153e-18 2.576860e-18 0.000000e+00 1.347807e-28 0.000000e+00 0.000000e+00 +1.112270 2.366670e-18 2.430625e-18 0.000000e+00 1.289597e-28 0.000000e+00 0.000000e+00 +1.113943 2.233198e-18 2.292757e-18 0.000000e+00 1.234084e-28 0.000000e+00 0.000000e+00 +1.115611 2.107271e-18 2.162761e-18 0.000000e+00 1.181128e-28 0.000000e+00 0.000000e+00 +1.117271 1.988454e-18 2.040175e-18 0.000000e+00 1.130593e-28 0.000000e+00 0.000000e+00 +1.118926 1.876339e-18 1.924565e-18 0.000000e+00 1.082354e-28 0.000000e+00 0.000000e+00 +1.120574 1.770540e-18 1.815525e-18 0.000000e+00 1.036294e-28 0.000000e+00 0.000000e+00 +1.122216 1.670697e-18 1.712674e-18 0.000000e+00 9.923029e-29 0.000000e+00 0.000000e+00 +1.123852 1.576469e-18 1.615653e-18 0.000000e+00 9.502763e-29 0.000000e+00 0.000000e+00 +1.125481 1.487538e-18 1.524125e-18 0.000000e+00 9.101169e-29 0.000000e+00 0.000000e+00 +1.127105 1.403603e-18 1.437776e-18 0.000000e+00 8.717329e-29 0.000000e+00 0.000000e+00 +1.128722 1.324380e-18 1.356307e-18 0.000000e+00 8.350379e-29 0.000000e+00 0.000000e+00 +1.130334 1.249604e-18 1.279439e-18 0.000000e+00 7.999501e-29 0.000000e+00 0.000000e+00 +1.131939 1.179024e-18 1.206910e-18 0.000000e+00 7.663928e-29 0.000000e+00 0.000000e+00 +1.133539 1.112403e-18 1.138472e-18 0.000000e+00 7.342931e-29 0.000000e+00 0.000000e+00 +1.135133 1.049518e-18 1.073893e-18 0.000000e+00 7.035825e-29 0.000000e+00 0.000000e+00 +1.136721 0.000000e+00 1.012954e-18 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.138303 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.139879 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.141450 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.143015 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.144574 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.146128 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.147676 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.149219 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.150756 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.152288 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.153815 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.155336 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.156852 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.158362 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.159868 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.161368 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.162863 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.164353 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.165838 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.167317 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.168792 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.170262 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.171726 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.173186 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.174641 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.176091 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.177536 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.178977 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.180413 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.181844 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.183270 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.184691 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.186108 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.187521 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.188928 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.190332 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.191730 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.193125 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.194514 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.195900 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.197281 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.198657 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.200029 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.201397 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.202761 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 +1.204120 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 diff --git a/Exec/HaloFinder_GPU/ascent_actions.yaml b/Exec/HaloFinder_GPU/ascent_actions.yaml new file mode 100644 index 000000000..8fbab5703 --- /dev/null +++ b/Exec/HaloFinder_GPU/ascent_actions.yaml @@ -0,0 +1,35 @@ +--- +- + action: add_pipelines + pipelines: + slice: + f1: + params: + normal: + x: 0.0 + ? "y" + : 0.0 + z: 1.0 + point: + x: 0.0 + ? "y" + : 0.0 + z: 0.0 + topology: topo + type: exaslice +- + action: add_extracts + extracts: + e1: + params: + fields: + - Density + - Xmom + - Ymom + - Zmom + - rho_E + - rho_e + path: the_slice + protocol: blueprint/mesh/hdf5 + pipeline: slice + type: relay diff --git a/Exec/HaloFinder_GPU/ascent_actions_nohydro.yaml b/Exec/HaloFinder_GPU/ascent_actions_nohydro.yaml new file mode 100644 index 000000000..597c9446b --- /dev/null +++ b/Exec/HaloFinder_GPU/ascent_actions_nohydro.yaml @@ -0,0 +1,25 @@ +- + action: "add_scenes" + scenes: + scene1: + image_prefix: "ascent_render_mesh_particle_x_velocity_" + plots: + plt1: + type: "pseudocolor" + field: "particle_x_velocity" + scene2: + image_prefix: "ascent_render_particles_xvel_" + plots: + plt1: + type: "pseudocolor" + field: "particle_xvel" + scene3: + image_prefix: "ascent_render_both_" + plots: + plt1: + type: "pseudocolor" + field: "particle_x_velocity" + plt2: + type: "pseudocolor" + field: "particle_xvel" + diff --git a/Exec/HaloFinder_GPU/ascent_actions_slicefile.yaml b/Exec/HaloFinder_GPU/ascent_actions_slicefile.yaml new file mode 100644 index 000000000..bbb174af3 --- /dev/null +++ b/Exec/HaloFinder_GPU/ascent_actions_slicefile.yaml @@ -0,0 +1,35 @@ +--- +- + action: add_pipelines + pipelines: + slice: + f1: + params: + normal: + x: 0.0 + ? "y" + : 0.0 + z: 1.0 + point: + x: 0.0 + ? "y" + : 0.0 + z: 0.0 + topology: topo + type: exaslice +- + action: add_extracts + extracts: + e1: + params: + fields: + - Density + - Xmom + - Ymom + - Zmom + - rho_E + - rho_e + path: the_slice + pipeline: slice + protocol: blueprint/mesh/hdf5 + type: relay diff --git a/Exec/HaloFinder_GPU/comoving.cpp b/Exec/HaloFinder_GPU/comoving.cpp new file mode 100644 index 000000000..70d5071e8 --- /dev/null +++ b/Exec/HaloFinder_GPU/comoving.cpp @@ -0,0 +1,969 @@ +#include +#include +#include + +using namespace amrex; + +Real Nyx::initial_z = -1.0; +Real Nyx::final_a = -1.0; +Real Nyx::final_z = -1.0; +Real Nyx::relative_max_change_a = 0.01; +Real Nyx::absolute_max_change_a = -1.0; +Real Nyx::dt_binpow = -1.0; +Real Nyx::initial_time = -1.0; +Real Nyx::final_time = -1.0; + +void +Nyx::read_comoving_params () +{ + ParmParse pp("nyx"); + + pp.get("comoving_OmB" , comoving_OmB); + pp.get("comoving_OmM" , comoving_OmM); + pp.get("comoving_h" , comoving_h); + pp.query("comoving_OmR" , comoving_OmR); + + pp.query("initial_z", initial_z); + pp.query("final_a", final_a); + pp.query("final_z", final_z); + + if (final_z >= 0) + { + if (final_a > 0) + { + std::cerr << "ERROR::dont specify both final_a and final_z\n"; + amrex::Error(); + } + else + { + final_a = 1 / (1 + final_z); + } + } + + pp.query("relative_max_change_a", relative_max_change_a); + pp.query("absolute_max_change_a", absolute_max_change_a); + pp.query("dt_binpow", dt_binpow); + + + // for shrinking box tests, initial_z < 0 is ok + if (initial_z < 0) + { + std::cerr << "ERROR::Need to specify non-negative initial redshift \n"; + amrex::Error(); + } + + if (comoving_h > 0) + { + // save start/end times, for reporting purposes + Real a0 = 0.0, a1 = 1.0/(1.0+initial_z); + integrate_time_given_a(a0, a1, initial_time); + + if (final_z >= 0) { + a1 = 1.0/(1.0+final_z); + } else { + a1 = final_a; + } + integrate_time_given_a(a0, a1, final_time); + } else { + // These are just defaults so the values are defined + initial_time = 0.0; + final_time = 0.0; + } +} + +void +Nyx::integrate_comoving_a_to_a(const Real old_a_local, const Real a_value, Real& dt) +{ + Real H_0, OmL; + Real Delta_t; + Real start_a, end_a, start_slope, end_slope; + int j, nsteps; + + if (comoving_h == 0.0) + amrex::Abort("fort_integrate_comoving_a_to_z: Shouldn't be setting plot_z_values if not evolving a"); + + H_0 = comoving_h * Hubble_const; + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + // Use lots of steps if we want to nail the z_value + // nsteps = 1024 + + // Use enough steps if we want to be close to the a_value + nsteps = 2048; + + // We integrate a, but stop when a = a_value (or close enough) + Delta_t = dt/nsteps; + end_a = old_a_local; + for(j = 1; j <= nsteps; j++) + { + // This uses RK2 to integrate the ODE: + // da / dt = H_0 * sqrt(OmM/a + OmR/a^2 + OmL*a^2) + start_a = end_a; + + // Compute the slope at the old time + if (comoving_type > 0) + start_slope = H_0*std::sqrt(comoving_OmM/start_a + comoving_OmR/(start_a*start_a) + OmL*start_a*start_a); + else + start_slope = comoving_h; + + // Compute a provisional value of ln(a) at the new time + end_a = start_a + start_slope * Delta_t; + + // Compute the slope at the new time + if (comoving_type > 0) + end_slope = H_0*std::sqrt(comoving_OmM/end_a + comoving_OmR/(end_a*end_a) + OmL*end_a*end_a); + else + end_slope = comoving_h; + + // Now recompute a at the new time using the average of the two slopes + end_a = start_a + 0.5e0 * (start_slope + end_slope) * Delta_t; + + // We have crossed from a too small to a too big in this step + if ( (end_a - a_value) * (start_a - a_value) < 0) + { + dt = ( ( end_a - a_value) * double(j ) + + (a_value - start_a) * double(j+1) ) / (end_a - start_a) * Delta_t; + break; + } + + } +} + +void +Nyx::integrate_comoving_a_to_z(const Real old_a_local, const Real z_value, Real& dt) +{ + Real H_0, OmL; + Real Delta_t; + Real start_a, end_a, start_slope, end_slope; + Real a_value; + int j, nsteps; + + if (comoving_h == 0.0) + amrex::Abort("fort_integrate_comoving_a_to_z: Shouldn't be setting plot_z_values if not evolving a"); + + H_0 = comoving_h * Hubble_const; + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + // Translate the target "z" into a target "a" + a_value = 1.e0 / (1.e0 + z_value); + + // Use lots of steps if we want to nail the z_value + nsteps = 1024; + + // We integrate a, but stop when a = a_value (or close enough) + Delta_t = dt/nsteps; + end_a = old_a_local; + for(j = 1; j <= nsteps; j++) + { + // This uses RK2 to integrate the ODE: + // da / dt = H_0 * sqrt(OmM/a + OmR/a^2 + OmL*a^2) + start_a = end_a; + + // Compute the slope at the old time + if (comoving_type > 0) + start_slope = H_0*std::sqrt(comoving_OmM/start_a + comoving_OmR/(start_a*start_a) + OmL*start_a*start_a); + else + start_slope = comoving_h; + + // Compute a provisional value of ln(a) at the new time + end_a = start_a + start_slope * Delta_t; + + // Compute the slope at the new time + if (comoving_type > 0) + end_slope = H_0*std::sqrt(comoving_OmM/end_a + comoving_OmR/(end_a*end_a) + OmL*end_a*end_a); + else + end_slope = comoving_h; + + // Now recompute a at the new time using the average of the two slopes + end_a = start_a + 0.5e0 * (start_slope + end_slope) * Delta_t; + + // We have crossed from a too small to a too big in this step + if ( (end_a - a_value) * (start_a - a_value) < 0) + { + dt = ( ( end_a - a_value) * double(j ) + + (a_value - start_a) * double(j+1) ) / (end_a - start_a) * Delta_t; + break; + } + + } +} + +void +Nyx::est_maxdt_comoving_a(const Real old_a_local, Real & dt) +{ + Real H_0, OmL; + Real max_dt; + + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + // This subroutine computes dt based on not changing a by more than 5% + // if we use forward Euler integration + // d(ln(a)) / dt = H_0 * sqrt(OmM/a^3 + OmL) + + H_0 = comoving_h * Hubble_const; + + if (H_0 != 0.0) + { + if (comoving_type > 0) + max_dt = (0.05) / H_0 / std::sqrt( comoving_OmM/(old_a_local * old_a_local *old_a_local) + +comoving_OmR/(old_a_local * old_a_local *old_a_local * old_a_local) + OmL); + else + max_dt = (0.05) / std::abs(comoving_h); + dt = amrex::min(dt,max_dt); + } +} + +// This might only work for t=0=> a=.00625, although constant canceled +void +Nyx::est_lindt_comoving_a(const Real old_a_local, const Real new_a_local, Real& dt) +{ + Real H_0; + Real lin_dt; + + // This subroutine computes dt based on not changing a by more than 5% + // if we use forward Euler integration + // OmL = 1.e0 - comoving_OmM - comoving_OmR; + // d(ln(a)) / dt = H_0 * sqrt(OmM/a^3 + OmR/a^4 + OmL) + + H_0 = comoving_h * Hubble_const; + + // Could definately be optimized better + if (H_0 != 0.0) + { + lin_dt= ( pow((new_a_local/(pow(.75,(2_rt/3_rt)))),(.75)) + -pow((old_a_local/(pow(.75,(2_rt/3_rt)))),(.75)) ) / H_0; + dt=lin_dt; + } +} + +void +Nyx::estdt_comoving_a(const Real old_a_local, Real& new_a_local, + Real& dt, const Real change_allowed, + const Real fixed_da, const Real final_a_in, int& dt_modified) +{ + if (comoving_h != 0.0e0) + { + if( fixed_da <= 0.0e0) + { + // First call this to make sure dt that we send to integration routine isnt outrageous + est_maxdt_comoving_a(old_a_local,dt); + + // Initial call to see if existing dt will work + integrate_comoving_a(old_a_local,new_a_local,dt); + + // Make sure a isn't growing too fast + enforce_percent_change(old_a_local,new_a_local,dt,change_allowed); + } + else + { + // First call this to make sure dt that we send to integration routine isnt outrageous + new_a_local = (old_a_local + fixed_da); + est_lindt_comoving_a(old_a_local,new_a_local,dt); + est_maxdt_comoving_a(old_a_local,dt); + + // Then integrate old_a_local to a_value using dt as a guess for the maximum dt + // output dt is based on a fraction of the input dt + integrate_comoving_a_to_a(old_a_local,new_a_local,dt); + } + // Make sure we don't go past final_a_in (if final_a_in is set) + if (final_a_in > 0.0e0) + enforce_final_a(old_a_local,new_a_local,dt,final_a_in); + + dt_modified = 1; + } + else + { + // dt is unchanged by this call + + dt_modified = 0; + } +} + +void +Nyx::enforce_percent_change(const Real old_a_local, Real& new_a_local, + Real& dt,const Real change_allowed) +{ + + int i; + Real factor = ( (new_a_local - old_a_local) / old_a_local ) / change_allowed; + + // Only go into this process if percent change exceeds change_allowed + + if(factor > 1.0) + { + for(i = 1; i <= 100; i++) + { + factor = ( (new_a_local - old_a_local) / old_a_local ) / change_allowed; + + // Note: 0.99 is just a fudge factor so we don't get bogged down. + if(factor > 1.0) + { + dt = (1.0 / factor) * dt * 0.99; + integrate_comoving_a(old_a_local,new_a_local,dt); + } + else if (i < 100) + { + integrate_comoving_a(old_a_local,new_a_local,dt); + // We're done + return; + } + else + amrex::Abort("Too many iterations in enforce_percent_change"); + + } + } + else + return; +} + +void +Nyx::enforce_final_a(const Real old_a_local, Real& new_a_local, + Real& dt, const Real final_a_in) +{ + int i; + Real factor; + const Real eps = 1.e-10; + + if (old_a_local > final_a_in) + amrex::Abort("Oops -- old_a > final_a_in"); + + // Only go into this process if new_a is past final_a_in + if (new_a_local > final_a_in) + { + for(i = 1; i <= 100; i++) + { + if ( (new_a_local > (final_a_in+eps)) || (new_a_local < final_a_in) ) + { + factor = (final_a_in - old_a_local) / (new_a_local - old_a_local); + dt = dt * factor; + integrate_comoving_a(old_a_local,new_a_local,dt); + } + else if (i<100) + return; + else + amrex::Abort("Too many iterations in enforce_final_a"); + } + } + else + // We don't need to do anything + return; +} + + +void +Nyx::comoving_est_time_step (Real& cur_time, Real& estdt) +{ + Real change_allowed = relative_max_change_a; + Real fixed_da = absolute_max_change_a; + Real dt = estdt; + Real new_dummy_a; + int dt_modified; + + if ( std::abs(cur_time - new_a_time) <= 1.e-12 * cur_time) + { + + // Initial guess -- note that we send in "new_a" because we haven't yet swapped + // "old_a" and "new_a" -- we can't do that until after we compute dt and then + // integrate a forward. + estdt_comoving_a + (new_a, new_dummy_a, dt, change_allowed, fixed_da, final_a, dt_modified); + + if(dt_binpow >= 0) + { + if(estdt>=dt) + estdt=dt; + else if(estdt>.5*dt) + { + estdt=.5*dt; + // std::cout << "Lavel = 1" <.25*dt) + { + estdt=.25*dt; + // std::cout << "Lavel = 2" <.125*dt) + { + estdt=.125*dt; + // std::cout << "Lavel = 3" <.0625*dt) + { + estdt=.0625*dt; + // std::cout << "Lavel = 4" < 4" <= 0) + { + if(estdt>=dt) + estdt=dt; + else if(estdt>.5*dt) + { + estdt=.5*dt; + // std::cout << "Lavel = 1" <.25*dt) + { + estdt=.25*dt; + // std::cout << "Lavel = 2" <.125*dt) + { + estdt=.125*dt; + // std::cout << "Lavel = 3" <.0625*dt) + { + estdt=.0625*dt; + // std::cout << "Lavel = 4" < 4" < old_a_time - eps && time < old_a_time + eps) + { + return old_a; + } + else if (time > new_a_time - eps && time < new_a_time + eps) + { + return new_a; + } + else if (time > old_a_time && time < new_a_time) + { + Real frac = (time - old_a_time) / (new_a_time - old_a_time); + Real a = frac*new_a + (1.0-frac)*old_a; + return a; + } + else + { + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "Invalid:get comoving_a at " << time << std::endl; + std::cout << "Old / new a_time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new a " << old_a << " " << new_a << std::endl; + } + amrex::Error("get_comoving_a: invalid time"); + return 0; + } +} + +void +Nyx::integrate_comoving_a (Real time,Real dt) +{ + if (level > 0) + return; + + bool first; + + if ( std::abs(time-new_a_time) <= (1.e-10 * time) ) + { + first = true; + } else { + first = false; + } + + if (first) + { + + // Update a + old_a = new_a; + integrate_comoving_a(old_a, new_a, dt); + + // Update the times + old_a_time = new_a_time; + new_a_time = old_a_time + dt; + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "Integrating a from time " << time << " by dt = " << dt << '\n'; + std::cout << "Old / new A time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new A " << old_a << " " << new_a << std::endl; + std::cout << "Old / new z " << 1./old_a-1.<< " " << 1./new_a-1. << std::endl; + } + } + else if (std::abs(time-old_a_time) <= 1.e-10 * time) + { + // Leave old_a and old_a_time alone -- we have already swapped them + integrate_comoving_a(old_a, new_a, dt); + + // Update the new time only + new_a_time = old_a_time + dt; + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << "Re-integrating a from time " << time << " by dt = " << dt << '\n'; + std::cout << "Old / new A time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new A " << old_a << " " << new_a << std::endl; + std::cout << "Old / new z " << 1./old_a-1.<< " " << 1./new_a-1. << std::endl; + } + } + else + { + std::cout << "Time passed to integrate_comoving_a " << time << std::endl; + std::cout << "Old / new A time " << old_a_time << " " << new_a_time << std::endl; + std::cout << "Old / new A " << old_a << " " << new_a << std::endl; + std::cerr << "ERROR::dont know what to do in integrate_comoving_a" << std::endl; + amrex::Error(); + } +} + +Real invEz(Real H0, Real Om, Real Or, Real a) +{ + // invEz = + return 1.0e0 / ( H0*std::sqrt(Om/a + Or/(a*a) + (1.0e0-Om-Or)*a*a) ); +} + +void +Nyx::integrate_time_given_a(const Real a0, const Real a1, Real& dt) +{ + const Real small_dt_fac = 1.0e-6; + Real H0, OmM, OmR, prev_soln, h; + int iter, n, j; + + H0 = comoving_h*Hubble_const; + OmM = comoving_OmM; + OmR = comoving_OmR; + + prev_soln = -1.0e0; + // trapezoidal integration + for(iter = 1; iter<= 20; iter++) + { + n = static_cast(std::round(std::pow(2,iter))); + + h = (a1-a0)/(n-1); + + if (a0 < 1.0e-10) // prevent division by zero in invEz + dt = 0.5*invEz(H0, OmM, OmR, a1); + else + dt = 0.5*(invEz(H0, OmM, OmR, a0) + invEz(H0, OmM, OmR, a1)); + + for(j = 1; j <= n-2; j++) + { + dt = dt + invEz(H0, OmM, OmR, a0+j*h); + } + dt = dt*h; + + if (iter > 4) + if (std::abs(dt-prev_soln) < small_dt_fac*std::abs(prev_soln)) + return; + + prev_soln = dt; + } +} + +void +Nyx::integrate_distance_given_a(const Real a0, const Real a1, Real& dt) +{ + const Real small_dt_fac = 1.0e-6; + Real H0, OmM, OmR, prev_soln, h; + int iter, n, j; + + H0 = comoving_h*Hubble_const; + OmM = comoving_OmM; + OmR = comoving_OmR; + + prev_soln = -1.0e0; + // trapezoidal integration + for(iter = 1; iter<= 20; iter++) + { + n = static_cast(std::round(std::pow(2,iter))); + + h = (a1-a0)/(n-1); + + if (a0 < 1.0e-10) // prevent division by zero in invEz + dt = 0.5*invEz(H0, OmM, OmR, a1)/a1; + else + dt = 0.5*(invEz(H0, OmM, OmR, a0)/a0 + invEz(H0, OmM, OmR, a1)/a1); + + for(j = 1; j <= n-2; j++) + { + dt = dt + invEz(H0, OmM, OmR, a0+j*h)/(a0+j*h); + } + dt = dt*h*c_light; + + if (iter > 4) + if (std::abs(dt-prev_soln) < small_dt_fac*std::abs(prev_soln)) + return; + + prev_soln = dt; + } +} + +void +Nyx::integrate_comoving_a (const Real old_a_local, Real& new_a_local, const Real dt) +{ + + const Real small_a_fac = 1.0e-8; + Real H_0, OmL, Delta_t, prev_soln; + Real start_a, end_a, start_slope, end_slope; + int iter, j, nsteps; + + if (comoving_h == 0.0) + { + new_a_local = old_a_local; + return; + } + + H_0 = comoving_h * Hubble_const; + OmL = 1.e0 - comoving_OmM - comoving_OmR; + + prev_soln = 2.0e0; // 0(std::round(std::pow(2,iter-1))); + + Delta_t = dt/nsteps; + end_a = old_a_local; + + for(j = 1; j <= nsteps; j++) + { + // This uses RK2 to integrate the ODE: + // da / dt = H_0 * sqrt(OmM/a + OmR/a^2 + OmL*a^2) + start_a = end_a; + + // Compute the slope at the old time + if (comoving_type > 0) + start_slope = H_0*std::sqrt(comoving_OmM/start_a + comoving_OmR/(start_a*start_a) + OmL*start_a*start_a); + else + start_slope = comoving_h; + + // Compute a provisional value of ln(a) at the new time + end_a = start_a + start_slope * Delta_t; + + // Compute the slope at the new time + if (comoving_type > 0) + end_slope = H_0*std::sqrt(comoving_OmM/end_a + comoving_OmR/(end_a*end_a) + OmL*end_a*end_a); + else + end_slope = comoving_h; + + // Now recompute a at the new time using the average of the two slopes + end_a = start_a + 0.5e0 * (start_slope + end_slope) * Delta_t; + + } + + new_a_local = end_a; + + if (std::abs(1.0e0-new_a_local/prev_soln) <= small_a_fac) + return; + prev_soln = new_a_local; + + } +} + +void +Nyx::comoving_a_post_restart (const std::string& restart_file) +{ + if (level > 0) + return; + + if (ParallelDescriptor::IOProcessor()) + { + std::string FileName = restart_file + "/comoving_a"; + std::ifstream File; + File.open(FileName.c_str(),std::ios::in); + if (!File.good()) + amrex::FileOpenFailed(FileName); + File >> old_a; + } + ParallelDescriptor::Bcast(&old_a, 1, ParallelDescriptor::IOProcessorNumber()); + + new_a = old_a; + +#ifdef NO_HYDRO + old_a_time = state[PhiGrav_Type].curTime(); +#else + old_a_time = state[State_Type].curTime(); +#endif + new_a_time = old_a_time; + if (ParallelDescriptor::IOProcessor()) + { + std::cout << "...setting old_a_time to " << old_a_time << std::endl; + } + +} + +void +Nyx::plot_z_est_time_step (Real& dt_0, bool& dt_changed) +{ + Real dt = dt_0; + Real a_old, z_old, a_new, z_new; + Real z_value; + + // This is where we are now +#ifdef NO_HYDRO + Real cur_time = state[PhiGrav_Type].curTime(); +#else + Real cur_time = state[State_Type].curTime(); +#endif + a_old = get_comoving_a(cur_time); + z_old = (1. / a_old) - 1.; + + // ***************************************************** + // First test whether we are within dt of a plot_z value + // ***************************************************** + + // This is where we would be if we use the current dt_0 + Real new_time = cur_time + dt_0; + integrate_comoving_a (cur_time,dt_0); + a_new = get_comoving_a(new_time); + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the plot_z_values array + bool found_one = false; + for (int i = 0; i < plot_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - plot_z_values[i]) * (z_old - plot_z_values[i]) < 0 && !found_one) + { + z_value = plot_z_values[i]; + found_one = true; + } + } + + // Now that we know that dt_0 is too big and makes us pass z_value, + // we must figure out what value of dt < dt_0 makes us exactly reach z_value + if (found_one) + { + integrate_comoving_a_to_z(old_a, z_value, dt); + + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << dt << std::endl; + std::cout << " ... in order to write a plotfile at z = " << z_value << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = dt; + dt_changed = true; + } + else + { + + // ***************************************************** + // If not within one dt, now test whether we are within 2*dt of a plot_z value + // ***************************************************** + + // This is where we would be if we advance by twice the current dt_0 + Real two_dt = 2.0*dt_0; + + integrate_comoving_a(a_old, a_new, two_dt); + + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the plot_z_values array + found_one = false; + for (int i = 0; i < plot_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - plot_z_values[i]) * (z_old - plot_z_values[i]) < 0 && !found_one) + { + z_value = plot_z_values[i]; + found_one = true; + } + } + + // Now that we know that 2*dt_0 will make us pass z_value, we set the current dt + // as half the interval to reach that z_value + if (found_one) + { + integrate_comoving_a_to_z(old_a, z_value, two_dt); + + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << 0.5 * two_dt << std::endl; + std::cout << " ... in order to write a plotfile at z = " << z_value + << " two steps from now " << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = 0.5 * two_dt; + dt_changed = true; + } + + } +} + +void +Nyx::analysis_z_est_time_step (Real& dt_0, bool& dt_changed) +{ + Real dt = dt_0; + Real a_old, z_old, a_new, z_new, z_value; + + // This is where we are now +#ifdef NO_HYDRO + Real cur_time = state[PhiGrav_Type].curTime(); +#else + Real cur_time = state[State_Type].curTime(); +#endif + a_old = get_comoving_a(cur_time); + z_old = (1. / a_old) - 1.; + + // ***************************************************** + // First test whether we are within dt of a analysis_z value + // ***************************************************** + + // This is where we would be if we use the current dt_0 + Real new_time = cur_time + dt_0; + integrate_comoving_a (cur_time,dt_0); + a_new = get_comoving_a(new_time); + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the analysis_z_values array + bool found_one = false; + for (int i = 0; i < analysis_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - analysis_z_values[i]) * (z_old - analysis_z_values[i]) < 0 && !found_one) + { + z_value = analysis_z_values[i]; + found_one = true; + } + } + + // Now that we know that dt_0 is too big and makes us pass z_value, + // we must figure out what value of dt < dt_0 makes us exactly reach z_value + if (found_one) + { + integrate_comoving_a_to_z(old_a, z_value, dt); + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << dt << std::endl; + std::cout << " ... in order to do analysis at z = " << z_value << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = dt; + dt_changed = true; + } + else + { + + // ***************************************************** + // If not within one dt, now test whether we are within 2*dt of a analysis_z value + // ***************************************************** + + // This is where we would be if we advance by twice the current dt_0 + Real two_dt = 2.0*dt_0; + + integrate_comoving_a(a_old, a_new, two_dt); + z_new = (1. / a_new) - 1.; + + // Find the relevant entry of the analysis_z_values array + found_one = false; + for (int i = 0; i < analysis_z_values.size(); i++) + { + // We have gone from before to after one of the specified values + if ( (z_new - analysis_z_values[i]) * (z_old - analysis_z_values[i]) < 0 && !found_one) + { + z_value = analysis_z_values[i]; + found_one = true; + } + } + + // Now that we know that 2*dt_0 will make us pass z_value, we set the current dt + // as half the interval to reach that z_value + if (found_one) + { + two_dt = 2.0*dt_0; + integrate_comoving_a_to_z(old_a, z_value, two_dt); + + if (verbose && ParallelDescriptor::IOProcessor()) + { + std::cout << " " << std::endl; + std::cout << " ... modifying time step from " << dt_0 << " to " << 0.5 * two_dt << std::endl; + std::cout << " ... in order to do analysis at z = " << z_value + << " two steps from now " << std::endl; + std::cout << " " << std::endl; + } + + // We want to pass this value back out + dt_0 = 0.5 * two_dt; + dt_changed = true; + } + + } +} diff --git a/Exec/HaloFinder_GPU/inputs b/Exec/HaloFinder_GPU/inputs new file mode 100644 index 000000000..cec255ca8 --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs @@ -0,0 +1,170 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 10000000 + +nyx.ppm_type = 1 +nyx.use_colglaz = 0 +nyx.corner_coupling = 1 + +nyx.strang_split = 0 +nyx.sdc_split = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 11 +#nyx.simd_width = 8 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +#nyx.h_species=.76 +#nyx.he_species=.24 +nyx.h_species=.7546 +nyx.he_species=.2454 + +nyx.small_dens = 1.e-2 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.ml_tol = 1.e-10 +gravity.sl_tol = 1.e-10 +gravity.mlmg_agglomeration=1 +gravity.mlmg_consolidation=1 +nyx.reuse_mlpoisson = 1 + +nyx.initial_z = 50.0 +nyx.final_z = 2.0 + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00100 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 7700 7700 7700 + +amr.n_cell = 32 32 32 +amr.max_grid_size = 32 +fabarray.mfiter_tile_size = 1024000 8 8 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COSMOLOGY +nyx.comoving_OmM = 0.31 +nyx.comoving_OmB = 0.049389 +nyx.comoving_h = 0.68 + +# UVB and reionization +nyx.inhomo_reion = 0 +nyx.inhomo_zhi_file = "zhi.bin" +nyx.inhomo_grid = 512 +nyx.uvb_rates_file = "TREECOOL_middle" +nyx.uvb_density_A = 1.0 +nyx.uvb_density_B = 0.0 +nyx.reionization_zHI_flash = -1.0 +nyx.reionization_zHeII_flash = -1.0 +nyx.reionization_T_zHI = 2.0e4 +nyx.reionization_T_zHeII = 1.5e4 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryMetaFile +nyx.binary_particle_file = Filelist.txt +particles.nparts_per_read = 2097152 + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.5 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 2.0 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Nyx.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 1 # verbosity in Amr.cpp +particles.v = 2 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 +#amr.nosub = 1 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = 1 +amr.check_file = chk +amr.check_int = 100 +amr.checkpoint_nfiles = 64 + +# PLOTFILES +amr.plot_files_output = 1 +amr.plot_file = plt +amr.plot_int = -1 +nyx.plot_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 + +amr.plot_vars = density xmom ymom zmom rho_e Temp phi_grav +amr.derive_plot_vars = particle_mass_density particle_count + +# Lightcone +nyx.lightcone_start_z=200 + +# Halo Finder +#nyx.analysis_z_values = 150 10 5 4 3 2 +reeber.halo_int = 1 +reeber.negate = 1 +reeber.halo_density_vars = density particle_mass_density +reeber.halo_extrema_threshold = 20 +reeber.halo_component_threshold = 10 +#nyx.mass_halo_min = 1.e11 +#nyx.mass_seed = 1.e6 + +# ANALYSIS in situ +nyx.analysis_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 +insitu.int = 100 +insitu.start = 0 +insitu.reeber_int = 100 + +# SENSEI in situ +sensei.enabled = 0 +#sensei.config = write_vtk.xml +sensei.config = render_iso_catalyst_3d.xml +sensei.frequency = 2 + +#PROBIN FILENAME +amr.probin_file = "" diff --git a/Exec/HaloFinder_GPU/inputs.rt b/Exec/HaloFinder_GPU/inputs.rt new file mode 100644 index 000000000..5fc5420aa --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs.rt @@ -0,0 +1,131 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 10000000 + +# We set this to 1 so that we eliminate the OpenMP-induced variability +# in the MultiFab and Nyx sums that was making the solution to the gravity solve +# have sufficient variability that this failed the regression test using OpenMP. +amrex.regtest_reduction = 1 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +nyx.h_species=.76 +nyx.he_species=.24 + +nyx.ppm_type = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 11 + +nyx.strang_split = 0 +nyx.sdc_split = 1 + +nyx.inhomo_reion = 0 +nyx.uvb_rates_file = "TREECOOL_middle" + +#This is 1e-8 times the lowest density in plt00000 +nyx.small_dens = 5.162470e1 + +#This is 1e-5 times the constant temparature in plt00000 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.sl_tol = 1.e-12 +nyx.reuse_mlpoisson = 1 + +nyx.initial_z = 100.0 +nyx.final_z = 2.0 + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00070 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 8.0 8.0 8.0 + +amr.n_cell = 32 32 32 +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 1024000 1024 1024 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COMOVING +nyx.comoving_OmM = 0.27 +nyx.comoving_OmB = 0.045 +nyx.comoving_h = 0.71e0 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryFile +nyx.binary_particle_file = 32.nyx + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.9 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 1.1 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Castro.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 0 # verbosity in Amr.cpp +particles.v = 1 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +#amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = 0 +amr.check_file = chk +amr.check_int = 10000 + +# PLOTFILES +amr.plot_file = plt +amr.plot_int = 10000 + +amr.plot_vars = ALL +amr.derive_plot_vars = particle_count particle_mass_density pressure + +#PROBIN FILENAME +amr.probin_file = "" diff --git a/Exec/HaloFinder_GPU/inputs.rt.garuda b/Exec/HaloFinder_GPU/inputs.rt.garuda new file mode 100644 index 000000000..2a5a3252a --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs.rt.garuda @@ -0,0 +1,130 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 10000000 + +# We set this to 1 so that we eliminate the OpenMP-induced variability +# in the MultiFab and Nyx sums that was making the solution to the gravity solve +# have sufficient variability that this failed the regression test using OpenMP. +amrex.regtest_reduction = 1 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +nyx.h_species=.76 +nyx.he_species=.24 + +nyx.ppm_type = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 0 + +nyx.strang_split = 1 + +nyx.inhomo_reion = 0 +#nyx.uvb_rates_file = "TREECOOL_middle" + +#This is 1e-8 times the lowest density in plt00000 +nyx.small_dens = 5.162470e1 + +#This is 1e-5 times the constant temparature in plt00000 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.sl_tol = 1.e-12 +nyx.reuse_mlpoisson = 1 + +nyx.initial_z = 100.0 +nyx.final_z = 2.0 + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00070 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 8.0 8.0 8.0 + +amr.n_cell = 32 32 32 +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 1024000 1024 1024 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COMOVING +nyx.comoving_OmM = 0.27 +nyx.comoving_OmB = 0.045 +nyx.comoving_h = 0.71e0 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryFile +nyx.binary_particle_file = 32.nyx + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.9 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 1.1 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Castro.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 0 # verbosity in Amr.cpp +particles.v = 1 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +#amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = 0 +amr.check_file = chk +amr.check_int = 10000 + +# PLOTFILES +amr.plot_file = plt +amr.plot_int = 10000 + +amr.plot_vars = ALL +amr.derive_plot_vars = particle_count particle_mass_density pressure + +#PROBIN FILENAME +amr.probin_file = "" diff --git a/Exec/HaloFinder_GPU/inputs_128x128x128 b/Exec/HaloFinder_GPU/inputs_128x128x128 new file mode 100644 index 000000000..f2aa087e5 --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs_128x128x128 @@ -0,0 +1,218 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +nyx.ppm_type = 1 +nyx.use_colglaz = 0 +nyx.corner_coupling = 1 + +nyx.strang_split = 0 +nyx.sdc_split = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 11 +#nyx.simd_width = 8 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +nyx.h_species=.7546 +nyx.he_species=.2454 + +nyx.small_dens = 1.e-2 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.ml_tol = 1.e-10 +gravity.sl_tol = 1.e-10 +gravity.mlmg_agglomeration=1 +gravity.mlmg_consolidation=1 +nyx.reuse_mlpoisson = 1 + +nyx.initial_z = 159.0 +nyx.final_z = 0.0 + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00100 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 28.49002849 28.49002849 28.49002849 + +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 128 8 8 +fabarray.mfiter_tile_size = 1024000 8 8 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COSMOLOGY +nyx.comoving_OmM = 0.275 +nyx.comoving_OmB = 0.046 +nyx.comoving_h = 0.702e0 + +# UVB and reionization +nyx.inhomo_reion = 0 +nyx.inhomo_zhi_file = "zhi.bin" +nyx.inhomo_grid = 512 +nyx.uvb_rates_file = "TREECOOL_middle" +nyx.uvb_density_A = 1.0 +nyx.uvb_density_B = 0.0 +nyx.reionization_zHI_flash = -1.0 +nyx.reionization_zHeII_flash = -1.0 +nyx.reionization_T_zHI = 2.0e4 +nyx.reionization_T_zHeII = 1.5e4 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryMetaFile +nyx.binary_particle_file = 64sssss_20mpc.nyx +particles.nparts_per_read = 2097152 + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.5 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 2.0 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Nyx.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 1 # verbosity in Amr.cpp +particles.v = 2 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 +#amr.nosub = 1 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = -1 +amr.check_file = chk +amr.check_int = 100 +amr.checkpoint_nfiles = 64 + +# PLOTFILES +amr.plot_files_output = -1 +amr.plot_file = plt +amr.plot_int = -1 + +amr.plot_vars = density xmom ymom zmom rho_e Temp phi_grav +amr.derive_plot_vars = particle_mass_density particle_count + +# Halo Finder +#nyx.analysis_z_values = 150 10 5 4 3 2 +reeber.halo_int = 1 +reeber.negate = 1 +reeber.halo_density_vars = particle_mass_density +reeber.halo_component_threshold = 3.0 +reeber.halo_extrema_threshold = 8.0 +#nyx.mass_halo_min = 1.e6 +#nyx.mass_seed = 1.e6 + +# ANALYSIS in situ +nyx.analysis_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 +insitu.int = 1 +insitu.start = 0 +insitu.reeber_int = 1 + +# SENSEI in situ +sensei.enabled = 0 +#sensei.config = write_vtk.xml +sensei.config = render_iso_catalyst_3d.xml +sensei.frequency = 2 + +#PROBIN FILENAME +amr.probin_file = "" +#TESTING FLAGS +amrex.abort_on_out_of_gpu_memory=1 +amrex.async_out=1 +max_step=3 +nyx.minimize_memory=1 +nyx.hydro_tile_size=64 64 64 +nyx.sundials_tile_size=64 64 64 +nyx.sundials_use_tiling=1 +amr.max_grid_size=128 128 128 +nyx.sundials_alloc_type=5 +amrex.max_gpu_streams=4 +amrex.the_arena_is_managed=1 +nyx.minimize_memory=0 +amrex.abort_on_out_of_gpu_memory=1 +amr.checkpoint_files_output=-1 +amr.plot_files_output=-1 +amrex.async_out=1 +amr.regrid_on_restart=0 +amr.max_grid_size=256 +tiny_profiler.device_synchronize_around_region=0 +amrex.use_profiler_syncs=0 +nyx.sundials_alloc_type=5 +amr.check_int=-1 +nyx.reuse_mlpoisson=1 +amrex.the_arena_release_threshold=35651584000 +amrex.the_pinned_arena_release_threshold=1258291200 +gravity.ml_tol=1e-08 +gravity.sl_tol=1e-08 +nyx.fix_random_seed=1 +amrex.use_gpu_aware_mpi=0 +amrex.async_out=1 +amr.n_cell=128 128 128 +amr.derive_plot_vars=particle_mass_density particle_count particle_x_velocity particle_y_velocity particle_z_velocity +amr.plot_int=-1 +#nyx.plot_z_values=200.0 8.0 7.5 7.0 6.6 6.3 6.0 5.8 5.6 5.4 5.2 5.0 4.8 4.6 4.4 4.2 4.0 3.8 3.6 3.4 3.2 3.0 2.8 2.6 2.4 2.2 2.0 1.0 0.5 0.0 +nyx.binary_particle_file=Filelist_128x128x128.txt +geometry.prob_hi=7700 7700 7700 +particles.nreaders=32 +particles.nparts_per_read=2097152 +nyx.comoving_h=0.68 +nyx.comoving_OmB=0.0493890 +nyx.comoving_OmM=.31 +particles.nparts_per_read=2097152 +nyx.initial_z=50 +amrex.max_gpu_streams=8 +nyx.sundials_reltol=1e-6 +nyx.sundials_abstol=1e-6 +DistributionMapping.strategy=ROUNDROBIN +nyx.load_balance_start_z=0.0 +nyx.load_balance_int=100 +nyx.load_balance_wgt_strategy=1 +nyx.load_balance_strategy=KNAPSACK +amr.max_grid_size_x=256 +amr.max_grid_size_y=256 +amr.max_grid_size_z=256 diff --git a/Exec/HaloFinder_GPU/inputs_2048x2048x2048 b/Exec/HaloFinder_GPU/inputs_2048x2048x2048 new file mode 100644 index 000000000..fcccbdff3 --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs_2048x2048x2048 @@ -0,0 +1,217 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +nyx.ppm_type = 1 +nyx.use_colglaz = 0 +nyx.corner_coupling = 1 + +nyx.strang_split = 0 +nyx.sdc_split = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 11 +#nyx.simd_width = 8 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +nyx.h_species=.7546 +nyx.he_species=.2454 + +nyx.small_dens = 1.e-2 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.ml_tol = 1.e-10 +gravity.sl_tol = 1.e-10 +gravity.mlmg_agglomeration=1 +gravity.mlmg_consolidation=1 +nyx.reuse_mlpoisson = 1 + + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00100 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 28.49002849 28.49002849 28.49002849 + +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 128 8 8 +fabarray.mfiter_tile_size = 1024000 8 8 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COSMOLOGY +nyx.comoving_OmM = 0.31 +nyx.comoving_OmB = 0.049389 +nyx.comoving_h = 0.68e0 + +# UVB and reionization +nyx.inhomo_reion = 0 +nyx.inhomo_zhi_file = "zhi.bin" +nyx.inhomo_grid = 512 +nyx.uvb_rates_file = "TREECOOL_middle" +nyx.uvb_density_A = 1.0 +nyx.uvb_density_B = 0.0 +nyx.reionization_zHI_flash = -1.0 +nyx.reionization_zHeII_flash = -1.0 +nyx.reionization_T_zHI = 2.0e4 +nyx.reionization_T_zHeII = 1.5e4 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryMetaFile +nyx.binary_particle_file = 64sssss_20mpc.nyx +particles.nparts_per_read = 2097152 + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.5 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 2.0 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Nyx.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 1 # verbosity in Amr.cpp +particles.v = 2 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 +#amr.nosub = 1 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = 1 +amr.check_file = chk +amr.check_int = 100 +amr.checkpoint_nfiles = 64 + +# PLOTFILES +amr.plot_files_output = -1 +amr.plot_file = plt +amr.plot_int = -1 +nyx.plot_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 + +amr.plot_vars = density xmom ymom zmom rho_e Temp phi_grav +amr.derive_plot_vars = particle_mass_density particle_count + +# Halo Finder +#nyx.analysis_z_values = 150 10 5 4 3 2 +reeber.halo_int = 1 +reeber.negate = 1 +reeber.halo_density_vars = density particle_mass_density +reeber.halo_extrema_threshold = 20 +reeber.halo_component_threshold = 10 +#nyx.mass_halo_min = 1.e11 +#nyx.mass_seed = 1.e6 + +# ANALYSIS in situ +nyx.analysis_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 +insitu.int = 100 +insitu.start = 0 +insitu.reeber_int = 100 + +# SENSEI in situ +sensei.enabled = 0 +#sensei.config = write_vtk.xml +sensei.config = render_iso_catalyst_3d.xml +sensei.frequency = 2 + +#PROBIN FILENAME +amr.probin_file = "" +#TESTING FLAGS +amrex.abort_on_out_of_gpu_memory=1 +amrex.async_out=1 +max_step=2000000 +nyx.minimize_memory=1 +nyx.hydro_tile_size=64 64 64 +nyx.sundials_tile_size=64 64 64 +nyx.sundials_use_tiling=1 +amr.max_grid_size=128 128 128 +nyx.sundials_alloc_type=5 +amrex.max_gpu_streams=4 +amrex.the_arena_is_managed=1 +nyx.minimize_memory=0 +amrex.abort_on_out_of_gpu_memory=1 +amr.checkpoint_files_output=1 +amr.plot_files_output=-1 +amrex.async_out=1 +amr.regrid_on_restart=0 +amr.max_grid_size=256 +tiny_profiler.device_synchronize_around_region=0 +amrex.use_profiler_syncs=0 +nyx.sundials_alloc_type=5 +amr.check_int=-1 +nyx.reuse_mlpoisson=1 +amrex.the_arena_release_threshold=35651584000 +amrex.the_pinned_arena_release_threshold=1258291200 +gravity.ml_tol=1e-08 +gravity.sl_tol=1e-08 +nyx.fix_random_seed=1 +amrex.use_gpu_aware_mpi=0 +amrex.async_out=1 +amr.n_cell=2048 2048 2048 +amr.derive_plot_vars=particle_mass_density particle_count particle_x_velocity particle_y_velocity particle_z_velocity +amr.plot_int=1000000 +nyx.plot_z_values=200.0 8.0 7.5 7.0 6.6 6.3 6.0 5.8 5.6 5.4 5.2 5.0 4.8 4.6 4.4 4.2 4.0 3.8 3.6 3.4 3.2 3.0 2.8 2.6 2.4 2.2 2.0 1.0 0.5 0.0 +nyx.binary_particle_file=Filelist_2048x2048x2048.txt +geometry.prob_hi=7700 7700 7700 +particles.nreaders=1 +nyx.comoving_h=0.68 +nyx.comoving_OmB=0.0493890 +nyx.comoving_OmM=.31 +particles.nparts_per_read=2097152 +nyx.initial_z=50.0 +nyx.final_z = 0.0 +amrex.max_gpu_streams=8 +nyx.sundials_reltol=1e-6 +nyx.sundials_abstol=1e-6 +DistributionMapping.strategy=ROUNDROBIN +nyx.load_balance_start_z=12.0 +nyx.load_balance_int=100 +nyx.load_balance_wgt_strategy=1 +nyx.load_balance_strategy=KNAPSACK +amr.max_grid_size_x=256 +amr.max_grid_size_y=256 +amr.max_grid_size_z=256 diff --git a/Exec/HaloFinder_GPU/inputs_4096x4096x4096 b/Exec/HaloFinder_GPU/inputs_4096x4096x4096 new file mode 100755 index 000000000..ad3d22c6e --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs_4096x4096x4096 @@ -0,0 +1,218 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +nyx.ppm_type = 1 +nyx.use_colglaz = 0 +nyx.corner_coupling = 1 + +nyx.strang_split = 0 +nyx.sdc_split = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 11 +#nyx.simd_width = 8 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +nyx.h_species=.7546 +nyx.he_species=.2454 + +nyx.small_dens = 1.e-2 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.ml_tol = 1.e-10 +gravity.sl_tol = 1.e-10 +gravity.mlmg_agglomeration=1 +gravity.mlmg_consolidation=1 +nyx.reuse_mlpoisson = 1 + + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00100 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 28.49002849 28.49002849 28.49002849 + +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 128 8 8 +fabarray.mfiter_tile_size = 1024000 8 8 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COSMOLOGY +nyx.comoving_OmM = 0.31 +nyx.comoving_OmB = 0.049389 +nyx.comoving_h = 0.68e0 + +# UVB and reionization +nyx.inhomo_reion = 0 +nyx.inhomo_zhi_file = "zhi.bin" +nyx.inhomo_grid = 512 +nyx.uvb_rates_file = "TREECOOL_middle" +nyx.uvb_density_A = 1.0 +nyx.uvb_density_B = 0.0 +nyx.reionization_zHI_flash = -1.0 +nyx.reionization_zHeII_flash = -1.0 +nyx.reionization_T_zHI = 2.0e4 +nyx.reionization_T_zHeII = 1.5e4 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryMetaFile +nyx.binary_particle_file = 64sssss_20mpc.nyx +particles.nparts_per_read = 2097152 + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.5 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 2.0 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Nyx.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 1 # verbosity in Amr.cpp +particles.v = 2 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 +#amr.nosub = 1 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = -1 +amr.check_file = chk +amr.check_int = -1 +amr.checkpoint_nfiles = 64 + +# PLOTFILES +amr.plot_files_output = -1 +amr.plot_file = plt +amr.plot_int = -1 +#nyx.plot_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 + +amr.plot_vars = density xmom ymom zmom rho_e Temp phi_grav +amr.derive_plot_vars = particle_mass_density particle_count + +# Halo Finder +#nyx.analysis_z_values = 150 10 5 4 3 2 +reeber.halo_int = 1 +reeber.negate = 1 +reeber.halo_density_vars = particle_mass_density +reeber.halo_component_threshold = 82.0 +reeber.halo_extrema_threshold = 200.0 +#nyx.mass_halo_min = 1.e6 +#nyx.mass_seed = 1.e6 + +# ANALYSIS in situ +nyx.analysis_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 +insitu.int = 1 +insitu.start = 290 +insitu.reeber_int = 1 + +# SENSEI in situ +sensei.enabled = 0 +#sensei.config = write_vtk.xml +sensei.config = render_iso_catalyst_3d.xml +sensei.frequency = 2 + +#PROBIN FILENAME +amr.probin_file = "" +#TESTING FLAGS +amrex.abort_on_out_of_gpu_memory=1 +amrex.async_out=1 +max_step=2000000 +nyx.minimize_memory=1 +nyx.hydro_tile_size=64 64 64 +nyx.sundials_tile_size=64 64 64 +nyx.sundials_use_tiling=1 +amr.max_grid_size=128 128 128 +nyx.sundials_alloc_type=5 +amrex.max_gpu_streams=4 +amrex.the_arena_is_managed=1 +nyx.minimize_memory=0 +amrex.abort_on_out_of_gpu_memory=1 +amr.checkpoint_files_output=1 +amr.plot_files_output=-1 +amrex.async_out=1 +amr.regrid_on_restart=0 +amr.max_grid_size=256 +tiny_profiler.device_synchronize_around_region=0 +amrex.use_profiler_syncs=0 +nyx.sundials_alloc_type=5 +amr.check_int=-1 +nyx.reuse_mlpoisson=1 +amrex.the_arena_release_threshold=35651584000 +amrex.the_pinned_arena_release_threshold=1258291200 +gravity.ml_tol=1e-08 +gravity.sl_tol=1e-08 +nyx.fix_random_seed=1 +amrex.use_gpu_aware_mpi=0 +amrex.async_out=1 +amr.n_cell=4096 4096 4096 +amr.derive_plot_vars=particle_mass_density particle_count particle_x_velocity particle_y_velocity particle_z_velocity +amr.plot_int=-1 +#nyx.plot_z_values=200.0 8.0 7.5 7.0 6.6 6.3 6.0 5.8 5.6 5.4 5.2 5.0 4.8 4.6 4.4 4.2 4.0 3.8 3.6 3.4 3.2 3.0 2.8 2.6 2.4 2.2 2.0 1.0 0.5 0.0 +nyx.binary_particle_file=Filelist_4096x4096x4096.txt +geometry.prob_hi=7700 7700 7700 +particles.nreaders=32 +particles.nparts_per_read=2097152 +nyx.comoving_h=0.6777 +nyx.comoving_OmB=0.048 +nyx.comoving_OmM=0.307 +particles.nparts_per_read=2097152 +nyx.initial_z=100.0 +nyx.final_z = 0.0 +amrex.max_gpu_streams=8 +nyx.sundials_reltol=1e-6 +nyx.sundials_abstol=1e-6 +DistributionMapping.strategy=ROUNDROBIN +nyx.load_balance_start_z=0.0 +nyx.load_balance_int=100 +nyx.load_balance_wgt_strategy=1 +nyx.load_balance_strategy=KNAPSACK +amr.max_grid_size_x=128 +amr.max_grid_size_y=128 +amr.max_grid_size_z=128 diff --git a/Exec/HaloFinder_GPU/inputs_768x768x768 b/Exec/HaloFinder_GPU/inputs_768x768x768 new file mode 100644 index 000000000..3b332222b --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs_768x768x768 @@ -0,0 +1,215 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +nyx.ppm_type = 1 +nyx.use_colglaz = 0 +nyx.corner_coupling = 1 + +nyx.strang_split = 0 +nyx.sdc_split = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 11 +#nyx.simd_width = 8 + +# Note we now set USE_CONST_SPECIES = TRUE in the GNUmakefile +nyx.h_species=.7546 +nyx.he_species=.2454 + +nyx.small_dens = 1.e-2 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 1 +nyx.init_sb_vels = 1 +gravity.ml_tol = 1.e-10 +gravity.sl_tol = 1.e-10 +gravity.mlmg_agglomeration=1 +gravity.mlmg_consolidation=1 +nyx.reuse_mlpoisson = 1 + + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00100 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 28.49002849 28.49002849 28.49002849 + +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 128 8 8 +fabarray.mfiter_tile_size = 1024000 8 8 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 1 +nyx.do_grav = 1 + +# COSMOLOGY +nyx.comoving_OmM = 0.31 +nyx.comoving_OmB = 0.049389 +nyx.comoving_h = 0.68e0 + +# UVB and reionization +nyx.inhomo_reion = 0 +nyx.inhomo_zhi_file = "zhi.bin" +nyx.inhomo_grid = 512 +nyx.uvb_rates_file = "TREECOOL_middle" +nyx.uvb_density_A = 1.0 +nyx.uvb_density_B = 0.0 +nyx.reionization_zHI_flash = -1.0 +nyx.reionization_zHeII_flash = -1.0 +nyx.reionization_T_zHI = 2.0e4 +nyx.reionization_T_zHeII = 1.5e4 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryMetaFile +nyx.binary_particle_file = 64sssss_20mpc.nyx +particles.nparts_per_read = 2097152 + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.5 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 2.0 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Nyx.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 1 # verbosity in Amr.cpp +particles.v = 2 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 +#amr.nosub = 1 + +amr.refinement_indicators = density +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = -1 +amr.check_file = chk +amr.check_int = 100 +amr.checkpoint_nfiles = 64 + +# PLOTFILES +amr.plot_files_output = -1 +amr.plot_file = plt +amr.plot_int = -1 + +amr.plot_vars = density xmom ymom zmom rho_e Temp phi_grav +amr.derive_plot_vars = particle_mass_density particle_count + +# Halo Finder +#nyx.analysis_z_values = 150 10 5 4 3 2 +reeber.halo_int = 1 +reeber.negate = 1 +reeber.halo_density_vars = density particle_mass_density +reeber.halo_extrema_threshold = 20 +reeber.halo_component_threshold = 10 +nyx.mass_halo_min = 1.e11 +nyx.mass_seed = 1.e6 + +# ANALYSIS in situ +nyx.analysis_z_values = 7.0 6.0 5.0 4.0 3.0 2.0 +insitu.int = 1 +insitu.start = 0 +insitu.reeber_int = 1 + +# SENSEI in situ +sensei.enabled = 0 +#sensei.config = write_vtk.xml +sensei.config = render_iso_catalyst_3d.xml +sensei.frequency = 2 + +#PROBIN FILENAME +amr.probin_file = "" +#TESTING FLAGS +amrex.abort_on_out_of_gpu_memory=1 +amrex.async_out=1 +max_step=10 +nyx.hydro_tile_size=64 64 64 +nyx.sundials_tile_size=64 64 64 +nyx.sundials_use_tiling=1 +amr.max_grid_size=64 64 64 +nyx.sundials_alloc_type=5 +amrex.max_gpu_streams=4 +amrex.the_arena_is_managed=1 +nyx.minimize_memory=1 +amrex.abort_on_out_of_gpu_memory=1 +amr.checkpoint_files_output=1 +amr.plot_files_output=-1 +amrex.async_out=1 +amr.regrid_on_restart=0 +amr.max_grid_size=256 +tiny_profiler.device_synchronize_around_region=0 +amrex.use_profiler_syncs=0 +nyx.sundials_alloc_type=5 +amr.check_int=-1 +nyx.reuse_mlpoisson=1 +amrex.the_arena_release_threshold=35651584000 +amrex.the_pinned_arena_release_threshold=1258291200 +gravity.ml_tol=1e-08 +gravity.sl_tol=1e-08 +nyx.fix_random_seed=1 +amrex.use_gpu_aware_mpi=0 +amrex.async_out=1 +amr.n_cell=768 768 768 +amr.derive_plot_vars=particle_mass_density particle_count particle_x_velocity particle_y_velocity particle_z_velocity +amr.plot_int=-1 +#nyx.plot_z_values=200.0 8.0 7.5 7.0 6.6 6.3 6.0 5.8 5.6 5.4 5.2 5.0 4.8 4.6 4.4 4.2 4.0 3.8 3.6 3.4 3.2 3.0 2.8 2.6 2.4 2.2 2.0 1.0 0.5 0.0 +nyx.binary_particle_file=Filelist_768x768x768.txt +geometry.prob_hi=7700 7700 7700 +particles.nreaders=32 +particles.nparts_per_read=2097152 +nyx.comoving_h=0.68 +nyx.comoving_OmB=0.0493890 +nyx.comoving_OmM=.31 +particles.nparts_per_read=2097152 +nyx.initial_z=50.0 +nyx.final_z = 0.0 +amrex.max_gpu_streams=8 +nyx.sundials_reltol=1e-6 +nyx.sundials_abstol=1e-6 +DistributionMapping.strategy=ROUNDROBIN +nyx.load_balance_start_z=0.0 +nyx.load_balance_int=100 +nyx.load_balance_wgt_strategy=1 +nyx.load_balance_strategy=KNAPSACK +amr.max_grid_size_x=64 +amr.max_grid_size_y=64 +amr.max_grid_size_z=64 diff --git a/Exec/HaloFinder_GPU/inputs_nohydro.rt b/Exec/HaloFinder_GPU/inputs_nohydro.rt new file mode 100644 index 000000000..094169b87 --- /dev/null +++ b/Exec/HaloFinder_GPU/inputs_nohydro.rt @@ -0,0 +1,129 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 10000000 + +# We set this to 1 so that we eliminate the OpenMP-induced variability +# in the MultiFab and Nyx sums that was making the solution to the gravity solve +# have sufficient variability that this failed the regression test using OpenMP. +amrex.regtest_reduction = 1 + +nyx.ppm_type = 1 +nyx.add_ext_src = 0 +nyx.heat_cool_type = 0 + +nyx.strang_split = 1 + +nyx.inhomo_reion = 0 +nyx.uvb_rates_file = "TREECOOL_middle" + +#This is 1e-8 times the lowest density in plt00000 +nyx.small_dens = 5.162470e1 + +#This is 1e-5 times the constant temparature in plt00000 +nyx.small_temp = 1.e-2 + +nyx.do_santa_barbara = 0 +nyx.init_sb_vels = 0 +gravity.sl_tol = 1.e-12 +nyx.reuse_mlpoisson = 1 + +nyx.initial_z = 100.0 +nyx.final_z = 2.0 + +#File written during the run: nstep | time | dt | redshift | a +amr.data_log = runlog +#amr.grid_log = grdlog + +#This is how we restart from a checkpoint and write an ascii particle file +#Leave this commented out in cvs version +#amr.restart = chk00070 +#max_step = 4 +#particles.particle_output_file = particle_output + +gravity.no_sync = 1 +gravity.no_composite = 1 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 + +geometry.prob_lo = 0 0 0 + +#Domain size in Mpc +geometry.prob_hi = 8.0 8.0 8.0 + +amr.n_cell = 32 32 32 +amr.max_grid_size = 32 +#fabarray.mfiter_tile_size = 1024000 1024 1024 + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +nyx.lo_bc = 0 0 0 +nyx.hi_bc = 0 0 0 + +# WHICH PHYSICS +nyx.do_hydro = 0 +nyx.do_grav = 1 + +# COMOVING +nyx.comoving_OmM = 0.27 +nyx.comoving_OmB = 0.045 +nyx.comoving_h = 0.71e0 + +# PARTICLES +nyx.do_dm_particles = 1 + +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +# "AsciiFile" "Random" "Cosmological" +# >>>>>>>>>>>>> PARTICLE INIT OPTIONS <<<<<<<<<<<<<<<< +nyx.particle_init_type = BinaryFile +nyx.binary_particle_file = 32.nyx +nyx.particle_launch_ics = 0 + +# TIME STEP CONTROL +nyx.relative_max_change_a = 0.01 # max change in scale factor +particles.cfl = 0.5 # 'cfl' for particles +nyx.cfl = 0.9 # cfl number for hyperbolic system +nyx.init_shrink = 1.0 # scale back initial timestep +nyx.change_max = 1.1 # factor by which timestep can change +nyx.dt_cutoff = 5.e-20 # level 0 timestep below which we halt + +# DIAGNOSTICS & VERBOSITY +nyx.sum_interval = -1 # timesteps between computing mass +nyx.v = 1 # verbosity in Castro.cpp +gravity.v = 1 # verbosity in Gravity.cpp +amr.v = 1 # verbosity in Amr.cpp +mg.v = 0 # verbosity in Amr.cpp +particles.v = 2 # verbosity in Particle class + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +#amr.ref_ratio = 2 2 2 2 +#amr.regrid_int = 4 4 4 4 +#amr.n_error_buf = 0 0 0 8 +#amr.refine_grid_layout = 1 +#amr.regrid_on_restart = 1 +#amr.blocking_factor = 32 + +amr.refinement_indicators = density +amr.density.max_level = 0 +amr.density.value_greater = 3.5e9 +amr.density.field_name = denvol + +# CHECKPOINT FILES +amr.checkpoint_files_output = 0 +amr.check_file = chk +amr.check_int = 10000 + +# PLOTFILES +amr.plot_file = plt +amr.plot_int = 10000 + +amr.plot_vars = ALL +amr.derive_plot_vars = particle_count particle_mass_density pressure magvel + +#PROBIN FILENAME +amr.probin_file = "" + diff --git a/Exec/HaloFinder_GPU/run_batch_job_4096.qsub b/Exec/HaloFinder_GPU/run_batch_job_4096.qsub new file mode 100755 index 000000000..6b69ad5f0 --- /dev/null +++ b/Exec/HaloFinder_GPU/run_batch_job_4096.qsub @@ -0,0 +1,21 @@ +#!/bin/bash +#!/bin/bash + +## specify your allocation (with the _g) and that you want GPU nodes +#SBATCH -A m4749 +#SBATCH -C cpu +#SBATCH -q regular + +#SBATCH -J NyxHalos +#SBATCH -o nyx_dm.out +#SBATCH -e nyx_dm_error.out + +## set the max walltime and memory +#SBATCH -t 24:00:00 +#SBATCH --mem=0 + +## specify the number of nodes you want +#SBATCH -N 256 +#SBATCH --exclusive +#srun -N 8 -n 1024 Nyx3d.gnu.x86-milan.MTMPI.ex inputs_768x768x768 nyx.lightcone_start_z=4.5 +srun -N 256 -n 32768 Nyx3d.gnu.x86-milan.MTMPI.ex inputs_4096x4096x4096 nyx.lightcone_start_z=4.5 diff --git a/Exec/HaloFinder_GPU/run_batch_job_4096_GPU.qsub b/Exec/HaloFinder_GPU/run_batch_job_4096_GPU.qsub new file mode 100755 index 000000000..ab266bd6b --- /dev/null +++ b/Exec/HaloFinder_GPU/run_batch_job_4096_GPU.qsub @@ -0,0 +1,20 @@ +#!/bin/bash + +## specify your allocation (with the _g) and that you want GPU nodes +#SBATCH -A nyx_g +#SBATCH -C gpu +#SBATCH -q regular + +#SBATCH -J NyxHalos +#SBATCH -o nyx_dm.out +#SBATCH -e nyx_dm_error.out + +## set the max walltime and memory +#SBATCH -t 2:00:00 + +## specify the number of nodes you want +#SBATCH -N 256 +#SBATCH --exclusive +#SBATCH --gpus-per-node=4 +#SBATCH --ntasks-per-node=4 +srun Nyx3d.gnu.MTMPI.CUDA.ex inputs_4096x4096x4096 nyx.lightcone_start_z=4.5 diff --git a/Exec/HaloFinder_GPU/run_write_series.sh b/Exec/HaloFinder_GPU/run_write_series.sh new file mode 100644 index 000000000..693161855 --- /dev/null +++ b/Exec/HaloFinder_GPU/run_write_series.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Specify the root directory for traversal +root_directory="./" + +# Create a temporary file to store the list of directories +temp_file=$(mktemp) +find "$root_directory" -type d -print0 > "$temp_file" + +# Initialize an empty string to store file information +files_list="" + +# file counter to be used as time entry +count=0 + +ls -d */ | sort -z > temporary_file + +# Loop through all files in the current directory +for file in *; do + if [[ -f "$file" ]]; then # Check if it's a regular file + file_name=$(basename "$file") + + # Check if the file name starts with "lightcone" + if [[ "$file_name" == lightcone*.vtk ]]; then + # Extract version number from file name + version="${file_name#lightcone}" + + # Create file information + files_list+="$(printf "{ \"name\": \"lightcone$version\", \"time\": $count},")" + files_list+=$'\n' + + ((count++)) + fi + fi +done < "$temp_file" + +# Remove trailing comma from the last entry +files_list="${files_list%,}" + +# Create the final JSON structure +# Header line +header_line="{ \"file-series-version\": \"1.0\", \"files\": [" +# Write the files list +all_files="$(printf '%s\n' "$files_list") ] }" + +file_series_data="$header_line" +file_series_data+=$'\n' +file_series_data+="$all_files" + +# Write the generated JSON structure to a file named plot_files.series +echo "$file_series_data" > plot_files.series + +# Remove the temporary file +rm "$temp_file" + +echo "JSON structure has been written to plot_files.series" diff --git a/Source/Driver/Nyx.H b/Source/Driver/Nyx.H index 0890237a0..51c73b3f0 100644 --- a/Source/Driver/Nyx.H +++ b/Source/Driver/Nyx.H @@ -251,6 +251,7 @@ public: // amrex::Real get_comoving_a(amrex::Real time); static void integrate_time_given_a(const amrex::Real a0, const amrex::Real a1, amrex::Real& dt); + static void integrate_distance_given_a(const amrex::Real a0, const amrex::Real a1, amrex::Real& dt); void integrate_comoving_a(const amrex::Real old_a, amrex::Real& new_a, const amrex::Real dt); void integrate_comoving_a(amrex::Real time, amrex::Real dt); @@ -595,6 +596,15 @@ int integrate_state_struct #ifdef REEBER void halo_find(amrex::Real dt, amrex::Vector& reeber_halos); void halo_print(amrex::Vector& reeber_halos); + void SwapEnd(float& val); + void writeHaloBinaryVTK(const std::string& filename_vtk, + const amrex::Vector& reeber_halos, + const amrex::Real& radius_outer, + const amrex::Real& radius_inner); + void writeHaloSimpleBinary(const std::string& filename_bin, + const amrex::Vector& reeber_halos, + const amrex::Real& radius_outer, + const amrex::Real& radius_inner); #endif void agn_halo_find(amrex::Real dt); void agn_halo_merge(); @@ -757,6 +767,7 @@ int integrate_state_struct static NeutrinoParticleContainer* theGhostNPC(); #endif #endif + static DarkMatterParticleContainer* theShellPC(); #endif static int NUM_STATE; @@ -986,6 +997,11 @@ protected: // There can be only one forcing object (Fourier space): static class StochasticForcing *forcing; + static amrex::Real lightcone_start_z; + static amrex::Real lightcone_end_z; + static int write_lightcones_and_halos_vtk; + static int write_lightcones_and_halos_simplebinary; + #ifdef REEBER // // Threshold for halo to create SMBH @@ -996,6 +1012,10 @@ protected: // Seed mass of SMBH // static amrex::Real mass_seed; + + static int min_halo_n_cells; + static amrex::Real halo_component_threshold; + static amrex::Real halo_extrema_threshold; #endif // Previous maximum number of steps for sundials diff --git a/Source/Driver/Nyx.cpp b/Source/Driver/Nyx.cpp index ba44135a2..ddf9a4c5f 100644 --- a/Source/Driver/Nyx.cpp +++ b/Source/Driver/Nyx.cpp @@ -200,10 +200,17 @@ std::string Nyx::enforce_min_density_type = "floor"; Real Nyx:: h_species = 0.76; Real Nyx::he_species = 0.24; +Real Nyx::lightcone_start_z = 7; +Real Nyx::lightcone_end_z = 0; +int Nyx::write_lightcones_and_halos_simplebinary = 0; +int Nyx::write_lightcones_and_halos_vtk = 0; #ifdef REEBER Real Nyx::mass_halo_min = 1.e10; Real Nyx::mass_seed = 1.e5; +Real Nyx::halo_component_threshold = 81.66; +Real Nyx::halo_extrema_threshold = 160.0; +int Nyx::min_halo_n_cells = 10; #endif #ifdef _OPENMP @@ -442,9 +449,17 @@ Nyx::read_params () } pp_nyx.query("gimlet_int", gimlet_int); + pp_nyx.query("lightcone_start_z", lightcone_start_z); + pp_nyx.query("lightcone_end_z", lightcone_end_z); + pp_nyx.query("write_lightcones_and_halos_simplebinary", write_lightcones_and_halos_simplebinary); + pp_nyx.query("write_lightcones_and_halos_vtk", write_lightcones_and_halos_vtk); #ifdef REEBER pp_nyx.query("mass_halo_min", mass_halo_min); pp_nyx.query("mass_seed", mass_seed); + ParmParse pp_reeber("reeber"); + pp_reeber.query("min_halo_n_cells", min_halo_n_cells); + pp_reeber.query("halo_component_threshold", halo_component_threshold); + pp_reeber.query("halo_extrema_threshold", halo_extrema_threshold); #endif } diff --git a/Source/Driver/nyx_main.cpp b/Source/Driver/nyx_main.cpp index ca188e164..886b2a817 100644 --- a/Source/Driver/nyx_main.cpp +++ b/Source/Driver/nyx_main.cpp @@ -41,6 +41,9 @@ std::string inputs_name = ""; #include #endif +#include // C++17 +namespace fs = std::filesystem; + using namespace amrex; const int NyxHaloFinderSignal(42); @@ -140,6 +143,18 @@ nyx_main (int argc, char* argv[]) amrptr->RegridOnly(amrptr->cumTime()); } } +#ifdef REEBER + fs::path output_dir("Output"); + + // Only create the directory structure if "output" does not exist + if (!fs::exists(output_dir)) { + // Create nested directory structure + fs::create_directories("Output/LightCones/VTK"); + fs::create_directories("Output/LightCones/SimpleBinary"); + fs::create_directories("Output/Halos/VTK"); + fs::create_directories("Output/Halos/SimpleBinary"); + } +#endif if (amrptr->okToContinue() && (amrptr->levelSteps(0) < max_step || max_step < 0) diff --git a/Source/IO/Nyx_output.cpp b/Source/IO/Nyx_output.cpp index fdc708329..edeb28c09 100644 --- a/Source/IO/Nyx_output.cpp +++ b/Source/IO/Nyx_output.cpp @@ -1086,9 +1086,31 @@ Nyx::updateInSitu () #endif #ifdef REEBER - amrex::Vector& reeber_halos); + amrex::Vector reeber_halos; halo_find(parent->dtLevel(level), reeber_halos); halo_print(reeber_halos); + const Real prev_time = state[State_Type].prevTime(); + const Real cur_time = state[State_Type].curTime(); + const Real a_old = get_comoving_a(prev_time); + const Real a_new = get_comoving_a(cur_time); + Real z_old = 1/a_old-1; + if(z_old < lightcone_start_z && z_old >= lightcone_end_z) { + std::string filename=Concatenate("reeber_halos_", int(100*(1/a_old-1)), 7); + std::string filename_vtk="./Output/Halos/VTK/"+filename+".vtk"; + std::string filename_bin="./Output/Halos/SimpleBinary/"+filename+".bin"; + Real radius_outer, radius_inner; + integrate_distance_given_a(a_old, 1.0, radius_outer); + integrate_distance_given_a(a_new, 1.0, radius_inner); + if(ParallelDescriptor::IOProcessor()) { + std::cout << "Values of radius outer and inner are " << radius_outer << " " << radius_inner << std::endl; + } + if(write_lightcones_and_halos_vtk) { + writeHaloBinaryVTK(filename_vtk, reeber_halos, radius_outer, radius_inner); + } + if(write_lightcones_and_halos_simplebinary) { + writeHaloSimpleBinary(filename_bin, reeber_halos, radius_outer, radius_inner); + } + } #endif #endif @@ -1271,3 +1293,261 @@ Nyx::blueprint_check_point () } #endif + +#ifdef REEBER + +void +Nyx::SwapEnd(float& val) +{ + // Swap endianess if necessary + char* bytes = reinterpret_cast(&val); + std::swap(bytes[0], bytes[3]); + std::swap(bytes[1], bytes[2]); +} + +void +Nyx::writeHaloBinaryVTK(const std::string& filename_vtk, + const amrex::Vector& reeber_halos, + const Real& radius_outer, + const Real& radius_inner) +{ + + int rank = amrex::ParallelDescriptor::MyProc(); + int size = amrex::ParallelDescriptor::NProcs(); + + amrex::Real halo_mass; + amrex::IntVect halo_pos ; + const auto dx = geom.CellSizeArray(); + + std::vector data; + const Real* prob_lo = geom.ProbLo(); + const Real* prob_hi = geom.ProbHi(); + + amrex::GpuArray center; + for (int i = 0; i < 3; ++i) { + center[i] = 0.5 * (prob_lo[i] + prob_hi[i]); + } + + Real lenx = prob_hi[0]-prob_lo[0]; + Real leny = prob_hi[1]-prob_lo[1]; + Real lenz = prob_hi[2]-prob_lo[2]; + int maxind[3]; + maxind[0] = floor((radius_outer+lenx*0.5)/lenx); + maxind[1] = floor((radius_outer+leny*0.5)/leny); + maxind[2] = floor((radius_outer+lenz*0.5)/lenz); + + Real xlen, ylen, zlen; + + for (const Halo& h : reeber_halos) + { + halo_mass = h.total_mass; + halo_pos = h.position; + + amrex::Real x = (halo_pos[0]+0.5) * dx[0]; + amrex::Real y = (halo_pos[1]+0.5) * dx[1]; + amrex::Real z = (halo_pos[2]+0.5) * dx[2]; + + for(int idir=-maxind[0];idir<=maxind[0];idir++) { + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) { + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) { + xlen = x + idir*lenx - center[0]; + ylen = y + jdir*leny - center[1]; + zlen = z + kdir*lenz - center[2]; + Real rad = sqrt(xlen*xlen+ylen*ylen+zlen*zlen); + + if(rad < radius_inner or rad > radius_outer){ + continue; + } + std::cout << "Found one inside " << rad << " " << radius_inner << " " << radius_outer << std::endl; + // Cast amrex::Real to float if needed (assuming amrex::Real is double) + float xf = static_cast(x + idir*lenx); + float yf = static_cast(y + jdir*leny); + float zf = static_cast(z + kdir*lenz); + + SwapEnd(xf); + SwapEnd(yf); + SwapEnd(zf); + + data.push_back(xf); + data.push_back(yf); + data.push_back(zf); + } + } + } + } // end of loop over creating new particles from halos + + long int local_num_halos = data.size()/3; + long int total_num_halos = local_num_halos; + + // Get total particles across all ranks + amrex::ParallelDescriptor::ReduceLongSum(total_num_halos); + + // Compute offset for this rank's data + size_t offset = 0; + MPI_Exscan(&local_num_halos, &offset, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD); + + // Header handling + size_t header_size = 0; + + if (rank == 0) { + std::ofstream file(filename_vtk, std::ios::binary); + if (!file) { + amrex::Abort("Error: Could not open file " + filename_vtk + "\n"); + } + + // Write the header + file << "# vtk DataFile Version 2.0\n"; + file << "Reeber halos\n"; + file << "BINARY\n"; + file << "DATASET POLYDATA\n"; + file << "POINTS " << total_num_halos << " float\n"; + + // Determine header size + file.seekp(0, std::ios::end); + header_size = file.tellp(); + file.close(); + } + + // Broadcast the header size to all ranks + amrex::ParallelDescriptor::Bcast(&header_size, 1, 0); + + // Use MPI collective I/O for binary data + MPI_File mpi_file; + MPI_File_open(MPI_COMM_WORLD, filename_vtk.c_str(), + MPI_MODE_WRONLY | MPI_MODE_APPEND, MPI_INFO_NULL, &mpi_file); + + // Compute byte offset for this rank + size_t byte_offset = header_size + sizeof(float) * 3 * offset; + + // Write particle data collectively + MPI_File_write_at_all(mpi_file, byte_offset, data.data(), data.size(), MPI_FLOAT, MPI_STATUS_IGNORE); + + MPI_File_close(&mpi_file); + + if (rank == 0) { + std::cout << "Successfully wrote halo VTK file: " << filename_vtk << "\n"; + } +} + +void +Nyx::writeHaloSimpleBinary(const std::string& filename_bin, + const amrex::Vector& reeber_halos, + const Real& radius_outer, + const Real& radius_inner) +{ + int rank = amrex::ParallelDescriptor::MyProc(); + int size = amrex::ParallelDescriptor::NProcs(); + + amrex::Real halo_mass; + amrex::IntVect halo_pos ; + const auto dx = geom.CellSizeArray(); + + std::vector data; + const Real* prob_lo = geom.ProbLo(); + const Real* prob_hi = geom.ProbHi(); + + amrex::GpuArray center; + for (int i = 0; i < 3; ++i) { + center[i] = 0.5 * (prob_lo[i] + prob_hi[i]); + } + + Real lenx = prob_hi[0]-prob_lo[0]; + Real leny = prob_hi[1]-prob_lo[1]; + Real lenz = prob_hi[2]-prob_lo[2]; + int maxind[3]; + maxind[0] = floor((radius_outer+lenx*0.5)/lenx); + maxind[1] = floor((radius_outer+leny*0.5)/leny); + maxind[2] = floor((radius_outer+lenz*0.5)/lenz); + + Real xlen, ylen, zlen; + + for (const Halo& h : reeber_halos) + { + halo_mass = h.total_mass; + halo_pos = h.position; + + amrex::Real x = (halo_pos[0]+0.5) * dx[0]; + amrex::Real y = (halo_pos[1]+0.5) * dx[1]; + amrex::Real z = (halo_pos[2]+0.5) * dx[2]; + + for(int idir=-maxind[0];idir<=maxind[0];idir++) { + for(int jdir=-maxind[1];jdir<=maxind[1];jdir++) { + for(int kdir=-maxind[2];kdir<=maxind[2];kdir++) { + xlen = x + idir*lenx - center[0]; + ylen = y + jdir*leny - center[1]; + zlen = z + kdir*lenz - center[2]; + Real rad = sqrt(xlen*xlen+ylen*ylen+zlen*zlen); + + if(rad < radius_inner or rad > radius_outer){ + continue; + } + std::cout << "Found one inside " << rad << " " << radius_inner << " " << radius_outer << std::endl; + // Cast amrex::Real to float if needed (assuming amrex::Real is double) + float xf = static_cast(x + idir*lenx); + float yf = static_cast(y + jdir*leny); + float zf = static_cast(z + kdir*lenz); + float halo_mass = static_cast(h.total_mass); + float halo_n_cells = static_cast(h.n_cells); + + SwapEnd(xf); + SwapEnd(yf); + SwapEnd(zf); + SwapEnd(halo_mass); + SwapEnd(halo_n_cells); + + data.push_back(xf); + data.push_back(yf); + data.push_back(zf); + data.push_back(halo_mass); + data.push_back(halo_n_cells); + } + } + } + } // end of loop over creating new particles from halos + + long int local_num_halos = data.size()/5; + long int total_num_halos = local_num_halos; + + // Get total particles across all ranks + amrex::ParallelDescriptor::ReduceLongSum(total_num_halos); + + // Compute offset for this rank's data + size_t offset = 0; + MPI_Exscan(&local_num_halos, &offset, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD); + + // Header handling + size_t header_size = 0; + + // Broadcast the header size to all ranks + amrex::ParallelDescriptor::Bcast(&header_size, 1, 0); + + if (rank == 0) { + std::ofstream file(filename_bin, std::ios::binary); + if (!file) { + amrex::Abort("Error: Could not open file " + filename_bin + "\n"); + } + + // Determine header size + file.seekp(0, std::ios::end); + header_size = file.tellp(); + file.close(); + } + + // Use MPI collective I/O for binary data + MPI_File mpi_file; + MPI_File_open(MPI_COMM_WORLD, filename_bin.c_str(), + MPI_MODE_WRONLY | MPI_MODE_APPEND, MPI_INFO_NULL, &mpi_file); + + // Compute byte offset for this rank + size_t byte_offset = header_size + sizeof(float) * 5 * offset; + + // Write particle data collectively + MPI_File_write_at_all(mpi_file, byte_offset, data.data(), data.size(), MPI_FLOAT, MPI_STATUS_IGNORE); + + MPI_File_close(&mpi_file); + + if (rank == 0) { + std::cout << "Successfully wrote SimpleBinary file for Halo: " << filename_bin << "\n"; + } +} +#endif diff --git a/Util/reeber/src/reeber.cpp b/Util/reeber/src/reeber.cpp index 7e193c868..860e1eb6f 100644 --- a/Util/reeber/src/reeber.cpp +++ b/Util/reeber/src/reeber.cpp @@ -159,7 +159,7 @@ void prepare_master_reader( int finest_level, const Vector& level_refinements, const amrex::Geometry geom_in, - std::vector>& pointers_to_copied_data) + std::vector>& pointers_to_copied_data) { std::vector new_state_vars { "density", "xmom", "ymom", "zmom" }; const size_t density_var_idx = 0; @@ -227,7 +227,10 @@ void prepare_master_reader( gid_to_fab_size[gid] = core_fab_size; pointers_to_copied_data.emplace_back(new Real[core_fab_size]); - Real* core_fab_ptr = pointers_to_copied_data.back().get(); + // Real* core_fab_ptr = pointers_to_copied_data.back().get(); + + // HACK: use raw pointer to avoid double free + Real* core_fab_ptr = pointers_to_copied_data.back().get(); GridRef core_grid_ref(core_fab_ptr, core_shape, false); @@ -266,12 +269,16 @@ void prepare_master_reader( // reserve memory for dm_density pointers_to_copied_data.emplace_back(new Real[core_fab_size]); - Real* extra_ptr_copy = pointers_to_copied_data.back().get(); + // Real* extra_ptr_copy = pointers_to_copied_data.back().get(); + Real* extra_ptr_copy = pointers_to_copied_data.back().get(); + + // HACK: use raw pointer to avoid double free extra_pointers.push_back(extra_ptr_copy); // reserve memory for variables in new_state for(int i = 0; i < new_state_vars.size(); ++i) { - Real* extra_ptr_copy = new Real[core_fab_size]; + pointers_to_copied_data.emplace_back(new Real[core_fab_size]); + extra_ptr_copy = pointers_to_copied_data.back().get(); extra_pointers.push_back(extra_ptr_copy); } @@ -389,7 +396,8 @@ std::vector compute_halos(diy::mpi::communicator& world, diy::DiscreteBounds diy_domain, Real absolute_rho, bool negate, - Real min_halo_n_cells) + Real min_halo_n_cells, + Real halo_extrema_threshold) { bool debug = false; //world.rank() == 0; std::string prefix = "./DIY.XXXXXX"; @@ -430,7 +438,7 @@ std::vector compute_halos(diy::mpi::communicator& world, new Block(b->fab, b->extra_names_, b->extra_fabs_, local_ref, local_lev, diy_domain, l->bounds(), l->core(), cp.gid(), - new_link, absolute_rho, negate, /*absolute = */ true, cell_volume), + new_link, absolute_rho, negate, /* wrap = */ true, /*absolute = */ true, cell_volume), new_link); //if (debug) fmt::print(std::cerr, "Added block gid = {}\n", cp.gid()); @@ -461,9 +469,9 @@ std::vector compute_halos(diy::mpi::communicator& world, } - master.foreach([](Block* b, const diy::Master::ProxyWithLink& cp) { + master.foreach([halo_extrema_threshold](Block* b, const diy::Master::ProxyWithLink& cp) { b->compute_final_connected_components(); - b->compute_local_integral(); + b->compute_local_integral(halo_extrema_threshold); }); //if (debug) fmt::print(std::cerr, "Local integrals computed"); @@ -571,7 +579,8 @@ void Nyx::runReeberAnalysis(Vector& new_state, // store pointers to all dynamically allocated arrays, so that // data will be freed automatically after exiting compute_halos - std::vector> pointers_to_copied_data; + // std::vector> pointers_to_copied_data; + std::vector> pointers_to_copied_data; diy::mpi::communicator world = ParallelDescriptor::Communicator(); @@ -591,9 +600,7 @@ void Nyx::runReeberAnalysis(Vector& new_state, diy::DiscreteBounds diy_domain(3); - // TODO: take rho, min_halo_n_cells as parameters - Real min_halo_n_cells = 10; - Real rho = 81.66; + Real rho = halo_component_threshold; Real absolute_rho = (Nyx::average_dm_density + Nyx::average_gas_density) * rho; bool negate = true; // sweep superlevel sets, highest density = root @@ -612,7 +619,7 @@ void Nyx::runReeberAnalysis(Vector& new_state, BL_PROFILE_VAR("Nyx::runReeberAnalysis()::compute_halos",compute_halos_var); - reeber_halos = compute_halos(world, master_reader, geom_in, threads, diy_domain, absolute_rho, negate, min_halo_n_cells); + reeber_halos = compute_halos(world, master_reader, geom_in, threads, diy_domain, absolute_rho, negate, min_halo_n_cells, halo_extrema_threshold); if (verbose and world.rank() == 0) fmt::print(std::cerr, "compute_halos finished, result.size = {}\n", reeber_halos.size()); BL_PROFILE_VAR_STOP(compute_halos_var); diff --git a/Util/reeber/src/reeber.cpp.bkp b/Util/reeber/src/reeber.cpp.bkp new file mode 100644 index 000000000..bfc449898 --- /dev/null +++ b/Util/reeber/src/reeber.cpp.bkp @@ -0,0 +1,626 @@ +#include +#include + +#include "Nyx.H" +#include "reeber.H" + +#define REEBER_EXTRA_INTEGRAL +#define REEBER_COMPUTE_GAS_VELOCITIES + +// Reeber and DIY includes +#include + + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + + +using namespace amrex; + +// block-independent types +using AMRLink = diy::AMRLink; + +using Bounds = diy::DiscreteBounds; +using AmrVertexId = reeber::AmrVertexId; +using AmrEdge = reeber::AmrEdge; + +using FabBlockR = FabBlock; + +using Block = FabComponentBlock; +using Vertex = Block::Vertex; +using Component = Block::Component; +using MaskedBox = Block::MaskedBox; +using GidVector = Block::GidVector; +using GidSet = Block::GidSet; + +using TripletMergeTree = Block::TripletMergeTree; +using Neighbor = TripletMergeTree::Neighbor; + +using Grid = Block::Grid; +using GridRef = Block::GridRef; +using Shape = Block::Shape; + +struct IsAmrVertexLocal +{ + bool operator()(const Block& b, const Neighbor& from) const + { + return from->vertex.gid == b.gid; + } +}; + + +diy::AMRLink::Bounds bounds(const amrex::Box& box) +{ + diy::AMRLink::Bounds bounds(3); + for(int i = 0; i < 3; ++i) { + bounds.min[i] = box.loVect()[i]; + bounds.max[i] = box.hiVect()[i]; + } + return bounds; +} + +void set_wrap(const Box& domain, const Box& valid_box, const std::array& is_periodic, diy::AMRLink* link) +{ + for(int dir_x : {-1, 0, 1}) { + if (!is_periodic[0] && dir_x) continue; + if (dir_x < 0 && valid_box.loVect()[0] != domain.loVect()[0]) continue; + if (dir_x > 0 && valid_box.hiVect()[0] != domain.hiVect()[0]) continue; + + for(int dir_y : {-1, 0, 1}) { + if (!is_periodic[1] && dir_y) continue; + if (dir_y < 0 && valid_box.loVect()[1] != domain.loVect()[1]) continue; + if (dir_y > 0 && valid_box.hiVect()[1] != domain.hiVect()[1]) continue; + for(int dir_z : {-1, 0, 1}) { + if (dir_x == 0 && dir_y == 0 && dir_z == 0) + continue; + + if (!is_periodic[2] && dir_z) continue; + if (dir_z < 0 && valid_box.loVect()[2] != domain.loVect()[2]) continue; + if (dir_z > 0 && valid_box.hiVect()[2] != domain.hiVect()[2]) continue; + + link->add_wrap(diy::Direction{dir_x, dir_y, dir_z}); + } + } + } +} + +void set_neighbors(int level, int finest_level, const std::vector& gid_offsets, const std::vector& refinements, + const Box& domain, Vector >& particle_mf, const Box& valid_box, + const std::array& is_periodic, diy::AMRLink* link) +{ + for(int nbr_lev = std::max(0, level - 1); nbr_lev <= std::min(finest_level, level + 1); ++nbr_lev) { + // gotta do this yoga to work around AMReX's static variables + + Box nbr_lev_domain = domain; + nbr_lev_domain.refine(refinements[level]); + + Periodicity periodicity(IntVect(AMREX_D_DECL(nbr_lev_domain.length(0) * is_periodic[0], + nbr_lev_domain.length(1) * is_periodic[1], + nbr_lev_domain.length(2) * is_periodic[2]))); + + const std::vector& pshifts = periodicity.shiftIntVect(); + int ng = 0; + const BoxArray& ba = particle_mf[nbr_lev]->boxArray(); + // TODO: check! + int ratio; + if (nbr_lev < level) { + ratio = refinements[level] / refinements[nbr_lev]; + } else { + ratio = refinements[nbr_lev] / refinements[level]; + } + + Box gbx = valid_box; + if (nbr_lev < level) + gbx.coarsen(ratio); + else if (nbr_lev > level) + gbx.refine(ratio); + gbx.grow(1); + + std::vector> isects; + + for(const auto& piv : pshifts) { + ba.intersections(gbx + piv, isects); + for(const auto& is : isects) { + // is.first is the index of neighbor box + // ba[is.first] is the neighbor box + int nbr_gid = gid_offsets.at(nbr_lev) + is.first; + const Box& nbr_box = ba[is.first]; + Box nbr_ghost_box = grow(nbr_box, ng); + link->add_neighbor(diy::BlockID{nbr_gid, + -1}); // we don't know the proc, but we'll figure it out later through DynamicAssigner + // we ignore ghost data, hence nbr_box in last two parameter + link->add_bounds(nbr_lev, refinements.at(nbr_lev), bounds(nbr_box), bounds(nbr_box)); + } + } + } // loop to set neighbors +} + + +void prepare_master_reader( + diy::Master& master_reader, + diy::MemoryBuffer& header, + diy::DiscreteBounds& diy_domain, + Vector& new_state, + Vector >& particle_mf, + int finest_level, + const Vector& level_refinements, + const amrex::Geometry geom_in, + std::vector& pointers_to_copied_data) +{ + std::vector new_state_vars { "density", "xmom", "ymom", "zmom" }; + const size_t density_var_idx = 0; + + std::vector all_vars { "particle_mass_density", "density", "xmom", "ymom", "zmom" }; + + bool debug = false; + const int n_levels = finest_level + 1; + + std::array is_periodic; + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + is_periodic[i] = geom_in.isPeriodic(i); + } + + const Box& domain = geom_in.Domain(); + + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + diy_domain.min[i] = domain.loVect()[i]; + diy_domain.max[i] = domain.hiVect()[i]; + } + + std::vector gid_offsets = {0}; + std::vector refinements = {1}; + int nblocks = 0; + + // iterate over all levels to collect refinements and box array sizes (=gid offsets) + for(int level = 0; level < n_levels; ++level) { + const MultiFab& mf = *new_state[level]; + + BoxArray ba = mf.boxArray(); + nblocks += ba.size(); + gid_offsets.push_back(nblocks); + + if (level > 0) { + // we accumulate in refinements ratio to the coarsest level + // assuming uniform refinement ratio in all dimensions + // level_refinements contains fine_ratio + refinements.push_back(refinements.back() * level_refinements[level][0]); + } + } + + //fmt::print("REFINEMENTS = {}\n", container_to_string(refinements)); + + std::map gid_to_fab; + std::map gid_to_fab_size; + std::map> gid_to_extra_pointers; + + for(int level = 0; level < n_levels; ++level) { + const MultiFab& dm_mf = *particle_mf[level]; + const BoxArray ba = dm_mf.boxArray(); + + // false is for no tiling in MFIter; we want boxes exactly as they are in plotfile + for(MFIter mfi(dm_mf, false); mfi.isValid(); ++mfi) { + const FArrayBox& dm_fab = dm_mf[mfi]; + int gid = gid_offsets[level] + mfi.index(); + + + // core stuff + const Box& core_box = mfi.validbox(); + Shape core_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { + core_shape[i] = core_box.bigEnd()[i] - core_box.smallEnd()[i] + 1; + } + long long int core_fab_size = core_box.numPts(); + gid_to_fab_size[gid] = core_fab_size; + + pointers_to_copied_data.emplace_back(new Real[core_fab_size]); + // Real* core_fab_ptr = pointers_to_copied_data.back().get(); + + // HACK: use raw pointer to avoid double free + Real* core_fab_ptr = pointers_to_copied_data.back(); + + GridRef core_grid_ref(core_fab_ptr, core_shape, false); + + // ghost stuff + Box ghost_box = dm_fab.box(); + Shape ghost_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { + ghost_shape[i] = ghost_box.bigEnd()[i] - ghost_box.smallEnd()[i] + 1; + } + // this grid ref points directly to dm_fab data + // index 0, since dm_fab contains only dark matter density + GridRef ghost_grid_ref(const_cast(dm_fab.dataPtr(0)), ghost_shape, /* c_order = */ false); + + Shape core_ghost_adjustment; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { + core_ghost_adjustment[i] = core_box.smallEnd()[i] - ghost_box.smallEnd()[i]; + if (core_ghost_adjustment[i] < 0) { throw std::runtime_error("ghost box must be larger than core box"); } + } + + // copy particle data in the core to core_grid_ref + // reason: dm_fab has 2 ghosts, new_state has 1, and we don't need + // ghosts + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, core_ghost_adjustment](const Vertex& v) { + core_grid_ref(v) = ghost_grid_ref(v + core_ghost_adjustment); + }); + + std::vector> isects; + diy::AMRLink* link = new diy::AMRLink(3, level, refinements[level], bounds(core_box), bounds(core_box)); + + + //if (debug) fmt::print(std::cerr, "Added box, particle_mf ghost_box = {}, core_box = {}, core_fab_size = {}\n", ghost_box, core_box, core_fab_size); + + // allocate memory for all fields that we store in FabBlock + // actual copying for next fields will happen later + std::vector extra_pointers; + + // reserve memory for dm_density + pointers_to_copied_data.emplace_back(new Real[core_fab_size]); + // Real* extra_ptr_copy = pointers_to_copied_data.back().get(); + + // HACK: use raw pointer to avoid double free + Real* extra_ptr_copy = pointers_to_copied_data.back(); + extra_pointers.push_back(extra_ptr_copy); + + // reserve memory for variables in new_state + for(int i = 0; i < new_state_vars.size(); ++i) { + Real* extra_ptr_copy = new Real[core_fab_size]; + extra_pointers.push_back(extra_ptr_copy); + } + + gid_to_fab[gid] = core_fab_ptr; + gid_to_extra_pointers[gid] = extra_pointers; + + // copy dark matter density to extra_data + memcpy(extra_pointers[0], core_fab_ptr, sizeof(Real) * core_fab_size); + + if (all_vars.size() != extra_pointers.size()) + throw std::runtime_error("all_vars.size() != extra_pointers.size()"); + + master_reader.add(gid, new FabBlockR(core_fab_ptr, all_vars, extra_pointers, core_shape), link); + + set_wrap(domain, core_box, is_periodic, link); + + set_neighbors(level, finest_level, gid_offsets, refinements, domain, particle_mf, core_box, is_periodic, link); + } // loop over tiles + } // loop over levels for dark matter + + // now FabBlocks are created, it remains to add gas density + // and copy velocities to FabBlocks + for(int level = 0; level < n_levels; ++level) { + const MultiFab& state_mf = *new_state[level]; + // false is for no tiling in MFIter; we want boxes exactly as they are + for(MFIter mfi(state_mf, false); mfi.isValid(); ++mfi) { + const FArrayBox& state_fab = state_mf[mfi]; + + int gid = gid_offsets[level] + mfi.index(); + long long int fab_size = gid_to_fab_size.at(gid); + + //fmt::print(std::cerr, "new_state_vars, level = {}, gid = {}, fab_size = {}, mfi.index = {}, gid[offsets[level] = {}\n", + //level, gid, fab_size, mfi.index(), gid_offsets[level]); + + // core box and shape + const Box& core_box = mfi.validbox(); + if (core_box.numPts() != fab_size) { throw std::runtime_error("shape mismatch state_mf != particle_mf"); } + Shape core_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { core_shape[i] = core_box.length(i); } + + // ghost box and shape + Box ghost_box = state_fab.box(); + Shape ghost_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { ghost_shape[i] = ghost_box.length(i); } + + Shape core_ghost_adjustment; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { core_ghost_adjustment[i] = core_box.smallEnd()[i] - ghost_box.smallEnd()[i]; } + + //if (debug) fmt::print(std::cerr, "state_fab ghost_box = {}, core_box = {}, state_mf.contains_nan = {} (grow = 0)\n", + // ghost_box, core_box, state_mf.contains_nan(0, new_state_vars.size(), 0, false)); + + size_t gas_density_idx = 0; + + for(int var_idx = 0; var_idx < new_state_vars.size(); ++var_idx) { + + bool add_to_fab = (var_idx == 0); + + // this grid ref points directly to dm_fab data + GridRef ghost_grid_ref(const_cast(state_fab.dataPtr(var_idx)), ghost_shape, /* c_order = */ false); + + Real* block_extra_ptr = gid_to_extra_pointers.at(gid).at(var_idx); + GridRef core_grid_ref(block_extra_ptr, core_shape, false); + + if (add_to_fab) { + // density - copy to extra_data as is, add to main grid + + // copy core data to block_extra_ptr + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, core_ghost_adjustment](const Vertex& v) { + core_grid_ref(v) = ghost_grid_ref(v + core_ghost_adjustment); + }); + + //if (debug) fmt::print(std::cerr, "copied variable {} to extra_pointers, add_to_fab = {}\n", + // new_state_vars[var_idx], add_to_fab); + + Real* block_fab_ptr = gid_to_fab.at(gid); + GridRef core_grid_ref(block_extra_ptr, core_shape, false); + // add core density data to block_fab_ptr + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, core_ghost_adjustment](const Vertex& v) { + core_grid_ref(v) += ghost_grid_ref(v + core_ghost_adjustment); + }); + + } else { + // momenta - divide by density + + GridRef density_grid_ref(const_cast(state_fab.dataPtr(density_var_idx)), ghost_shape, /* c_order = */ false); + + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, &density_grid_ref, core_ghost_adjustment](const Vertex& v) { + auto ghost_v = v + core_ghost_adjustment; + core_grid_ref(v) = ghost_grid_ref(ghost_v) / density_grid_ref(ghost_v); + }); + + //if (debug) fmt::print(std::cerr, "copied {} to extra_pointers, divided by density\n", + // new_state_vars[var_idx], add_to_fab); + } // if - density or momenta + } // loop over vars + } // loop over tiles + } // loop over levels for new state + + // fill dynamic assigner and fix links + diy::DynamicAssigner assigner(master_reader.communicator(), master_reader.communicator().size(), nblocks); + diy::fix_links(master_reader, assigner); + + master_reader.foreach([](FabBlockR* b, const diy::Master::ProxyWithLink& cp) { + auto* l = static_cast(cp.link()); + auto receivers = link_unique(l, cp.gid()); + } + ); +} + + +std::vector compute_halos(diy::mpi::communicator& world, + diy::Master& master_reader, + const amrex::Geometry geom_in, + int threads, + diy::DiscreteBounds diy_domain, + Real absolute_rho, + bool negate, + Real min_halo_n_cells, + Real halo_extrema_threshold) +{ + bool debug = false; //world.rank() == 0; + std::string prefix = "./DIY.XXXXXX"; + int in_memory = -1; + std::string log_level = "info"; + diy::FileStorage storage(prefix); + + std::vector result; + + //if (debug) fmt::print(std::cerr, "Master reader - started\n"); + + // copy FabBlocks to FabComponentBlocks + // in FabTmtConstructor mask will be set and local trees will be computed + // FabBlock can be safely discarded afterwards + diy::Master master(world, threads, in_memory, &Block::create, &Block::destroy, &storage, &Block::save, + &Block::load); + + //if (debug) fmt::print(std::cerr, "Master reader created\n"); + + Real cell_volume = geom_in.ProbDomain().volume() / geom_in.Domain().numPts(); + + + //if (debug) fmt::print(std::cerr, "Cell volume = {}\n", cell_volume); + + master_reader.foreach( + [&master, debug, diy_domain, absolute_rho, negate, cell_volume](FabBlockR* b, const diy::Master::ProxyWithLink& cp) { + auto* l = static_cast(cp.link()); + AMRLink* new_link = new AMRLink(*l); + + // prepare neighbor box info to save in MaskedBox + // TODO: refinement vector + int local_ref = l->refinement()[0]; + int local_lev = l->level(); + + //if (debug) fmt::print(std::cerr, "adding block, bounds = {}, core = {}, gid = {}\n", l->bounds(), l->core(), cp.gid()); + + master.add(cp.gid(), + new Block(b->fab, b->extra_names_, b->extra_fabs_, local_ref, local_lev, diy_domain, + l->bounds(), + l->core(), cp.gid(), + new_link, absolute_rho, negate, /* wrap = */ true, /*absolute = */ true, cell_volume), + new_link); + + //if (debug) fmt::print(std::cerr, "Added block gid = {}\n", cp.gid()); + }); + + //if (debug) fmt::print(std::cerr, "Master populated\n"); + + int global_n_undone = 1; + + master.foreach(&send_edges_to_neighbors_cc); + master.exchange(); + master.foreach(&delete_low_edges_cc); + + //if (debug) fmt::print(std::cerr, "Low edges deleted\n"); + + int rounds = 0; + while(global_n_undone) { + rounds++; + + master.foreach(&amr_cc_send); + master.exchange(); + master.foreach(&amr_cc_receive); + master.exchange(); + + global_n_undone = master.proxy(master.loaded_block()).read(); + + //if (debug) fmt::print(std::cerr, "global_n_undone = {}\n", global_n_undone); + + } + + master.foreach([halo_extrema_threshold](Block* b, const diy::Master::ProxyWithLink& cp) { + b->compute_final_connected_components(); + b->compute_local_integral(halo_extrema_threshold); + }); + + //if (debug) fmt::print(std::cerr, "Local integrals computed"); + + amrex::Box domain = geom_in.Domain(); + RealBox prob_domain = geom_in.ProbDomain(); + + master.foreach( + [&result, domain, prob_domain, min_halo_n_cells]( + Block* b, + const diy::Master::ProxyWithLink& cp) { + + diy::Point domain_shape; + Real domain_volume = domain.numPts(); + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + domain_shape[i] = domain.length(i); + } + + diy::GridRef domain_box(nullptr, domain_shape, /* c_order = */ false); + + for(auto& root_values_pair : b->local_integral_) { + AmrVertexId root = root_values_pair.first; + if (root.gid != b->gid) + continue; + + auto& values = root_values_pair.second; + + Real halo_n_cells = values.at("n_cells"); + + if (halo_n_cells < min_halo_n_cells) + continue; + + Halo new_halo; + + const Real cell_volume = b->cell_volume_; + + new_halo.n_cells = halo_n_cells; + new_halo.volume = (Real)halo_n_cells * cell_volume; + new_halo.gas_mass = cell_volume * values["density"]; + new_halo.dm_mass = cell_volume * values["particle_mass_density"]; + new_halo.total_mass = new_halo.gas_mass + new_halo.dm_mass; + + // division by density was done in prepare_master_reader, + // it remains to multiply with volume to get velocities + new_halo.gas_vel_x = cell_volume * values["xmom"]; + new_halo.gas_vel_y = cell_volume * values["ymom"]; + new_halo.gas_vel_z = cell_volume * values["zmom"]; + + auto halo_root_position = b->local_.global_position(root); + + int ref = b->refinement(); + amrex::IntVect ref_ratio (ref); + amrex::Box domain_at_level = domain; + domain_at_level.refine(ref_ratio); + + new_halo.id = domain_box.index(halo_root_position); + + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + new_halo.position[i] = halo_root_position[i]; + + new_halo.real_position[i] = prob_domain.lo(i) + + prob_domain.length(i) * (Real)(halo_root_position[i]) / domain_at_level.length(i); + } + result.push_back(new_halo); + } + }); + + return result; +} + + + +/* // Global components of state +int Nyx::Density = -1; +int Nyx::Eden = -1; +int Nyx::Eint = -1; +int Nyx::Xmom = -1; +int Nyx::Ymom = -1; +int Nyx::Zmom = -1;*/ + +/* // Global density values + average_gas_density; + average_dm_density; + average_neutr_density; + average_total_density;*/ + +void Nyx::runReeberAnalysis(Vector& new_state, + Vector >& particle_mf, + const amrex::Geometry geom_in, + const Vector& level_refinements, + int n_step, + bool do_analysis, + std::vector& reeber_halos) +{ + if (!do_analysis) + return; + + verbose = true; + + BL_PROFILE("Nyx::runReeberAnalysis()"); + + if(verbose) { + amrex::Print() << "Running Reeber anaylsis" << std::endl; + } + + // store pointers to all dynamically allocated arrays, so that + // data will be freed automatically after exiting compute_halos + // std::vector> pointers_to_copied_data; + std::vector pointers_to_copied_data; // HACK: avoid double free + + diy::mpi::communicator world = ParallelDescriptor::Communicator(); + + int threads = 1; + std::string prefix = "./DIY.XXXXXX"; + int in_memory = -1; + diy::FileStorage storage(prefix); + + diy::MemoryBuffer header; + + BL_PROFILE_VAR("Nyx::runReeberAnalysis()::master_reader",master_reader_var); + + diy::Master master_reader(world, threads, in_memory, &FabBlockR::create, &FabBlockR::destroy, + &storage, &FabBlockR::save, &FabBlockR::load); + + BL_PROFILE_VAR_STOP(master_reader_var); + + diy::DiscreteBounds diy_domain(3); + + Real rho = halo_component_threshold; + + Real absolute_rho = (Nyx::average_dm_density + Nyx::average_gas_density) * rho; + bool negate = true; // sweep superlevel sets, highest density = root + int finest_level = parent->finestLevel(); + + //if (verbose and world.rank() == 0) fmt::print(std::cerr, "fmt prepare_master_reader called, finest_level = {}, rho = {}\n", finest_level, absolute_rho); + + BL_PROFILE_VAR("Nyx::runReeberAnalysis()::prepare_master_reader",prepare_master_reader_var); + + prepare_master_reader(master_reader, header, diy_domain, new_state, particle_mf, + finest_level, level_refinements, geom_in, pointers_to_copied_data); + + BL_PROFILE_VAR_STOP(prepare_master_reader_var); + + //if (verbose and world.rank() == 0) fmt::print(std::cerr, "prepare_master_reader finished\n"); + + BL_PROFILE_VAR("Nyx::runReeberAnalysis()::compute_halos",compute_halos_var); + + reeber_halos = compute_halos(world, master_reader, geom_in, threads, diy_domain, absolute_rho, negate, min_halo_n_cells, halo_extrema_threshold); + if (verbose and world.rank() == 0) fmt::print(std::cerr, "compute_halos finished, result.size = {}\n", reeber_halos.size()); + + BL_PROFILE_VAR_STOP(compute_halos_var); + +} // runReeberAnalysis diff --git a/Util/reeber/src/reeber.cpp.new b/Util/reeber/src/reeber.cpp.new new file mode 100644 index 000000000..860e1eb6f --- /dev/null +++ b/Util/reeber/src/reeber.cpp.new @@ -0,0 +1,627 @@ +#include +#include + +#include "Nyx.H" +#include "reeber.H" + +#define REEBER_EXTRA_INTEGRAL +#define REEBER_COMPUTE_GAS_VELOCITIES + +// Reeber and DIY includes +#include + + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + + +using namespace amrex; + +// block-independent types +using AMRLink = diy::AMRLink; + +using Bounds = diy::DiscreteBounds; +using AmrVertexId = reeber::AmrVertexId; +using AmrEdge = reeber::AmrEdge; + +using FabBlockR = FabBlock; + +using Block = FabComponentBlock; +using Vertex = Block::Vertex; +using Component = Block::Component; +using MaskedBox = Block::MaskedBox; +using GidVector = Block::GidVector; +using GidSet = Block::GidSet; + +using TripletMergeTree = Block::TripletMergeTree; +using Neighbor = TripletMergeTree::Neighbor; + +using Grid = Block::Grid; +using GridRef = Block::GridRef; +using Shape = Block::Shape; + +struct IsAmrVertexLocal +{ + bool operator()(const Block& b, const Neighbor& from) const + { + return from->vertex.gid == b.gid; + } +}; + + +diy::AMRLink::Bounds bounds(const amrex::Box& box) +{ + diy::AMRLink::Bounds bounds(3); + for(int i = 0; i < 3; ++i) { + bounds.min[i] = box.loVect()[i]; + bounds.max[i] = box.hiVect()[i]; + } + return bounds; +} + +void set_wrap(const Box& domain, const Box& valid_box, const std::array& is_periodic, diy::AMRLink* link) +{ + for(int dir_x : {-1, 0, 1}) { + if (!is_periodic[0] && dir_x) continue; + if (dir_x < 0 && valid_box.loVect()[0] != domain.loVect()[0]) continue; + if (dir_x > 0 && valid_box.hiVect()[0] != domain.hiVect()[0]) continue; + + for(int dir_y : {-1, 0, 1}) { + if (!is_periodic[1] && dir_y) continue; + if (dir_y < 0 && valid_box.loVect()[1] != domain.loVect()[1]) continue; + if (dir_y > 0 && valid_box.hiVect()[1] != domain.hiVect()[1]) continue; + for(int dir_z : {-1, 0, 1}) { + if (dir_x == 0 && dir_y == 0 && dir_z == 0) + continue; + + if (!is_periodic[2] && dir_z) continue; + if (dir_z < 0 && valid_box.loVect()[2] != domain.loVect()[2]) continue; + if (dir_z > 0 && valid_box.hiVect()[2] != domain.hiVect()[2]) continue; + + link->add_wrap(diy::Direction{dir_x, dir_y, dir_z}); + } + } + } +} + +void set_neighbors(int level, int finest_level, const std::vector& gid_offsets, const std::vector& refinements, + const Box& domain, Vector >& particle_mf, const Box& valid_box, + const std::array& is_periodic, diy::AMRLink* link) +{ + for(int nbr_lev = std::max(0, level - 1); nbr_lev <= std::min(finest_level, level + 1); ++nbr_lev) { + // gotta do this yoga to work around AMReX's static variables + + Box nbr_lev_domain = domain; + nbr_lev_domain.refine(refinements[level]); + + Periodicity periodicity(IntVect(AMREX_D_DECL(nbr_lev_domain.length(0) * is_periodic[0], + nbr_lev_domain.length(1) * is_periodic[1], + nbr_lev_domain.length(2) * is_periodic[2]))); + + const std::vector& pshifts = periodicity.shiftIntVect(); + int ng = 0; + const BoxArray& ba = particle_mf[nbr_lev]->boxArray(); + // TODO: check! + int ratio; + if (nbr_lev < level) { + ratio = refinements[level] / refinements[nbr_lev]; + } else { + ratio = refinements[nbr_lev] / refinements[level]; + } + + Box gbx = valid_box; + if (nbr_lev < level) + gbx.coarsen(ratio); + else if (nbr_lev > level) + gbx.refine(ratio); + gbx.grow(1); + + std::vector> isects; + + for(const auto& piv : pshifts) { + ba.intersections(gbx + piv, isects); + for(const auto& is : isects) { + // is.first is the index of neighbor box + // ba[is.first] is the neighbor box + int nbr_gid = gid_offsets.at(nbr_lev) + is.first; + const Box& nbr_box = ba[is.first]; + Box nbr_ghost_box = grow(nbr_box, ng); + link->add_neighbor(diy::BlockID{nbr_gid, + -1}); // we don't know the proc, but we'll figure it out later through DynamicAssigner + // we ignore ghost data, hence nbr_box in last two parameter + link->add_bounds(nbr_lev, refinements.at(nbr_lev), bounds(nbr_box), bounds(nbr_box)); + } + } + } // loop to set neighbors +} + + +void prepare_master_reader( + diy::Master& master_reader, + diy::MemoryBuffer& header, + diy::DiscreteBounds& diy_domain, + Vector& new_state, + Vector >& particle_mf, + int finest_level, + const Vector& level_refinements, + const amrex::Geometry geom_in, + std::vector>& pointers_to_copied_data) +{ + std::vector new_state_vars { "density", "xmom", "ymom", "zmom" }; + const size_t density_var_idx = 0; + + std::vector all_vars { "particle_mass_density", "density", "xmom", "ymom", "zmom" }; + + bool debug = false; + const int n_levels = finest_level + 1; + + std::array is_periodic; + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + is_periodic[i] = geom_in.isPeriodic(i); + } + + const Box& domain = geom_in.Domain(); + + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + diy_domain.min[i] = domain.loVect()[i]; + diy_domain.max[i] = domain.hiVect()[i]; + } + + std::vector gid_offsets = {0}; + std::vector refinements = {1}; + int nblocks = 0; + + // iterate over all levels to collect refinements and box array sizes (=gid offsets) + for(int level = 0; level < n_levels; ++level) { + const MultiFab& mf = *new_state[level]; + + BoxArray ba = mf.boxArray(); + nblocks += ba.size(); + gid_offsets.push_back(nblocks); + + if (level > 0) { + // we accumulate in refinements ratio to the coarsest level + // assuming uniform refinement ratio in all dimensions + // level_refinements contains fine_ratio + refinements.push_back(refinements.back() * level_refinements[level][0]); + } + } + + //fmt::print("REFINEMENTS = {}\n", container_to_string(refinements)); + + std::map gid_to_fab; + std::map gid_to_fab_size; + std::map> gid_to_extra_pointers; + + for(int level = 0; level < n_levels; ++level) { + const MultiFab& dm_mf = *particle_mf[level]; + const BoxArray ba = dm_mf.boxArray(); + + // false is for no tiling in MFIter; we want boxes exactly as they are in plotfile + for(MFIter mfi(dm_mf, false); mfi.isValid(); ++mfi) { + const FArrayBox& dm_fab = dm_mf[mfi]; + int gid = gid_offsets[level] + mfi.index(); + + + // core stuff + const Box& core_box = mfi.validbox(); + Shape core_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { + core_shape[i] = core_box.bigEnd()[i] - core_box.smallEnd()[i] + 1; + } + long long int core_fab_size = core_box.numPts(); + gid_to_fab_size[gid] = core_fab_size; + + pointers_to_copied_data.emplace_back(new Real[core_fab_size]); + // Real* core_fab_ptr = pointers_to_copied_data.back().get(); + + // HACK: use raw pointer to avoid double free + Real* core_fab_ptr = pointers_to_copied_data.back().get(); + + GridRef core_grid_ref(core_fab_ptr, core_shape, false); + + // ghost stuff + Box ghost_box = dm_fab.box(); + Shape ghost_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { + ghost_shape[i] = ghost_box.bigEnd()[i] - ghost_box.smallEnd()[i] + 1; + } + // this grid ref points directly to dm_fab data + // index 0, since dm_fab contains only dark matter density + GridRef ghost_grid_ref(const_cast(dm_fab.dataPtr(0)), ghost_shape, /* c_order = */ false); + + Shape core_ghost_adjustment; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { + core_ghost_adjustment[i] = core_box.smallEnd()[i] - ghost_box.smallEnd()[i]; + if (core_ghost_adjustment[i] < 0) { throw std::runtime_error("ghost box must be larger than core box"); } + } + + // copy particle data in the core to core_grid_ref + // reason: dm_fab has 2 ghosts, new_state has 1, and we don't need + // ghosts + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, core_ghost_adjustment](const Vertex& v) { + core_grid_ref(v) = ghost_grid_ref(v + core_ghost_adjustment); + }); + + std::vector> isects; + diy::AMRLink* link = new diy::AMRLink(3, level, refinements[level], bounds(core_box), bounds(core_box)); + + + //if (debug) fmt::print(std::cerr, "Added box, particle_mf ghost_box = {}, core_box = {}, core_fab_size = {}\n", ghost_box, core_box, core_fab_size); + + // allocate memory for all fields that we store in FabBlock + // actual copying for next fields will happen later + std::vector extra_pointers; + + // reserve memory for dm_density + pointers_to_copied_data.emplace_back(new Real[core_fab_size]); + // Real* extra_ptr_copy = pointers_to_copied_data.back().get(); + Real* extra_ptr_copy = pointers_to_copied_data.back().get(); + + // HACK: use raw pointer to avoid double free + extra_pointers.push_back(extra_ptr_copy); + + // reserve memory for variables in new_state + for(int i = 0; i < new_state_vars.size(); ++i) { + pointers_to_copied_data.emplace_back(new Real[core_fab_size]); + extra_ptr_copy = pointers_to_copied_data.back().get(); + extra_pointers.push_back(extra_ptr_copy); + } + + gid_to_fab[gid] = core_fab_ptr; + gid_to_extra_pointers[gid] = extra_pointers; + + // copy dark matter density to extra_data + memcpy(extra_pointers[0], core_fab_ptr, sizeof(Real) * core_fab_size); + + if (all_vars.size() != extra_pointers.size()) + throw std::runtime_error("all_vars.size() != extra_pointers.size()"); + + master_reader.add(gid, new FabBlockR(core_fab_ptr, all_vars, extra_pointers, core_shape), link); + + set_wrap(domain, core_box, is_periodic, link); + + set_neighbors(level, finest_level, gid_offsets, refinements, domain, particle_mf, core_box, is_periodic, link); + } // loop over tiles + } // loop over levels for dark matter + + // now FabBlocks are created, it remains to add gas density + // and copy velocities to FabBlocks + for(int level = 0; level < n_levels; ++level) { + const MultiFab& state_mf = *new_state[level]; + // false is for no tiling in MFIter; we want boxes exactly as they are + for(MFIter mfi(state_mf, false); mfi.isValid(); ++mfi) { + const FArrayBox& state_fab = state_mf[mfi]; + + int gid = gid_offsets[level] + mfi.index(); + long long int fab_size = gid_to_fab_size.at(gid); + + //fmt::print(std::cerr, "new_state_vars, level = {}, gid = {}, fab_size = {}, mfi.index = {}, gid[offsets[level] = {}\n", + //level, gid, fab_size, mfi.index(), gid_offsets[level]); + + // core box and shape + const Box& core_box = mfi.validbox(); + if (core_box.numPts() != fab_size) { throw std::runtime_error("shape mismatch state_mf != particle_mf"); } + Shape core_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { core_shape[i] = core_box.length(i); } + + // ghost box and shape + Box ghost_box = state_fab.box(); + Shape ghost_shape; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { ghost_shape[i] = ghost_box.length(i); } + + Shape core_ghost_adjustment; + for(size_t i = 0; i < AMREX_SPACEDIM; ++i) { core_ghost_adjustment[i] = core_box.smallEnd()[i] - ghost_box.smallEnd()[i]; } + + //if (debug) fmt::print(std::cerr, "state_fab ghost_box = {}, core_box = {}, state_mf.contains_nan = {} (grow = 0)\n", + // ghost_box, core_box, state_mf.contains_nan(0, new_state_vars.size(), 0, false)); + + size_t gas_density_idx = 0; + + for(int var_idx = 0; var_idx < new_state_vars.size(); ++var_idx) { + + bool add_to_fab = (var_idx == 0); + + // this grid ref points directly to dm_fab data + GridRef ghost_grid_ref(const_cast(state_fab.dataPtr(var_idx)), ghost_shape, /* c_order = */ false); + + Real* block_extra_ptr = gid_to_extra_pointers.at(gid).at(var_idx); + GridRef core_grid_ref(block_extra_ptr, core_shape, false); + + if (add_to_fab) { + // density - copy to extra_data as is, add to main grid + + // copy core data to block_extra_ptr + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, core_ghost_adjustment](const Vertex& v) { + core_grid_ref(v) = ghost_grid_ref(v + core_ghost_adjustment); + }); + + //if (debug) fmt::print(std::cerr, "copied variable {} to extra_pointers, add_to_fab = {}\n", + // new_state_vars[var_idx], add_to_fab); + + Real* block_fab_ptr = gid_to_fab.at(gid); + GridRef core_grid_ref(block_extra_ptr, core_shape, false); + // add core density data to block_fab_ptr + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, core_ghost_adjustment](const Vertex& v) { + core_grid_ref(v) += ghost_grid_ref(v + core_ghost_adjustment); + }); + + } else { + // momenta - divide by density + + GridRef density_grid_ref(const_cast(state_fab.dataPtr(density_var_idx)), ghost_shape, /* c_order = */ false); + + diy::for_each(core_shape, [&core_grid_ref, &ghost_grid_ref, &density_grid_ref, core_ghost_adjustment](const Vertex& v) { + auto ghost_v = v + core_ghost_adjustment; + core_grid_ref(v) = ghost_grid_ref(ghost_v) / density_grid_ref(ghost_v); + }); + + //if (debug) fmt::print(std::cerr, "copied {} to extra_pointers, divided by density\n", + // new_state_vars[var_idx], add_to_fab); + } // if - density or momenta + } // loop over vars + } // loop over tiles + } // loop over levels for new state + + // fill dynamic assigner and fix links + diy::DynamicAssigner assigner(master_reader.communicator(), master_reader.communicator().size(), nblocks); + diy::fix_links(master_reader, assigner); + + master_reader.foreach([](FabBlockR* b, const diy::Master::ProxyWithLink& cp) { + auto* l = static_cast(cp.link()); + auto receivers = link_unique(l, cp.gid()); + } + ); +} + + +std::vector compute_halos(diy::mpi::communicator& world, + diy::Master& master_reader, + const amrex::Geometry geom_in, + int threads, + diy::DiscreteBounds diy_domain, + Real absolute_rho, + bool negate, + Real min_halo_n_cells, + Real halo_extrema_threshold) +{ + bool debug = false; //world.rank() == 0; + std::string prefix = "./DIY.XXXXXX"; + int in_memory = -1; + std::string log_level = "info"; + diy::FileStorage storage(prefix); + + std::vector result; + + //if (debug) fmt::print(std::cerr, "Master reader - started\n"); + + // copy FabBlocks to FabComponentBlocks + // in FabTmtConstructor mask will be set and local trees will be computed + // FabBlock can be safely discarded afterwards + diy::Master master(world, threads, in_memory, &Block::create, &Block::destroy, &storage, &Block::save, + &Block::load); + + //if (debug) fmt::print(std::cerr, "Master reader created\n"); + + Real cell_volume = geom_in.ProbDomain().volume() / geom_in.Domain().numPts(); + + + //if (debug) fmt::print(std::cerr, "Cell volume = {}\n", cell_volume); + + master_reader.foreach( + [&master, debug, diy_domain, absolute_rho, negate, cell_volume](FabBlockR* b, const diy::Master::ProxyWithLink& cp) { + auto* l = static_cast(cp.link()); + AMRLink* new_link = new AMRLink(*l); + + // prepare neighbor box info to save in MaskedBox + // TODO: refinement vector + int local_ref = l->refinement()[0]; + int local_lev = l->level(); + + //if (debug) fmt::print(std::cerr, "adding block, bounds = {}, core = {}, gid = {}\n", l->bounds(), l->core(), cp.gid()); + + master.add(cp.gid(), + new Block(b->fab, b->extra_names_, b->extra_fabs_, local_ref, local_lev, diy_domain, + l->bounds(), + l->core(), cp.gid(), + new_link, absolute_rho, negate, /* wrap = */ true, /*absolute = */ true, cell_volume), + new_link); + + //if (debug) fmt::print(std::cerr, "Added block gid = {}\n", cp.gid()); + }); + + //if (debug) fmt::print(std::cerr, "Master populated\n"); + + int global_n_undone = 1; + + master.foreach(&send_edges_to_neighbors_cc); + master.exchange(); + master.foreach(&delete_low_edges_cc); + + //if (debug) fmt::print(std::cerr, "Low edges deleted\n"); + + int rounds = 0; + while(global_n_undone) { + rounds++; + + master.foreach(&amr_cc_send); + master.exchange(); + master.foreach(&amr_cc_receive); + master.exchange(); + + global_n_undone = master.proxy(master.loaded_block()).read(); + + //if (debug) fmt::print(std::cerr, "global_n_undone = {}\n", global_n_undone); + + } + + master.foreach([halo_extrema_threshold](Block* b, const diy::Master::ProxyWithLink& cp) { + b->compute_final_connected_components(); + b->compute_local_integral(halo_extrema_threshold); + }); + + //if (debug) fmt::print(std::cerr, "Local integrals computed"); + + amrex::Box domain = geom_in.Domain(); + RealBox prob_domain = geom_in.ProbDomain(); + + master.foreach( + [&result, domain, prob_domain, min_halo_n_cells]( + Block* b, + const diy::Master::ProxyWithLink& cp) { + + diy::Point domain_shape; + Real domain_volume = domain.numPts(); + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + domain_shape[i] = domain.length(i); + } + + diy::GridRef domain_box(nullptr, domain_shape, /* c_order = */ false); + + for(auto& root_values_pair : b->local_integral_) { + AmrVertexId root = root_values_pair.first; + if (root.gid != b->gid) + continue; + + auto& values = root_values_pair.second; + + Real halo_n_cells = values.at("n_cells"); + + if (halo_n_cells < min_halo_n_cells) + continue; + + Halo new_halo; + + const Real cell_volume = b->cell_volume_; + + new_halo.n_cells = halo_n_cells; + new_halo.volume = (Real)halo_n_cells * cell_volume; + new_halo.gas_mass = cell_volume * values["density"]; + new_halo.dm_mass = cell_volume * values["particle_mass_density"]; + new_halo.total_mass = new_halo.gas_mass + new_halo.dm_mass; + + // division by density was done in prepare_master_reader, + // it remains to multiply with volume to get velocities + new_halo.gas_vel_x = cell_volume * values["xmom"]; + new_halo.gas_vel_y = cell_volume * values["ymom"]; + new_halo.gas_vel_z = cell_volume * values["zmom"]; + + auto halo_root_position = b->local_.global_position(root); + + int ref = b->refinement(); + amrex::IntVect ref_ratio (ref); + amrex::Box domain_at_level = domain; + domain_at_level.refine(ref_ratio); + + new_halo.id = domain_box.index(halo_root_position); + + for(int i = 0; i < AMREX_SPACEDIM; ++i) { + new_halo.position[i] = halo_root_position[i]; + + new_halo.real_position[i] = prob_domain.lo(i) + + prob_domain.length(i) * (Real)(halo_root_position[i]) / domain_at_level.length(i); + } + result.push_back(new_halo); + } + }); + + return result; +} + + + +/* // Global components of state +int Nyx::Density = -1; +int Nyx::Eden = -1; +int Nyx::Eint = -1; +int Nyx::Xmom = -1; +int Nyx::Ymom = -1; +int Nyx::Zmom = -1;*/ + +/* // Global density values + average_gas_density; + average_dm_density; + average_neutr_density; + average_total_density;*/ + +void Nyx::runReeberAnalysis(Vector& new_state, + Vector >& particle_mf, + const amrex::Geometry geom_in, + const Vector& level_refinements, + int n_step, + bool do_analysis, + std::vector& reeber_halos) +{ + if (!do_analysis) + return; + + verbose = true; + + BL_PROFILE("Nyx::runReeberAnalysis()"); + + if(verbose) { + amrex::Print() << "Running Reeber anaylsis" << std::endl; + } + + // store pointers to all dynamically allocated arrays, so that + // data will be freed automatically after exiting compute_halos + // std::vector> pointers_to_copied_data; + std::vector> pointers_to_copied_data; + + diy::mpi::communicator world = ParallelDescriptor::Communicator(); + + int threads = 1; + std::string prefix = "./DIY.XXXXXX"; + int in_memory = -1; + diy::FileStorage storage(prefix); + + diy::MemoryBuffer header; + + BL_PROFILE_VAR("Nyx::runReeberAnalysis()::master_reader",master_reader_var); + + diy::Master master_reader(world, threads, in_memory, &FabBlockR::create, &FabBlockR::destroy, + &storage, &FabBlockR::save, &FabBlockR::load); + + BL_PROFILE_VAR_STOP(master_reader_var); + + diy::DiscreteBounds diy_domain(3); + + Real rho = halo_component_threshold; + + Real absolute_rho = (Nyx::average_dm_density + Nyx::average_gas_density) * rho; + bool negate = true; // sweep superlevel sets, highest density = root + int finest_level = parent->finestLevel(); + + //if (verbose and world.rank() == 0) fmt::print(std::cerr, "fmt prepare_master_reader called, finest_level = {}, rho = {}\n", finest_level, absolute_rho); + + BL_PROFILE_VAR("Nyx::runReeberAnalysis()::prepare_master_reader",prepare_master_reader_var); + + prepare_master_reader(master_reader, header, diy_domain, new_state, particle_mf, + finest_level, level_refinements, geom_in, pointers_to_copied_data); + + BL_PROFILE_VAR_STOP(prepare_master_reader_var); + + //if (verbose and world.rank() == 0) fmt::print(std::cerr, "prepare_master_reader finished\n"); + + BL_PROFILE_VAR("Nyx::runReeberAnalysis()::compute_halos",compute_halos_var); + + reeber_halos = compute_halos(world, master_reader, geom_in, threads, diy_domain, absolute_rho, negate, min_halo_n_cells, halo_extrema_threshold); + if (verbose and world.rank() == 0) fmt::print(std::cerr, "compute_halos finished, result.size = {}\n", reeber_halos.size()); + + BL_PROFILE_VAR_STOP(compute_halos_var); + +} // runReeberAnalysis