diff --git a/Source/FieldSolver/WarpXSolveFieldsES.cpp b/Source/FieldSolver/WarpXSolveFieldsES.cpp index 6194570cd2d..169e9f5fc49 100644 --- a/Source/FieldSolver/WarpXSolveFieldsES.cpp +++ b/Source/FieldSolver/WarpXSolveFieldsES.cpp @@ -16,6 +16,8 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) { WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); + WARPX_PROFILE_REGION("WarpX::ComputeSpaceChargeField()"); + using ablastr::fields::Direction; using warpx::fields::FieldType; diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index bb8f8b82a15..235a41fe602 100755 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -51,12 +51,15 @@ #ifdef AMREX_USE_EB # include #endif +#include #include #include #include #include +#include "WarpX.H" + namespace ablastr::fields { @@ -265,10 +268,51 @@ computePhi ( const amrex::Real max_norm_b = getMaxNormRho( amrex::GetVecOfConstPtrs(rho), finest_level, absolute_tolerance); - amrex::LPInfo info; + static std::unique_ptr mlmg; + static std::unique_ptr linop; + static std::unique_ptr linop_distributed; - for (int lev=0; lev<=finest_level; lev++) { - amrex::Array const dx_scaled + static int ng = int(grid_type == utils::enums::GridType::Collocated); // ghost cells + + static int solver_type = 0; // 0 = MLMG, 1 = GMRES + + static amrex::BoxArray ba_reducedgrid; + static amrex::DistributionMapping dm_reducedgrid; + + if (!mlmg) { + + amrex::LPInfo info; + + // ensure memory is freed at the end of the program + amrex::ExecOnFinalize([&] () { mlmg.reset(); linop.reset(); linop_distributed.reset();}); + + amrex::ParmParse pp1("lpinfo"); + + int aaa; + if (pp1.query("do_agglomeration",aaa)) { + info.setAgglomeration(aaa); // default true + } + if (pp1.query("agg_grid_size",aaa)) { + info.setAgglomerationGridSize(aaa); // default -1 (architecture-dependent setting) + } + if (pp1.query("do_consolidation",aaa)) { + info.setConsolidation(aaa); // default true + } + if (pp1.query("con_grid_size",aaa)) { + info.setConsolidationGridSize(aaa); // default -1 (architecture-dependent setting) + } + if (pp1.query("con_ratio",aaa)) { + info.setConsolidationRatio(aaa); // default 2 + } + if (pp1.query("con_strategy",aaa)) { + info.setConsolidationStrategy(aaa); // default 3 + } + if (pp1.query("setMaxCoarseningLevel",aaa)) { + info.setMaxCoarseningLevel(aaa); // default 30 + } + + for (int lev=0; lev<=finest_level; lev++) { + amrex::Array const dx_scaled {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; @@ -281,143 +325,278 @@ computePhi ( } else { computePhiIGF( *rho[lev], *phi[lev], dx_scaled, is_igf_2d); } - continue; - } #endif - // Use the Multigrid (MLMG) solver if selected or on refined patches - // but first scale rho appropriately - rho[lev]->mult(-1._rt / ablastr::constant::SI::ep0); #ifdef WARPX_DIM_RZ - constexpr bool is_rz = true; + constexpr bool is_rz = true; #else - constexpr bool is_rz = false; + constexpr bool is_rz = false; #endif - if (!eb_enabled && !is_rz) { - // Determine whether to use semi-coarsening - int max_semicoarsening_level = 0; - int semicoarsening_direction = -1; - const auto min_dir = static_cast(std::distance(dx_scaled.begin(), - std::min_element(dx_scaled.begin(), dx_scaled.end()))); - const auto max_dir = static_cast(std::distance(dx_scaled.begin(), - std::max_element(dx_scaled.begin(), dx_scaled.end()))); - if (dx_scaled[max_dir] > dx_scaled[min_dir]) { - semicoarsening_direction = max_dir; - max_semicoarsening_level = static_cast(std::log2(dx_scaled[max_dir] / dx_scaled[min_dir])); + if (!eb_enabled && !is_rz) { + // Determine whether to use semi-coarsening + int max_semicoarsening_level = 0; + int semicoarsening_direction = -1; + const auto min_dir = static_cast(std::distance(dx_scaled.begin(), + std::min_element(dx_scaled.begin(), dx_scaled.end()))); + const auto max_dir = static_cast(std::distance(dx_scaled.begin(), + std::max_element(dx_scaled.begin(), dx_scaled.end()))); + if (dx_scaled[max_dir] > dx_scaled[min_dir]) { + semicoarsening_direction = max_dir; + max_semicoarsening_level = static_cast(std::log2(dx_scaled[max_dir] / dx_scaled[min_dir])); + } + if (max_semicoarsening_level > 0) { + info.setSemicoarsening(true); + info.setMaxSemicoarseningLevel(max_semicoarsening_level); + info.setSemicoarseningDirection(semicoarsening_direction); + } } - if (max_semicoarsening_level > 0) { - info.setSemicoarsening(true); - info.setMaxSemicoarseningLevel(max_semicoarsening_level); - info.setSemicoarseningDirection(semicoarsening_direction); + + auto & warpx = WarpX::GetInstance(); + + amrex::ParmParse pp2("mlmg"); + int xxx; + amrex::Real yyy; + + amrex::IntVect max_grid_size_poisson(AMREX_D_DECL(8192,8192,8192)); + if (pp2.query("max_grid_size_poisson_x",xxx)) { + max_grid_size_poisson[0] = xxx; } - } + if (pp2.query("max_grid_size_poisson_y",xxx)) { + max_grid_size_poisson[1] = xxx; + } +#if (AMREX_SPACEDIM == 3) + if (pp2.query("max_grid_size_poisson_z",xxx)) { + max_grid_size_poisson[2] = xxx; + } +#endif + + // create a new boxarray with reduced grids + amrex::Box domain_reducedgrid = grids[lev].minimalBox(); + ba_reducedgrid.define(domain_reducedgrid); + ba_reducedgrid.maxSize(max_grid_size_poisson); + + amrex::Print() << "Number of grids in reduced-rank Poisson solve: " << ba_reducedgrid.size() << std::endl; + + // create the distribution map + dm_reducedgrid.define(ba_reducedgrid); + + // don't need to recreate geometry since we can reuse - std::unique_ptr linop; - if (eb_enabled || is_rz) { - // In the presence of EB or RZ: the solver assumes that the beam is - // propagating along one of the axes of the grid, i.e. that only *one* - // of the components of `beta` is non-negligible. - auto linop_nodelap = std::make_unique(); - if (eb_enabled) { + // need std::unique_ptr > + std::unique_ptr > field_factory_reducedgrid; + const int max_guard = warpx.getngEB().max(); + field_factory_reducedgrid = amrex::makeEBFabFactory(geom[lev], ba_reducedgrid, dm_reducedgrid, + {max_guard, max_guard, max_guard}, + amrex::EBSupport::full); + + amrex::Vector factories_reducedgrid; + + amrex::EBFArrayBoxFactory const& fieldEBFactory_reducedgrid = static_cast(*field_factory_reducedgrid); + factories_reducedgrid.push_back(&fieldEBFactory_reducedgrid); + + if (eb_enabled || is_rz) { + // In the presence of EB or RZ: the solver assumes that the beam is + // propagating along one of the axes of the grid, i.e. that only *one* + // of the components of `beta` is non-negligible. + auto linop_nodelap = std::make_unique(); + auto linop_nodelap_distributed = std::make_unique(); + if (eb_enabled) { #if defined(AMREX_USE_EB) - if constexpr(std::is_same_v) { - throw std::runtime_error("EB requested by eb_farray_box_factory not provided!"); - } else { + if constexpr(std::is_same_v) { + throw std::runtime_error("EB requested by eb_farray_box_factory not provided!"); + } else { + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{ba_reducedgrid}, + amrex::Vector{dm_reducedgrid}, + info, + amrex::Vector{factories_reducedgrid} + ); + linop_nodelap_distributed->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info, + amrex::Vector{eb_farray_box_factory.value()[lev]} + ); + } +#endif + } + else { + // TODO: rather use MLNodeTensorLaplacian (for RZ w/o EB) here? Semi-Coarsening would be nice here linop_nodelap->define( amrex::Vector{geom[lev]}, amrex::Vector{grids[lev]}, amrex::Vector{dmap[lev]}, - info, - amrex::Vector{eb_farray_box_factory.value()[lev]} - ); + info + ); } + + // Note: this assumes that the beam is propagating along + // one of the axes of the grid, i.e. that only *one* of the + // components of `beta` is non-negligible. // we use this +#if defined(WARPX_DIM_RZ) + linop_nodelap->setRZ(true); + linop_nodelap->setSigma({0._rt, 1._rt-beta_solver[1]*beta_solver[1]}); + linop_nodelap_distributed->setRZ(true); + linop_nodelap_distributed->setSigma({0._rt, 1._rt-beta_solver[1]*beta_solver[1]}); +#else + linop_nodelap->setSigma({AMREX_D_DECL( + 1._rt-beta_solver[0]*beta_solver[0], + 1._rt-beta_solver[1]*beta_solver[1], + 1._rt-beta_solver[2]*beta_solver[2])}); + linop_nodelap_distributed->setSigma({AMREX_D_DECL( + 1._rt-beta_solver[0]*beta_solver[0], + 1._rt-beta_solver[1]*beta_solver[1], + 1._rt-beta_solver[2]*beta_solver[2])}); #endif - } - else { - // TODO: rather use MLNodeTensorLaplacian (for RZ w/o EB) here? Semi-Coarsening would be nice here - linop_nodelap->define( + linop = std::move(linop_nodelap); + linop_distributed = std::move(linop_nodelap_distributed); + } else { + // In the absence of EB and RZ: use a more generic solver + // that can handle beams propagating in any direction + auto linop_tenslap = std::make_unique( amrex::Vector{geom[lev]}, amrex::Vector{grids[lev]}, amrex::Vector{dmap[lev]}, info - ); + ); + linop_tenslap->setBeta(beta_solver); // for the non-axis-aligned solver + linop = std::move(linop_tenslap); } - // Note: this assumes that the beam is propagating along - // one of the axes of the grid, i.e. that only *one* of the - // components of `beta` is non-negligible. // we use this -#if defined(WARPX_DIM_RZ) - linop_nodelap->setRZ(true); - linop_nodelap->setSigma({0._rt, 1._rt-beta_solver[1]*beta_solver[1]}); -#else - linop_nodelap->setSigma({AMREX_D_DECL( - 1._rt-beta_solver[0]*beta_solver[0], - 1._rt-beta_solver[1]*beta_solver[1], - 1._rt-beta_solver[2]*beta_solver[2])}); -#endif -#if defined(AMREX_USE_EB) - if (eb_enabled) { - if constexpr (!std::is_same_v) { - // if the EB potential only depends on time, the potential can be passed - // as a float instead of a callable - if (boundary_handler.phi_EB_only_t) { - linop_nodelap->setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); - } else { - linop_nodelap->setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); - } - } else - { - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, - "EB Poisson solver enabled but no 'boundary_handler' passed!"); + // Level 0 domain boundary + if constexpr (std::is_same_v) { + amrex::Array const lobc = {AMREX_D_DECL( + amrex::LinOpBCType::Dirichlet, + amrex::LinOpBCType::Dirichlet, + amrex::LinOpBCType::Dirichlet + )}; + amrex::Array const hibc = lobc; + linop->setDomainBC(lobc, hibc); + linop_distributed->setDomainBC(lobc, hibc); + } else { + linop->setDomainBC(boundary_handler.lobc, boundary_handler.hibc); + linop_distributed->setDomainBC(boundary_handler.lobc, boundary_handler.hibc); + } + + // Solve the Poisson equation + mlmg = std::make_unique(*linop); // actual solver defined here + mlmg->setVerbose(verbosity); + mlmg->setMaxIter(max_iters); + mlmg->setConvergenceNormType((max_norm_b > 0) ? amrex::MLMGNormType::bnorm : amrex::MLMGNormType::greater); + + pp2.query("solver_type",solver_type); + if (solver_type == 0) { + amrex::Warning("solver_type=0 not supported for the poissontest 1 GPU feature; switching to solver_type=1"); + solver_type = 1; + } + + if (pp2.query("setPreSmooth",xxx)) { + mlmg->setPreSmooth(xxx); // default 2 + } + if (pp2.query("setPostSmooth",xxx)) { + mlmg->setPostSmooth(xxx); // default 2 + } + if (pp2.query("setFinalSmooth",xxx)) { + mlmg->setFinalSmooth(xxx); // default 8 (when smoother is used as bottom solver) + } + if (pp2.query("setBottomSmooth",xxx)) { + mlmg->setBottomSmooth(xxx); // default 0 + } + if (pp2.query("bottomSolver",xxx)) { + if (xxx == 0) { + mlmg->setBottomSolver(amrex::BottomSolver::Default); // default 0 + } else if (xxx == 1) { + mlmg->setBottomSolver(amrex::BottomSolver::smoother); + } else if (xxx == 2) { + mlmg->setBottomSolver(amrex::BottomSolver::bicgstab); + } else if (xxx == 3) { + mlmg->setBottomSolver(amrex::BottomSolver::cg); + } else if (xxx == 4) { + mlmg->setBottomSolver(amrex::BottomSolver::bicgcg); + } else if (xxx == 5) { + mlmg->setBottomSolver(amrex::BottomSolver::cgbicg); + } else if (xxx == 6) { + mlmg->setBottomSolver(amrex::BottomSolver::hypre); + } else if (xxx == 7) { + mlmg->setBottomSolver(amrex::BottomSolver::petsc); } } -#endif - linop = std::move(linop_nodelap); - } else { - // In the absence of EB and RZ: use a more generic solver - // that can handle beams propagating in any direction - auto linop_tenslap = std::make_unique( - amrex::Vector{geom[lev]}, - amrex::Vector{grids[lev]}, - amrex::Vector{dmap[lev]}, - info - ); - linop_tenslap->setBeta(beta_solver); // for the non-axis-aligned solver - linop = std::move(linop_tenslap); - } + if (pp2.query("setBottomTolerance",yyy)) { + mlmg->setBottomTolerance(yyy); // default 1.e-4 + } + if (pp2.query("setBottomToleranceAbs",yyy)) { + mlmg->setBottomToleranceAbs(yyy); // default -1. + } - // Level 0 domain boundary - if constexpr (std::is_same_v) { - amrex::Array const lobc = {AMREX_D_DECL( - amrex::LinOpBCType::Dirichlet, - amrex::LinOpBCType::Dirichlet, - amrex::LinOpBCType::Dirichlet - )}; - amrex::Array const hibc = lobc; - linop->setDomainBC(lobc, hibc); - } else { - linop->setDomainBC(boundary_handler.lobc, boundary_handler.hibc); + if (ng) { + // In this case, computeE needs to use ghost nodes data. So we + // ask MLMG to fill BC for us after it solves the problem. + mlmg->setFinalFillBC(true); + } } + } - // Solve the Poisson equation - amrex::MLMG mlmg(*linop); // actual solver defined here - mlmg.setVerbose(verbosity); - mlmg.setMaxIter(max_iters); - mlmg.setConvergenceNormType((max_norm_b > 0) ? amrex::MLMGNormType::bnorm : amrex::MLMGNormType::greater); - - const int ng = int(grid_type == utils::enums::GridType::Collocated); // ghost cells - if (ng) { - // In this case, computeE needs to use ghost nodes data. So we - // ask MLMG to fill BC for us after it solves the problem. - mlmg.setFinalFillBC(true); +#if defined(AMREX_USE_EB) + if (eb_enabled) { + if constexpr (!std::is_same_v) { + // if the EB potential only depends on time, the potential can be passed + // as a float instead of a callable + auto* p = dynamic_cast(linop.get()); + auto* p_distributed = dynamic_cast(linop_distributed.get()); + if (boundary_handler.phi_EB_only_t) { + p->setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); + p_distributed->setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); + } else { + p->setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); + p_distributed->setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); + } + } else + { + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "EB Poisson solver enabled but no 'boundary_handler' passed!"); } + } +#endif + for (int lev=0; lev<=finest_level; lev++) { + const auto mlmg_time_beg_step = static_cast(amrex::second()); + + // Use the Multigrid (MLMG) solver if selected or on refined patches + // but first scale rho appropriately + rho[lev]->mult(-1._rt / ablastr::constant::SI::ep0); + + amrex::MultiFab phi_reducedgrid(amrex::convert(ba_reducedgrid, amrex::IntVect::TheUnitVector()), dm_reducedgrid, 1, 1); + amrex::MultiFab rho_reducedgrid(amrex::convert(ba_reducedgrid, amrex::IntVect::TheUnitVector()), dm_reducedgrid, 1, 0); + + phi_reducedgrid.ParallelCopy(*phi[lev], 0, 0, 1, 1, 1); + rho_reducedgrid.ParallelCopy(*rho[lev], 0, 0, 1, 0, 0); // Solve Poisson equation at lev - mlmg.solve( {phi[lev]}, {rho[lev]}, - relative_tolerance, absolute_tolerance ); + if (solver_type == 0) { + mlmg->solve( {&phi_reducedgrid}, {&rho_reducedgrid}, relative_tolerance, absolute_tolerance ); + } else { + amrex::GMRESMLMG gmsolve(*mlmg); + gmsolve.solve( phi_reducedgrid, rho_reducedgrid, relative_tolerance, absolute_tolerance ); + } + linop->postSolve({&phi_reducedgrid}); + + phi_reducedgrid.FillBoundary(geom[lev].periodicity()); + phi[lev]->ParallelCopy(phi_reducedgrid, 0, 0, 1, 1, 1); + +#if 0 + amrex::VisMF::Write(*rho[lev],"rho"); + amrex::VisMF::Write(*phi[lev],"phi"); +#endif const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; + const auto mlmg_time_end_step = static_cast(amrex::second()); + + amrex::Print()<< "MLMG time = " + << mlmg_time_end_step-mlmg_time_beg_step + << " s\n"; + const int ncomp = linop->getNComp(); // needed for solving the levels by levels: @@ -435,14 +614,51 @@ computePhi ( ng); } - // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same_v) { - if (post_phi_calculation.has_value()) { - post_phi_calculation.value()(mlmg, lev); + if (solver_type == 0) { + // Run additional operations, such as calculation of the E field for embedded boundaries + if constexpr (!std::is_same_v) { + if (post_phi_calculation.has_value()) { + post_phi_calculation.value()(*mlmg, lev); + } } } - rho[lev]->mult(-ablastr::constant::SI::ep0); // Multiply rho by epsilon again + // create an alieas to the WarpX Efield + auto & warpx = WarpX::GetInstance(); + + amrex::Vector< amrex::Array > e_field; + e_field.push_back( +#if defined(WARPX_DIM_1D_Z) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, lev) + } +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, lev) + } +#elif defined(WARPX_DIM_3D) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{1}, lev), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, lev) + } +#endif + ); + + if (solver_type == 1) { + static bool called_prepare = false; + if (!called_prepare) { + linop_distributed->prepareForSolve(); + called_prepare = true; + } + linop_distributed->compGrad(0, e_field[0], *phi[0], amrex::MLLinOp::Location::CellCenter); + for(int dir=0; dirmult(-1.); + } + } + + rho[lev]->mult(-ablastr::constant::SI::ep0); // Multiply rho by epsilon again } // loop over lev(els) } // computePhi } // namespace ablastr::fields diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 5119c44d327..ca446370364 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -1,5 +1,5 @@ # please set your project account -export proj="" # change me! GPU projects must end in "..._g" +export proj="mp111_g" # change me! GPU projects must end in "..._g" # remembers the location of this script export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) diff --git a/run_plasma/CPU/inputs.2d_1024cpu b/run_plasma/CPU/inputs.2d_1024cpu new file mode 100644 index 00000000000..880b930a6ad --- /dev/null +++ b/run_plasma/CPU/inputs.2d_1024cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 4096 2048 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/inputs.2d_16384cpu b/run_plasma/CPU/inputs.2d_16384cpu new file mode 100644 index 00000000000..a03622ff40a --- /dev/null +++ b/run_plasma/CPU/inputs.2d_16384cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 16384 8192 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/inputs.2d_256cpu b/run_plasma/CPU/inputs.2d_256cpu new file mode 100644 index 00000000000..c32b7f9b394 --- /dev/null +++ b/run_plasma/CPU/inputs.2d_256cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 2048 1024 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/inputs.2d_4096cpu b/run_plasma/CPU/inputs.2d_4096cpu new file mode 100644 index 00000000000..0efbfef24bc --- /dev/null +++ b/run_plasma/CPU/inputs.2d_4096cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 8192 4096 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/inputs.2d_64cpu b/run_plasma/CPU/inputs.2d_64cpu new file mode 100644 index 00000000000..98cfbd8af90 --- /dev/null +++ b/run_plasma/CPU/inputs.2d_64cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 1024 512 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/inputs.2d_65536cpu b/run_plasma/CPU/inputs.2d_65536cpu new file mode 100644 index 00000000000..488f7efd962 --- /dev/null +++ b/run_plasma/CPU/inputs.2d_65536cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 32768 16384 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/inputs_ultimate_test b/run_plasma/CPU/inputs_ultimate_test new file mode 100644 index 00000000000..4838fa9a5b6 --- /dev/null +++ b/run_plasma/CPU/inputs_ultimate_test @@ -0,0 +1,135 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.solver_type = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 5 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 256 128 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +mlmg.max_grid_size_poisson_x = 256 +mlmg.max_grid_size_poisson_y = 128 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +diagnostics.diags_names = plt +plt.diag_type = Full +plt.intervals = 1 +plt.fields_to_plot = phi +plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/run_1024cpu.sh b/run_plasma/CPU/run_1024cpu.sh new file mode 100644 index 00000000000..bf60a12dc45 --- /dev/null +++ b/run_plasma/CPU/run_1024cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 8 +#SBATCH -C cpu +#SBATCH -q debug +#SBATCH -J 1024cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 1024 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_1024cpu + diff --git a/run_plasma/CPU/run_16384cpu.sh b/run_plasma/CPU/run_16384cpu.sh new file mode 100644 index 00000000000..7bdb031c37b --- /dev/null +++ b/run_plasma/CPU/run_16384cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 128 +#SBATCH -C cpu +#SBATCH -q regular +#SBATCH -J 16384cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 16384 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_16384cpu + diff --git a/run_plasma/CPU/run_256cpu.sh b/run_plasma/CPU/run_256cpu.sh new file mode 100644 index 00000000000..0089374ee25 --- /dev/null +++ b/run_plasma/CPU/run_256cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 2 +#SBATCH -C cpu +#SBATCH -q debug +#SBATCH -J 256cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 256 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_256cpu + diff --git a/run_plasma/CPU/run_4096cpu.sh b/run_plasma/CPU/run_4096cpu.sh new file mode 100644 index 00000000000..08850f2013c --- /dev/null +++ b/run_plasma/CPU/run_4096cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 32 +#SBATCH -C cpu +#SBATCH -q regular +#SBATCH -J 4096cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 4096 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_4096cpu + diff --git a/run_plasma/CPU/run_64cpu.sh b/run_plasma/CPU/run_64cpu.sh new file mode 100644 index 00000000000..cbfe75c15f0 --- /dev/null +++ b/run_plasma/CPU/run_64cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C cpu +#SBATCH -q debug +#SBATCH -J 64cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 64 -c 4 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_64cpu + diff --git a/run_plasma/CPU/run_65536cpu.sh b/run_plasma/CPU/run_65536cpu.sh new file mode 100644 index 00000000000..8801c926fd1 --- /dev/null +++ b/run_plasma/CPU/run_65536cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 512 +#SBATCH -C cpu +#SBATCH -q regular +#SBATCH -J 65536cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 65536 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_65536cpu + diff --git a/run_plasma/GPU/inputs.2d_128gpu b/run_plasma/GPU/inputs.2d_128gpu new file mode 100644 index 00000000000..9cb8f6100d0 --- /dev/null +++ b/run_plasma/GPU/inputs.2d_128gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 8192 4096 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/GPU/inputs.2d_2048gpu b/run_plasma/GPU/inputs.2d_2048gpu new file mode 100644 index 00000000000..e35de2ff27a --- /dev/null +++ b/run_plasma/GPU/inputs.2d_2048gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 32768 16384 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/GPU/inputs.2d_2gpu b/run_plasma/GPU/inputs.2d_2gpu new file mode 100644 index 00000000000..4f342cded91 --- /dev/null +++ b/run_plasma/GPU/inputs.2d_2gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 1024 512 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/GPU/inputs.2d_32gpu b/run_plasma/GPU/inputs.2d_32gpu new file mode 100644 index 00000000000..37ae7e71995 --- /dev/null +++ b/run_plasma/GPU/inputs.2d_32gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 4096 2048 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/GPU/inputs.2d_512gpu b/run_plasma/GPU/inputs.2d_512gpu new file mode 100644 index 00000000000..bd882ece0ac --- /dev/null +++ b/run_plasma/GPU/inputs.2d_512gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 16384 8192 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/GPU/inputs.2d_8gpu b/run_plasma/GPU/inputs.2d_8gpu new file mode 100644 index 00000000000..558e9ee7b54 --- /dev/null +++ b/run_plasma/GPU/inputs.2d_8gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 2048 1024 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/GPU/run_128gpu.sh b/run_plasma/GPU/run_128gpu.sh new file mode 100644 index 00000000000..d6761f4fa36 --- /dev/null +++ b/run_plasma/GPU/run_128gpu.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH -N 32 +#SBATCH -C gpu +#SBATCH -G 128 +#SBATCH -q regular +#SBATCH -J 128gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 128 -c 32 --cpu_bind=cores -G 128 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_128gpu + diff --git a/run_plasma/GPU/run_2048gpu.sh b/run_plasma/GPU/run_2048gpu.sh new file mode 100644 index 00000000000..1d0091ee52f --- /dev/null +++ b/run_plasma/GPU/run_2048gpu.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH -N 512 +#SBATCH -C gpu +#SBATCH -G 2048 +#SBATCH -q regular +#SBATCH -J 2048gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 2048 -c 32 --cpu_bind=cores -G 2048 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_2048gpu + diff --git a/run_plasma/GPU/run_2gpu.sh b/run_plasma/GPU/run_2gpu.sh new file mode 100644 index 00000000000..4c0d6ce3e15 --- /dev/null +++ b/run_plasma/GPU/run_2gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C gpu +#SBATCH -G 2 +#SBATCH -q debug +#SBATCH -J 2gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 2 -c 64 --cpu_bind=cores -G 2 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_2gpu diff --git a/run_plasma/GPU/run_32gpu.sh b/run_plasma/GPU/run_32gpu.sh new file mode 100644 index 00000000000..1afffb1177c --- /dev/null +++ b/run_plasma/GPU/run_32gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 8 +#SBATCH -C gpu +#SBATCH -G 32 +#SBATCH -q debug +#SBATCH -J 32gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 32 -c 32 --cpu_bind=cores -G 32 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_32gpu diff --git a/run_plasma/GPU/run_512gpu.sh b/run_plasma/GPU/run_512gpu.sh new file mode 100644 index 00000000000..39d2b70cd89 --- /dev/null +++ b/run_plasma/GPU/run_512gpu.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH -N 128 +#SBATCH -C gpu +#SBATCH -G 512 +#SBATCH -q regular +#SBATCH -J 512gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 512 -c 32 --cpu_bind=cores -G 512 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_512gpu + diff --git a/run_plasma/GPU/run_8gpu.sh b/run_plasma/GPU/run_8gpu.sh new file mode 100644 index 00000000000..369ceb317bb --- /dev/null +++ b/run_plasma/GPU/run_8gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 2 +#SBATCH -C gpu +#SBATCH -G 8 +#SBATCH -q debug +#SBATCH -J 8gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 8 -c 32 --cpu_bind=cores -G 8 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_8gpu diff --git a/run_plasma/GPU_3d/inputs.3d_2gpu b/run_plasma/GPU_3d/inputs.3d_2gpu new file mode 100644 index 00000000000..21658cac20d --- /dev/null +++ b/run_plasma/GPU_3d/inputs.3d_2gpu @@ -0,0 +1,135 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.solver_type = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 128 128 64 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 128 +amr.max_grid_size_z = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 3 +geometry.prob_lo = -0.1035 -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.1035 0.0527 +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.potential_lo_y = 0. +boundary.potential_hi_y = 0. +boundary.particle_lo = reflecting reflecting reflecting +boundary.particle_hi = reflecting reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = "sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 1 1 1 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 1 1 1 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +diagnostics.diags_names = plt +plt.diag_type = Full +plt.intervals = 1 +plt.fields_to_plot = phi +plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/GPU_3d/run_2gpu.sh b/run_plasma/GPU_3d/run_2gpu.sh new file mode 100644 index 00000000000..6a516142b51 --- /dev/null +++ b/run_plasma/GPU_3d/run_2gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C gpu +#SBATCH -G 2 +#SBATCH -q debug +#SBATCH -J 2gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 2 -c 64 --cpu_bind=cores -G 2 --gpu-bind=none ./warpx.3d.MPI.CUDA.DP.PDP.OPMD.EB inputs.3d_2gpu