CUFFTError

[Swarnali-D]

I simulated a code for 256 x 256 x 256 resolution on GPU which ran fine. But when I tried to run the same code for 512 x 512 x 512 resolution, the following error showed up:

ERROR: LoadError: CUFFTError: driver or internal cuFFT library error (code 5, CUFFT_INTERNAL_ERROR)
Stacktrace:
  [1] throw_api_error(res::CUDA.CUFFT.cufftResult_t)
    @ CUDA.CUFFT ~/.julia/packages/CUDA/rXson/lib/cufft/libcufft.jl:11
  [2] check
    @ ~/.julia/packages/CUDA/rXson/lib/cufft/libcufft.jl:21 [inlined]
  [3] cufftMakePlanMany(plan::Int32, rank::Int64, n::Vector{Int32}, inembed::Ptr{Nothing}, istride::Int64, idist::Int64, onembed::Ptr{Nothing}, ostride::Int64, odist::Int64, type::CUDA.CUFFT.cufftType_t, batch::Int64, workSize::Base.RefValue{UInt64})
    @ CUDA.CUFFT ~/.julia/packages/CUDA/rXson/lib/utils/call.jl:26
  [4] cufftMakePlan(xtype::CUDA.CUFFT.cufftType_t, xdims::Tuple{Int64, Int64, Int64}, region::Tuple{Int64, Int64})
    @ CUDA.CUFFT ~/.julia/packages/CUDA/rXson/lib/cufft/wrappers.jl:47
  [5] #71
    @ ~/.julia/packages/CUDA/rXson/lib/cufft/wrappers.jl:162 [inlined]
  [6] (::CUDA.APIUtils.var"#10#13"{CUDA.CUFFT.var"#71#72"{Tuple{CUDA.CUFFT.cufftType_t, Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64}}}, CUDA.APIUtils.HandleCache{Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Vararg{Int64, N}} where N, Any}, Int32}, Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64}}})()
    @ CUDA.APIUtils ~/.julia/packages/CUDA/rXson/lib/utils/cache.jl:28
  [7] lock(f::CUDA.APIUtils.var"#10#13"{CUDA.CUFFT.var"#71#72"{Tuple{CUDA.CUFFT.cufftType_t, Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64}}}, CUDA.APIUtils.HandleCache{Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Vararg{Int64, N}} where N, Any}, Int32}, Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64}}}, l::ReentrantLock)
    @ Base ./lock.jl:229
  [8] (::CUDA.APIUtils.var"#check_cache#11"{CUDA.APIUtils.HandleCache{Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Vararg{Int64, N}} where N, Any}, Int32}, Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64}}})(f::CUDA.CUFFT.var"#71#72"{Tuple{CUDA.CUFFT.cufftType_t, Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64}}})
    @ CUDA.APIUtils ~/.julia/packages/CUDA/rXson/lib/utils/cache.jl:26
  [9] pop!(f::Function, cache::CUDA.APIUtils.HandleCache{Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Vararg{Int64, N}} where N, Any}, Int32}, key::Tuple{CuContext, CUDA.CUFFT.cufftType_t, Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64}})
    @ CUDA.APIUtils ~/.julia/packages/CUDA/rXson/lib/utils/cache.jl:47
 [10] cufftGetPlan(::CUDA.CUFFT.cufftType_t, ::Vararg{Any})
    @ CUDA.CUFFT ~/.julia/packages/CUDA/rXson/lib/cufft/wrappers.jl:160
 [11] plan_bfft!(X::CuArray{ComplexF64, 3, CUDA.Mem.DeviceBuffer}, region::Vector{Int64})
    @ CUDA.CUFFT ~/.julia/packages/CUDA/rXson/lib/cufft/fft.jl:193
 [12] plan_ifft!(x::CuArray{ComplexF64, 3, CUDA.Mem.DeviceBuffer}, region::Vector{Int64}; kws::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ AbstractFFTs ~/.julia/packages/AbstractFFTs/4iQz5/src/definitions.jl:284
 [13] plan_ifft!
    @ ~/.julia/packages/AbstractFFTs/4iQz5/src/definitions.jl:284 [inlined]
 [14] plan_backward_transform
    @ ~/.julia/packages/Oceananigans/r28zw/src/Solvers/plan_transforms.jl:43 [inlined]
 [15] plan_transforms(grid::RectilinearGrid{Float64, Periodic, Periodic, Bounded, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, GPU}, storage::CuArray{ComplexF64, 3, CUDA.Mem.DeviceBuffer}, planner_flag::UInt32)
    @ Oceananigans.Solvers ~/.julia/packages/Oceananigans/r28zw/src/Solvers/plan_transforms.jl:134
 [16] Oceananigans.Solvers.FFTBasedPoissonSolver(grid::RectilinearGrid{Float64, Periodic, Periodic, Bounded, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, GPU}, planner_flag::UInt32)
    @ Oceananigans.Solvers ~/.julia/packages/Oceananigans/r28zw/src/Solvers/fft_based_poisson_solver.jl:65
 [17] FFTBasedPoissonSolver
    @ ~/.julia/packages/Oceananigans/r28zw/src/Solvers/fft_based_poisson_solver.jl:51 [inlined]
 [18] PressureSolver(arch::GPU, grid::RectilinearGrid{Float64, Periodic, Periodic, Bounded, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, GPU})
    @ Oceananigans.Models.NonhydrostaticModels ~/.julia/packages/Oceananigans/r28zw/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl:27
 [19] NonhydrostaticModel(; grid::RectilinearGrid{Float64, Periodic, Periodic, Bounded, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, GPU}, clock::Clock{Float64}, advection::WENO{3, Float64, Nothing, Nothing, Nothing, true, Nothing, WENO{2, Float64, Nothing, Nothing, Nothing, true, Nothing, UpwindBiased{1, Float64, Nothing, Nothing, Nothing, Nothing, Centered{1, Float64, Nothing, Nothing, Nothing, Nothing}}, Centered{1, Float64, Nothing, Nothing, Nothing, Nothing}}, Centered{2, Float64, Nothing, Nothing, Nothing, Centered{1, Float64, Nothing, Nothing, Nothing, Nothing}}}, buoyancy::BuoyancyTracer, coriolis::FPlane{Float64}, stokes_drift::UniformStokesDrift{Nothing, typeof(∂z_uˢ), typeof(Oceananigans.StokesDrift.addzero), typeof(Oceananigans.StokesDrift.addzero), typeof(Oceananigans.StokesDrift.addzero)}, forcing::NamedTuple{(:u, :v, :w), Tuple{Relaxation{Float64, typeof(mask_func), Int64}, Relaxation{Float64, typeof(mask_func), Int64}, Relaxation{Float64, typeof(mask_func), Int64}}}, closure::SmagorinskyLilly{Oceananigans.TurbulenceClosures.ExplicitTimeDiscretization, Float64, Float64}, boundary_conditions::NamedTuple{(:u, :b), Tuple{FieldBoundaryConditions{Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Float64}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}}, FieldBoundaryConditions{Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, BoundaryCondition{Oceananigans.BoundaryConditions.Gradient, Float64}, BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Float64}, Oceananigans.BoundaryConditions.DefaultBoundaryCondition{BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}}}}, tracers::Tuple{Symbol, Symbol, Symbol}, timestepper::Symbol, background_fields::NamedTuple{(), Tuple{}}, particles::Nothing, biogeochemistry::Nothing, velocities::Nothing, pressures::Nothing, diffusivity_fields::Nothing, pressure_solver::Nothing, immersed_boundary::Nothing, auxiliary_fields::NamedTuple{(), Tuple{}})
    @ Oceananigans.Models.NonhydrostaticModels ~/.julia/packages/Oceananigans/r28zw/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl:181
 

[glwagner]

That's weird. Not sure we can help but to start, you can paste the code you ran, and the CPU and GPU you are using.